outman/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix metadata

Browse Source
This commit is contained in:
jyong 2025-03-10 19:31:53 +08:00
parent 0d2f7dd688
commit 7a8f4bef16
4 changed files with 41 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/core/rag/entities/metadata_entities.py
View File

@ -13,14 +13,14 @@ SupportedComparisonOperator = Literal[
"is not", "is not",
"empty", "empty",
"not empty", "not empty",
# for number # for number
"=", "=",
"≠", "≠",
">", ">",
"<", "<",
"≥", "≥",
"≤", "≤",
# for time # for time
"before", "before",
"after", "after",
] ]

32
api/core/rag/retrieval/dataset_retrieval.py
View File

@ -547,8 +547,16 @@ class DatasetRetrieval:
db.session.add_all(dataset_queries) db.session.add_all(dataset_queries)
db.session.commit() db.session.commit()
def _retriever(self, flask_app: Flask, dataset_id: str, query: str, top_k: int, all_documents: list, def _retriever(
document_ids_filter: Optional[list[str]] = None, metadata_condition: Optional[MetadataCondition] = None): self,
flask_app: Flask,
dataset_id: str,
query: str,
top_k: int,
all_documents: list,
document_ids_filter: Optional[list[str]] = None,
metadata_condition: Optional[MetadataCondition] = None,
):
with flask_app.app_context(): with flask_app.app_context():
dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
@ -822,11 +830,13 @@ class DatasetRetrieval:
self._process_metadata_filter_func( self._process_metadata_filter_func(
filter.get("condition"), filter.get("metadata_name"), filter.get("value"), filters filter.get("condition"), filter.get("metadata_name"), filter.get("value"), filters
) )
conditions.append(Condition( conditions.append(
name=filter.get("metadata_name"), Condition(
comparison_operator=filter.get("condition"), name=filter.get("metadata_name"),
value=filter.get("value"), comparison_operator=filter.get("condition"),
)) value=filter.get("value"),
)
)
metadata_condition = MetadataCondition( metadata_condition = MetadataCondition(
logical_operator=metadata_filtering_conditions.logical_operator, logical_operator=metadata_filtering_conditions.logical_operator,
conditions=conditions, conditions=conditions,
@ -935,12 +945,16 @@ class DatasetRetrieval:
if isinstance(value, str): if isinstance(value, str):
filters.append(DatasetDocument.doc_metadata[metadata_name] == f'"{value}"') filters.append(DatasetDocument.doc_metadata[metadata_name] == f'"{value}"')
else: else:
filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) == value) filters.append(
sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) == value
)
case "is not" | "≠": case "is not" | "≠":
if isinstance(value, str): if isinstance(value, str):
filters.append(DatasetDocument.doc_metadata[metadata_name] != f'"{value}"') filters.append(DatasetDocument.doc_metadata[metadata_name] != f'"{value}"')
else: else:
filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) != value) filters.append(
sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) != value
)
case "is empty": case "is empty":
filters.append(DatasetDocument.doc_metadata[metadata_name].is_(None)) filters.append(DatasetDocument.doc_metadata[metadata_name].is_(None))
case "is not empty": case "is not empty":

18
api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py
View File

@ -304,11 +304,13 @@ class KnowledgeRetrievalNode(LLMNode):
self._process_metadata_filter_func( self._process_metadata_filter_func(
filter.get("condition"), filter.get("metadata_name"), filter.get("value"), filters filter.get("condition"), filter.get("metadata_name"), filter.get("value"), filters
) )
conditions.append(Condition( conditions.append(
name=filter.get("metadata_name"), Condition(
comparison_operator=filter.get("condition"), name=filter.get("metadata_name"),
value=filter.get("value"), comparison_operator=filter.get("condition"),
)) value=filter.get("value"),
)
)
metadata_condition = MetadataCondition( metadata_condition = MetadataCondition(
logical_operator="or", logical_operator="or",
conditions=conditions, conditions=conditions,
@ -321,7 +323,9 @@ class KnowledgeRetrievalNode(LLMNode):
expected_value = condition.value expected_value = condition.value
if expected_value: if expected_value:
if isinstance(expected_value, str): if isinstance(expected_value, str):
expected_value = self.graph_runtime_state.variable_pool.convert_template(expected_value).text expected_value = self.graph_runtime_state.variable_pool.convert_template(
expected_value
).text
filters = self._process_metadata_filter_func( filters = self._process_metadata_filter_func(
condition.comparison_operator, metadata_name, expected_value, filters condition.comparison_operator, metadata_name, expected_value, filters
@ -438,7 +442,7 @@ class KnowledgeRetrievalNode(LLMNode):
case _: case _:
pass pass
return filters return filters
@classmethod @classmethod
def _extract_variable_selector_to_variable_mapping( def _extract_variable_selector_to_variable_mapping(
cls, cls,

6
api/services/external_knowledge_service.py
View File

@ -246,7 +246,11 @@ class ExternalDatasetService:
@staticmethod @staticmethod
def fetch_external_knowledge_retrieval( def fetch_external_knowledge_retrieval(
tenant_id: str, dataset_id: str, query: str, external_retrieval_parameters: dict, metadata_condition: Optional[MetadataCondition] = None tenant_id: str,
dataset_id: str,
query: str,
external_retrieval_parameters: dict,
metadata_condition: Optional[MetadataCondition] = None,
) -> list: ) -> list:
external_knowledge_binding = ExternalKnowledgeBindings.query.filter_by( external_knowledge_binding = ExternalKnowledgeBindings.query.filter_by(
dataset_id=dataset_id, tenant_id=tenant_id dataset_id=dataset_id, tenant_id=tenant_id