From b89c9a61c91e3532262bf34b60e6b214f296d401 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Wed, 19 Mar 2025 21:26:08 +0800 Subject: [PATCH] add vdb document id index --- .../rag/datasource/vdb/elasticsearch/elasticsearch_vector.py | 4 +++- api/core/rag/datasource/vdb/field.py | 1 + api/core/rag/datasource/vdb/qdrant/qdrant_vector.py | 4 ++++ .../datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py | 4 ++++ api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py | 2 ++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py index 093368b0cc..9225c10c58 100644 --- a/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py +++ b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py @@ -196,9 +196,11 @@ class ElasticSearchVector(BaseVector): Field.METADATA_KEY.value: { "type": "object", "properties": { - "doc_id": {"type": "keyword"} # Map doc_id to keyword type + "doc_id": {"type": "keyword"}, # Map doc_id to keyword type + "document_id": {"type": "keyword"} # Map doc_id to keyword type }, }, + } } self._client.indices.create(index=self._collection_name, mappings=mappings) diff --git a/api/core/rag/datasource/vdb/field.py b/api/core/rag/datasource/vdb/field.py index a64407bce1..9887e21b7c 100644 --- a/api/core/rag/datasource/vdb/field.py +++ b/api/core/rag/datasource/vdb/field.py @@ -11,3 +11,4 @@ class Field(Enum): TEXT_KEY = "text" PRIMARY_KEY = "id" DOC_ID = "metadata.doc_id" + DOCUMENT_ID = "metadata.document_id" diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 73ce8201fd..4efd90667a 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -134,6 +134,10 @@ class QdrantVector(BaseVector): self._client.create_payload_index( collection_name, Field.DOC_ID.value, field_schema=PayloadSchemaType.KEYWORD ) + # create document_id payload index + self._client.create_payload_index( + collection_name, Field.DOCUMENT_ID.value, field_schema=PayloadSchemaType.KEYWORD + ) # create full text index text_index_params = TextIndexParams( type=TextIndexType.TEXT, diff --git a/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py b/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py index ae4baeb17e..e9353484b6 100644 --- a/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py +++ b/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py @@ -144,6 +144,10 @@ class TidbOnQdrantVector(BaseVector): self._client.create_payload_index( collection_name, Field.DOC_ID.value, field_schema=PayloadSchemaType.KEYWORD ) + # create document_id payload index + self._client.create_payload_index( + collection_name, Field.DOCUMENT_ID.value, field_schema=PayloadSchemaType.KEYWORD + ) # create full text index text_index_params = TextIndexParams( type=TextIndexType.TEXT, diff --git a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py index 77c5786042..efa68059e5 100644 --- a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py +++ b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py @@ -105,10 +105,12 @@ class TiDBVector(BaseVector): text TEXT NOT NULL, meta JSON NOT NULL, doc_id VARCHAR(64) AS (JSON_UNQUOTE(JSON_EXTRACT(meta, '$.doc_id'))) STORED, + document_id VARCHAR(64) AS (JSON_UNQUOTE(JSON_EXTRACT(meta, '$.document_id'))) STORED, vector VECTOR({dimension}) NOT NULL, create_time DATETIME DEFAULT CURRENT_TIMESTAMP, update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, KEY (doc_id), + KEY (document_id), VECTOR INDEX idx_vector (({tidb_dist_func}(vector))) USING HNSW ); """)