feat: support tencent vdb

api/.env

@@ -0,0 +1,170 @@
+# Server Edition
+EDITION=SELF_HOSTED
+
+# Your App secret key will be used for securely signing the session cookie
+# Make sure you are changing this key for your deployment with a strong key.
+# You can generate a strong key using `openssl rand -base64 42`.
+# Alternatively you can set it with `SECRET_KEY` environment variable.
+SECRET_KEY=
+
+# Console API base URL
+CONSOLE_API_URL=http://127.0.0.1:5001
+CONSOLE_WEB_URL=http://127.0.0.1:3000
+
+# Service API base URL
+SERVICE_API_URL=http://127.0.0.1:5001
+
+# Web APP base URL
+APP_WEB_URL=http://127.0.0.1:3000
+
+# Files URL
+FILES_URL=http://127.0.0.1:5001
+
+# celery configuration
+CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
+
+# redis configuration
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_USERNAME=
+REDIS_PASSWORD=difyai123456
+REDIS_DB=0
+
+# PostgreSQL database configuration
+DB_USERNAME=postgres
+DB_PASSWORD=difyai123456
+DB_HOST=localhost
+DB_PORT=5432
+DB_DATABASE=dify
+
+# Storage configuration
+# use for store upload files, private keys...
+# storage type: local, s3, azure-blob
+STORAGE_TYPE=local
+STORAGE_LOCAL_PATH=storage
+S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY=your-access-key
+S3_SECRET_KEY=your-secret-key
+S3_REGION=your-region
+# Azure Blob Storage configuration
+AZURE_BLOB_ACCOUNT_NAME=your-account-name
+AZURE_BLOB_ACCOUNT_KEY=your-account-key
+AZURE_BLOB_CONTAINER_NAME=yout-container-name
+AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
+
+# CORS configuration
+WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+
+# Vector database configuration, support: weaviate, qdrant, milvus, relyt
+VECTOR_STORE=tencent
+
+TENCENT_URL=http://10.6.1.224
+TENCENT_API_KEY=nTZEVu0UeShVmMXkMywZQpMLC3BCERM7nLOPH2Xf
+TENCENT_TIMEOUT=30
+TENCENT_USERNAME=root
+TENCENT_DATABASE=dify
+TENCENT_SHARD=1
+TENCENT_REPLICAS=2
+
+# Weaviate configuration
+WEAVIATE_ENDPOINT=http://localhost:8080
+WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+WEAVIATE_GRPC_ENABLED=false
+WEAVIATE_BATCH_SIZE=100
+
+# Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=difyai123456
+QDRANT_CLIENT_TIMEOUT=20
+
+# Milvus configuration
+MILVUS_HOST=127.0.0.1
+MILVUS_PORT=19530
+MILVUS_USER=root
+MILVUS_PASSWORD=Milvus
+MILVUS_SECURE=false
+
+# Relyt configuration
+RELYT_HOST=127.0.0.1
+RELYT_PORT=5432
+RELYT_USER=postgres
+RELYT_PASSWORD=postgres
+RELYT_DATABASE=postgres
+
+# Upload configuration
+UPLOAD_FILE_SIZE_LIMIT=15
+UPLOAD_FILE_BATCH_LIMIT=5
+UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
+
+# Model Configuration
+MULTIMODAL_SEND_IMAGE_FORMAT=base64
+
+# Mail configuration, support: resend, smtp
+MAIL_TYPE=
+MAIL_DEFAULT_SEND_FROM=no-reply <no-reply@dify.ai>
+RESEND_API_KEY=
+RESEND_API_URL=https://api.resend.com
+# smtp configuration
+SMTP_SERVER=smtp.gmail.com
+SMTP_PORT=587
+SMTP_USERNAME=123
+SMTP_PASSWORD=abc
+SMTP_USE_TLS=false
+
+# Sentry configuration
+SENTRY_DSN=
+
+# DEBUG
+DEBUG=false
+SQLALCHEMY_ECHO=false
+
+# Notion import configuration, support public and internal
+NOTION_INTEGRATION_TYPE=public
+NOTION_CLIENT_SECRET=you-client-secret
+NOTION_CLIENT_ID=you-client-id
+NOTION_INTERNAL_SECRET=you-internal-secret
+
+# Hosted Model Credentials
+HOSTED_OPENAI_API_KEY=
+HOSTED_OPENAI_API_BASE=
+HOSTED_OPENAI_API_ORGANIZATION=
+HOSTED_OPENAI_TRIAL_ENABLED=false
+HOSTED_OPENAI_QUOTA_LIMIT=200
+HOSTED_OPENAI_PAID_ENABLED=false
+
+HOSTED_AZURE_OPENAI_ENABLED=false
+HOSTED_AZURE_OPENAI_API_KEY=
+HOSTED_AZURE_OPENAI_API_BASE=
+HOSTED_AZURE_OPENAI_QUOTA_LIMIT=200
+
+HOSTED_ANTHROPIC_API_BASE=
+HOSTED_ANTHROPIC_API_KEY=
+HOSTED_ANTHROPIC_TRIAL_ENABLED=false
+HOSTED_ANTHROPIC_QUOTA_LIMIT=600000
+HOSTED_ANTHROPIC_PAID_ENABLED=false
+
+ETL_TYPE=dify
+UNSTRUCTURED_API_URL=
+
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=
+
+BATCH_UPLOAD_LIMIT=10
+KEYWORD_DATA_SOURCE_TYPE=database
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
+CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_MAX_NUMBER=9223372036854775807
+CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_STRING_LENGTH=80000
+TEMPLATE_TRANSFORM_MAX_LENGTH=80000
+CODE_MAX_STRING_ARRAY_LENGTH=30
+CODE_MAX_OBJECT_ARRAY_LENGTH=30
+CODE_MAX_NUMBER_ARRAY_LENGTH=1000
+
+# API Tool configuration
+API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
+API_TOOL_DEFAULT_READ_TIMEOUT=60

api/.env.example

@@ -85,6 +85,15 @@ RELYT_USER=postgres
 RELYT_PASSWORD=postgres
 RELYT_DATABASE=postgres
 
+# Tencent configuration
+TENCENT_URL=http://127.0.0.1
+TENCENT_API_KEY=dify
+TENCENT_TIMEOUT=30
+TENCENT_USERNAME=dify
+TENCENT_DATABASE=dify
+TENCENT_SHARD=1
+TENCENT_REPLICAS=2
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5

api/commands.py

@@ -305,6 +305,14 @@ def migrate_knowledge_vector_database():
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == "tencent":
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": 'tencent',
+                        "vector_store": {"class_prefix": collection_name}
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
                 else:
                     raise ValueError(f"Vector store {config.get('VECTOR_STORE')} is not supported.")
 

api/config.py

@@ -228,6 +228,15 @@ class Config:
         self.RELYT_PASSWORD = get_env('RELYT_PASSWORD')
         self.RELYT_DATABASE = get_env('RELYT_DATABASE')
 
+        # tencent settings
+        self.TENCENT_URL = get_env('TENCENT_URL')
+        self.TENCENT_API_KEY = get_env('TENCENT_API_KEY')
+        self.TENCENT_TIMEOUT = get_env('TENCENT_TIMEOUT')
+        self.TENCENT_USERNAME = get_env('TENCENT_USERNAME')
+        self.TENCENT_DATABASE = get_env('TENCENT_DATABASE')
+        self.TENCENT_SHARD = get_env('TENCENT_SHARD')
+        self.TENCENT_REPLICAS = get_env('TENCENT_REPLICAS')
+
         # ------------------------
         # Mail Configurations.
         # ------------------------

api/controllers/console/datasets/datasets.py

@@ -475,7 +475,7 @@ class DatasetRetrievalSettingApi(Resource):
                     'semantic_search'
                 ]
             }
-        elif vector_type == 'qdrant' or vector_type == 'weaviate':
+        elif vector_type == 'qdrant' or vector_type == 'weaviate' or vector_type == 'tencent':
             return {
                 'retrieval_method': [
                     'semantic_search', 'full_text_search', 'hybrid_search'
@@ -497,7 +497,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                     'semantic_search'
                 ]
             }
-        elif vector_type == 'qdrant' or vector_type == 'weaviate':
+        elif vector_type == 'qdrant' or vector_type == 'weaviate' or vector_type == 'tencent':
             return {
                 'retrieval_method': [
                     'semantic_search', 'full_text_search', 'hybrid_search'

api/core/rag/datasource/vdb/tencent/tencent_vector.py

@@ -0,0 +1,182 @@
+import json
+from typing import Any, Optional
+
+import tcvectordb
+from pydantic import BaseModel
+from tcvectordb.model import document, enum
+from tcvectordb.model import index as vdb_index
+from tcvectordb.model.document import Filter
+
+from core.rag.datasource.vdb.vector_base import BaseVector
+from core.rag.models.document import Document
+from extensions.ext_redis import redis_client
+
+
+class TencentConfig(BaseModel):
+    url: str
+    api_key: Optional[str]
+    timeout: float = 30
+    username: Optional[str]
+    database: Optional[str]
+    index_type: str = "HNSW"
+    metric_type: str = "L2"
+    shard: int = 1,
+    replicas: int = 2,
+
+    def to_tencent_params(self):
+        return {
+            'url': self.url,
+            'username': self.username,
+            'key': self.api_key,
+            'timeout': self.timeout
+        }
+
+
+class TencentVector(BaseVector):
+    field_id: str = "id"
+    field_vector: str = "vector"
+    field_text: str = "text"
+    field_metadata: str = "metadata"
+
+    def __init__(self, collection_name: str, config: TencentConfig):
+        super().__init__(collection_name)
+        self._client_config = config
+        self._client = tcvectordb.VectorDBClient(**self._client_config.to_tencent_params())
+        self._db = self._init_database()
+
+    def _init_database(self):
+        exists = False
+        for db in self._client.list_databases():
+            if db.database_name == self._client_config.database:
+                exists = True
+                break
+        if exists:
+            return self._client.database(self._client_config.database)
+        else:
+            return self._client.create_database(database_name=self._client_config.database)
+
+    def get_type(self) -> str:
+        return 'tencent'
+
+    def to_index_struct(self) -> dict:
+        return {
+            "type": self.get_type(),
+            "vector_store": {"class_prefix": self._collection_name}
+        }
+
+    def _create_collection(self, dimension: int) -> None:
+        lock_name = 'vector_indexing_lock_{}'.format(self._collection_name)
+        with redis_client.lock(lock_name, timeout=20):
+            self.delete()
+            index_type = None
+            for k, v in enum.IndexType.__members__.items():
+                if k == self._client_config.index_type:
+                    index_type = v
+            if index_type is None:
+                raise ValueError("unsupported index_type")
+            metric_type = None
+            for k, v in enum.MetricType.__members__.items():
+                if k == self._client_config.metric_type:
+                    metric_type = v
+            if metric_type is None:
+                raise ValueError("unsupported metric_type")
+            params = vdb_index.HNSWParams(m=16, efconstruction=200)
+            index = vdb_index.Index(
+                vdb_index.FilterIndex(
+                    self.field_id, enum.FieldType.String, enum.IndexType.PRIMARY_KEY
+                ),
+                vdb_index.VectorIndex(
+                    self.field_vector,
+                    dimension,
+                    index_type,
+                    metric_type,
+                    params,
+                ),
+                vdb_index.FilterIndex(
+                    self.field_text, enum.FieldType.String, enum.IndexType.FILTER
+                ),
+                vdb_index.FilterIndex(
+                    self.field_metadata, enum.FieldType.String, enum.IndexType.FILTER
+                ),
+            )
+
+            self.collection = self._db.create_collection(
+                name=self._collection_name,
+                shard=self._client_config.shard,
+                replicas=self._client_config.replicas,
+                description="Collection for Dify",
+                index=index,
+            )
+
+    def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
+        self._create_collection(len(embeddings[0]))
+        self.add_texts(texts, embeddings)
+
+    def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
+        texts = [doc.page_content for doc in documents]
+        metadatas = [doc.metadata for doc in documents]
+        total_count = len(embeddings)
+        docs = []
+        for id in range(0, total_count):
+            if metadatas is None:
+                continue
+            metadata = json.dumps(metadatas[id])
+            doc = document.Document(
+                id=metadatas[id]["doc_id"],
+                vector=embeddings[id],
+                text=texts[id],
+                metadata=metadata,
+            )
+            docs.append(doc)
+        self.collection.upsert(docs, self._client_config.timeout)
+
+    def text_exists(self, id: str) -> bool:
+        docs = self._db.collection(self._collection_name).query(document_ids=[id])
+        if docs and len(docs) > 0:
+            return True
+        return False
+
+    def delete_by_ids(self, ids: list[str]) -> None:
+        self._db.collection(self._collection_name).delete(document_ids=ids)
+
+    def delete_by_metadata_field(self, key: str, value: str) -> None:
+        docs = self._db.collection(self._collection_name).query(filter=Filter(Filter.In(key, [value])))
+        if docs and len(docs) > 0:
+            self.collection.delete(document_ids=[doc['id'] for doc in docs])
+
+    def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
+
+        res = self._db.collection(self._collection_name).search(vectors=[query_vector],
+                                                                params=document.HNSWSearchParams(
+                                                                    ef=kwargs.get("ef", 10)),
+                                                                retrieve_vector=False,
+                                                                limit=kwargs.get('top_k', 4),
+                                                                timeout=self._client_config.timeout,
+                                                                )
+        return self._get_search_res(res)
+
+    def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
+        res = (self._db.collection(self._collection_name)
+               .searchByText(embeddingItems=[query],
+                             params=document.HNSWSearchParams(ef=kwargs.get("ef", 10)),
+                             retrieve_vector=False,
+                             limit=kwargs.get('top_k', 4),
+                             timeout=self._client_config.timeout,
+                             ))
+        return self._get_search_res(res)
+
+    def _get_search_res(self, res):
+        docs = []
+        if res is None or len(res) == 0:
+            return docs
+
+        for result in res[0]:
+            meta = result.get(self.field_metadata)
+            if meta is not None:
+                meta = json.loads(meta)
+            doc = Document(page_content=result.get(self.field_text), metadata=meta)
+            docs.append(doc)
+        return docs
+
+    def delete(self) -> None:
+        self._db.drop_collection(name=self._collection_name)

api/core/rag/datasource/vdb/vector_factory.py

@@ -25,7 +25,6 @@ class Vector:
     def _init_vector(self) -> BaseVector:
         config = current_app.config
         vector_type = config.get('VECTOR_STORE')
-
         if self._dataset.index_struct_dict:
             vector_type = self._dataset.index_struct_dict['type']
 
@@ -138,6 +137,31 @@ class Vector:
                 ),
                 dim=dim
             )
+        elif vector_type == "tencent":
+            from core.rag.datasource.vdb.tencent.tencent_vector import TencentConfig, TencentVector
+            if self._dataset.index_struct_dict:
+                class_prefix: str = self._dataset.index_struct_dict['vector_store']['class_prefix']
+                collection_name = class_prefix
+            else:
+                dataset_id = self._dataset.id
+                collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                index_struct_dict = {
+                    "type": 'tencent',
+                    "vector_store": {"class_prefix": collection_name}
+                }
+                self._dataset.index_struct = json.dumps(index_struct_dict)
+            return TencentVector(
+                collection_name=collection_name,
+                config=TencentConfig(
+                    url=config.get('TENCENT_URL'),
+                    api_key=config.get('TENCENT_API_KEY'),
+                    timeout=config.get('TENCENT_TIMEOUT'),
+                    username=config.get('TENCENT_USERNAME'),
+                    database=config.get('TENCENT_DATABASE'),
+                    shard=config.get('TENCENT_SHARD'),
+                    replicas=config.get('TENCENT_REPLICAS'),
+                )
+            )
         else:
             raise ValueError(f"Vector store {config.get('VECTOR_STORE')} is not supported.")
 

docker/docker-compose.yaml

@@ -229,6 +229,14 @@ services:
       RELYT_USER: postgres
       RELYT_PASSWORD: difyai123456
       RELYT_DATABASE: postgres
+      # tencent configurations
+      TENCENT_URL: http://127.0.0.1
+      TENCENT_API_KEY: dify
+      TENCENT_TIMEOUT: 30
+      TENCENT_USERNAME: dify
+      TENCENT_DATABASE: dify
+      TENCENT_SHARD: 1
+      TENCENT_REPLICAS: 2
     depends_on:
       - db
       - redis