outman/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 388691323d into a30945312a

Browse Source
This commit is contained in:
rayshaw001 2025-03-21 15:01:29 +08:00 committed by GitHub
commit 37ad68285b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
api/core/rag/datasource/vdb/vector_factory.py
View File

@ -156,9 +156,16 @@ class Vector:
raise ValueError(f"Vector store {vector_type} is not supported.")
def create(self, texts: Optional[list] = None, **kwargs):
max_batch_documents = 1000
if texts:
embeddings = self._embeddings.embed_documents([document.page_content for document in texts])
self._vector_processor.create(texts=texts, embeddings=embeddings, **kwargs)
for i in range(0, len(texts), max_batch_documents):
batch_documents = texts[i : i + max_batch_documents]
batch_contents = [document.page_content for document in batch_documents]
batch_embeddings = self._embeddings.embed_documents(batch_contents)
if i < max_batch_documents:
self._vector_processor.create(texts=batch_documents, embeddings=batch_embeddings, **kwargs)
else:
self._vector_processor.add_texts(documents=batch_documents, embeddings=batch_embeddings, **kwargs)
def add_texts(self, documents: list[Document], **kwargs):
if kwargs.get("duplicate_check", False):