improve preview document tokenizer (#13328)

2025-02-07 16:08:25 +08:00 · 2025-02-07 16:08:25 +08:00 · d4a09805a3
commit d4a09805a3
parent 7e1d9894fb
1 changed files with 2 additions and 0 deletions
--- a/api/core/rag/index_processor/processor/parent_child_index_processor.py
+++ b/api/core/rag/index_processor/processor/parent_child_index_processor.py
@ -47,6 +47,8 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                embedding_model_instance=kwargs.get("embedding_model_instance"),
            )
            for document in documents:
+                if kwargs.get("preview") and len(all_documents) >= 10:
+                    return all_documents
                # document clean
                document_text = CleanProcessor.clean(document.page_content, process_rule)
                document.page_content = document_text