outman/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix image files not deleted on indexing_estimate #9541 (#10798)

Browse Source 
Co-authored-by: root <root@localhost.localdomain>
This commit is contained in:
wy96f 2024-11-21 13:03:16 +08:00 committed by GitHub
parent 2ae6460f46
commit 94c9cadbd8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/core/indexing_runner.py
View File

@ -30,6 +30,7 @@ from core.rag.splitter.fixed_text_splitter import (
) )
from core.rag.splitter.text_splitter import TextSplitter from core.rag.splitter.text_splitter import TextSplitter
from core.tools.utils.text_processing_utils import remove_leading_symbols from core.tools.utils.text_processing_utils import remove_leading_symbols
from core.tools.utils.web_reader_tool import get_image_upload_file_ids
from extensions.ext_database import db from extensions.ext_database import db
from extensions.ext_redis import redis_client from extensions.ext_redis import redis_client
from extensions.ext_storage import storage from extensions.ext_storage import storage
@ -279,6 +280,19 @@ class IndexingRunner:
if len(preview_texts) < 5: if len(preview_texts) < 5:
preview_texts.append(document.page_content) preview_texts.append(document.page_content)
# delete image files and related db records
image_upload_file_ids = get_image_upload_file_ids(document.page_content)
for upload_file_id in image_upload_file_ids:
image_file = db.session.query(UploadFile).filter(UploadFile.id == upload_file_id).first()
try:
storage.delete(image_file.key)
except Exception:
logging.exception(
"Delete image_files failed while indexing_estimate, \
image_upload_file_is: {}".format(upload_file_id)
)
db.session.delete(image_file)
if doc_form and doc_form == "qa_model": if doc_form and doc_form == "qa_model":
if len(preview_texts) > 0: if len(preview_texts) > 0:
# qa model document # qa model document

1
api/tasks/clean_dataset_task.py
View File

@ -78,6 +78,7 @@ def clean_dataset_task(
"Delete image_files failed when storage deleted, \ "Delete image_files failed when storage deleted, \
image_upload_file_is: {}".format(upload_file_id) image_upload_file_is: {}".format(upload_file_id)
) )
db.session.delete(image_file)
db.session.delete(segment) db.session.delete(segment)
db.session.query(DatasetProcessRule).filter(DatasetProcessRule.dataset_id == dataset_id).delete() db.session.query(DatasetProcessRule).filter(DatasetProcessRule.dataset_id == dataset_id).delete()

1
api/tasks/clean_document_task.py
View File

@ -51,6 +51,7 @@ def clean_document_task(document_id: str, dataset_id: str, doc_form: str, file_i
"Delete image_files failed when storage deleted, \ "Delete image_files failed when storage deleted, \
image_upload_file_is: {}".format(upload_file_id) image_upload_file_is: {}".format(upload_file_id)
) )
db.session.delete(image_file)
db.session.delete(segment) db.session.delete(segment)
db.session.commit() db.session.commit()