Merge branch 'main' into feat/support-extractor-tools

api/.env.example

@@ -120,7 +120,8 @@ SUPABASE_URL=your-server-url
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash
+
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -263,6 +264,11 @@ VIKINGDB_SCHEMA=http
 VIKINGDB_CONNECTION_TIMEOUT=30
 VIKINGDB_SOCKET_TIMEOUT=30
 
+# Lindorm configuration
+LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
+LINDORM_USERNAME=admin
+LINDORM_PASSWORD=admin
+
 # OceanBase Vector configuration
 OCEANBASE_VECTOR_HOST=127.0.0.1
 OCEANBASE_VECTOR_PORT=2881
@@ -271,6 +277,7 @@ OCEANBASE_VECTOR_PASSWORD=
 OCEANBASE_VECTOR_DATABASE=test
 OCEANBASE_MEMORY_LIMIT=6G
 
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5

api/configs/middleware/__init__.py

@@ -20,6 +20,7 @@ from configs.middleware.vdb.baidu_vector_config import BaiduVectorDBConfig
 from configs.middleware.vdb.chroma_config import ChromaConfig
 from configs.middleware.vdb.couchbase_config import CouchbaseConfig
 from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
+from configs.middleware.vdb.lindorm_config import LindormConfig
 from configs.middleware.vdb.milvus_config import MilvusConfig
 from configs.middleware.vdb.myscale_config import MyScaleConfig
 from configs.middleware.vdb.oceanbase_config import OceanBaseVectorConfig
@@ -259,6 +260,7 @@ class MiddlewareConfig(
     VikingDBConfig,
     UpstashConfig,
     TidbOnQdrantConfig,
+    LindormConfig,
     OceanBaseVectorConfig,
     BaiduVectorDBConfig,
 ):

api/configs/middleware/vdb/lindorm_config.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class LindormConfig(BaseSettings):
+    """
+    Lindorm configs
+    """
+
+    LINDORM_URL: Optional[str] = Field(
+        description="Lindorm url",
+        default=None,
+    )
+    LINDORM_USERNAME: Optional[str] = Field(
+        description="Lindorm user",
+        default=None,
+    )
+    LINDORM_PASSWORD: Optional[str] = Field(
+        description="Lindorm password",
+        default=None,
+    )

api/controllers/common/errors.py

@@ -0,0 +1,6 @@
+from werkzeug.exceptions import HTTPException
+
+
+class FilenameNotExistsError(HTTPException):
+    code = 400
+    description = "The specified filename does not exist."

api/controllers/common/helpers.py

@@ -0,0 +1,58 @@
+import mimetypes
+import os
+import re
+import urllib.parse
+from uuid import uuid4
+
+import httpx
+from pydantic import BaseModel
+
+
+class FileInfo(BaseModel):
+    filename: str
+    extension: str
+    mimetype: str
+    size: int
+
+
+def guess_file_info_from_response(response: httpx.Response):
+    url = str(response.url)
+    # Try to extract filename from URL
+    parsed_url = urllib.parse.urlparse(url)
+    url_path = parsed_url.path
+    filename = os.path.basename(url_path)
+
+    # If filename couldn't be extracted, use Content-Disposition header
+    if not filename:
+        content_disposition = response.headers.get("Content-Disposition")
+        if content_disposition:
+            filename_match = re.search(r'filename="?(.+)"?', content_disposition)
+            if filename_match:
+                filename = filename_match.group(1)
+
+    # If still no filename, generate a unique one
+    if not filename:
+        unique_name = str(uuid4())
+        filename = f"{unique_name}"
+
+    # Guess MIME type from filename first, then URL
+    mimetype, _ = mimetypes.guess_type(filename)
+    if mimetype is None:
+        mimetype, _ = mimetypes.guess_type(url)
+    if mimetype is None:
+        # If guessing fails, use Content-Type from response headers
+        mimetype = response.headers.get("Content-Type", "application/octet-stream")
+
+    extension = os.path.splitext(filename)[1]
+
+    # Ensure filename has an extension
+    if not extension:
+        extension = mimetypes.guess_extension(mimetype) or ".bin"
+        filename = f"{filename}{extension}"
+
+    return FileInfo(
+        filename=filename,
+        extension=extension,
+        mimetype=mimetype,
+        size=int(response.headers.get("Content-Length", -1)),
+    )

api/controllers/console/__init__.py

@@ -2,9 +2,21 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
+from .files import FileApi, FilePreviewApi, FileSupportTypeApi
+from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
+
 bp = Blueprint("console", __name__, url_prefix="/console/api")
 api = ExternalApi(bp)
 
+# File
+api.add_resource(FileApi, "/files/upload")
+api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
+api.add_resource(FileSupportTypeApi, "/files/support-type")
+
+# Remote files
+api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")
+api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
+
 # Import other controllers
 from . import admin, apikey, extension, feature, ping, setup, version
 
@@ -43,7 +55,6 @@ from .datasets import (
     datasets_document,
     datasets_segments,
     external,
-    file,
     hit_testing,
     website,
 )

api/controllers/console/apikey.py

@@ -10,8 +10,7 @@ from models.dataset import Dataset
 from models.model import ApiToken, App
 
 from . import api
-from .setup import setup_required
-from .wraps import account_initialization_required
+from .wraps import account_initialization_required, setup_required
 
 api_key_fields = {
     "id": fields.String,

api/controllers/console/app/advanced_prompt_template.py

@@ -1,8 +1,7 @@
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/agent.py

@@ -2,8 +2,7 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from libs.helper import uuid_value
 from libs.login import login_required
 from models.model import AppMode

api/controllers/console/app/annotation.py

@@ -6,8 +6,11 @@ from werkzeug.exceptions import Forbidden
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from extensions.ext_redis import redis_client
 from fields.annotation_fields import (
     annotation_fields,

api/controllers/console/app/app.py

@@ -6,8 +6,11 @@ from werkzeug.exceptions import BadRequest, Forbidden, abort
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from core.ops.ops_trace_manager import OpsTraceManager
 from fields.app_fields import (
     app_detail_fields,

api/controllers/console/app/audio.py

@@ -18,8 +18,7 @@ from controllers.console.app.error import (
     UnsupportedAudioTypeError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required

api/controllers/console/app/completion.py

@@ -15,8 +15,7 @@ from controllers.console.app.error import (
     ProviderQuotaExceededError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom

api/controllers/console/app/conversation.py

@@ -10,8 +10,7 @@ from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from fields.conversation_fields import (

api/controllers/console/app/conversation_variables.py

@@ -4,8 +4,7 @@ from sqlalchemy.orm import Session
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from fields.conversation_variable_fields import paginated_conversation_variable_fields
 from libs.login import login_required

api/controllers/console/app/generator.py

@@ -10,8 +10,7 @@ from controllers.console.app.error import (
     ProviderNotInitializeError,
     ProviderQuotaExceededError,
 )
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError

api/controllers/console/app/message.py

@@ -14,8 +14,11 @@ from controllers.console.app.error import (
 )
 from controllers.console.app.wraps import get_app_model
 from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError

api/controllers/console/app/model_config.py

@@ -6,8 +6,7 @@ from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.agent.entities import AgentToolEntity
 from core.tools.tool_manager import ToolManager
 from core.tools.utils.configuration import ToolParameterConfigurationManager

api/controllers/console/app/ops_trace.py

@@ -2,8 +2,7 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.ops_service import OpsService
 

api/controllers/console/app/site.py

@@ -7,8 +7,7 @@ from werkzeug.exceptions import Forbidden, NotFound
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
 from libs.login import login_required

api/controllers/console/app/statistic.py

@@ -8,8 +8,7 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required

api/controllers/console/app/workflow.py

@@ -9,8 +9,7 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from factories import variable_factory

api/controllers/console/app/workflow_app_log.py

@@ -3,8 +3,7 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
 from models import App

api/controllers/console/app/workflow_run.py

@@ -3,8 +3,7 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from fields.workflow_run_fields import (
     advanced_chat_workflow_run_pagination_fields,
     workflow_run_detail_fields,

api/controllers/console/app/workflow_statistic.py

@@ -8,8 +8,7 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required

api/controllers/console/auth/data_source_bearer_auth.py

@@ -7,8 +7,7 @@ from controllers.console.auth.error import ApiKeyAuthFailedError
 from libs.login import login_required
 from services.auth.api_key_auth_service import ApiKeyAuthService
 
-from ..setup import setup_required
-from ..wraps import account_initialization_required
+from ..wraps import account_initialization_required, setup_required
 
 
 class ApiKeyAuthDataSource(Resource):

api/controllers/console/auth/data_source_oauth.py

@@ -11,8 +11,7 @@ from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
 
-from ..setup import setup_required
-from ..wraps import account_initialization_required
+from ..wraps import account_initialization_required, setup_required
 
 
 def get_oauth_providers():

api/controllers/console/auth/forgot_password.py

@@ -13,7 +13,7 @@ from controllers.console.auth.error import (
     PasswordMismatchError,
 )
 from controllers.console.error import EmailSendIpLimitError, NotAllowedRegister
-from controllers.console.setup import setup_required
+from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
 from libs.helper import email, extract_remote_ip

api/controllers/console/auth/login.py

@@ -20,7 +20,7 @@ from controllers.console.error import (
     NotAllowedCreateWorkspace,
     NotAllowedRegister,
 )
-from controllers.console.setup import setup_required
+from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
 from libs.helper import email, extract_remote_ip
 from libs.password import valid_password

api/controllers/console/billing/billing.py

@@ -2,8 +2,7 @@ from flask_login import current_user
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, only_edition_cloud
+from controllers.console.wraps import account_initialization_required, only_edition_cloud, setup_required
 from libs.login import login_required
 from services.billing_service import BillingService
 

api/controllers/console/datasets/data_source.py

@@ -7,8 +7,7 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.indexing_runner import IndexingRunner
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.notion_extractor import NotionExtractor

api/controllers/console/datasets/datasets.py

@@ -10,8 +10,7 @@ from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
@@ -457,7 +456,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -621,6 +620,7 @@ class DatasetRetrievalSettingApi(Resource):
             case (
                 VectorType.MILVUS
                 | VectorType.RELYT
+                | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.TENCENT
@@ -641,6 +641,7 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.ELASTICSEARCH
                 | VectorType.PGVECTOR
                 | VectorType.TIDB_ON_QDRANT
+                | VectorType.LINDORM
                 | VectorType.COUCHBASE
             ):
                 return {
@@ -683,6 +684,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.ELASTICSEARCH
                 | VectorType.COUCHBASE
                 | VectorType.PGVECTOR
+                | VectorType.LINDORM
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -24,8 +24,11 @@ from controllers.console.datasets.error import (
     InvalidActionError,
     InvalidMetadataError,
 )
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from core.errors.error import (
     LLMBadRequestError,
     ModelCurrentlyNotSupportError,

api/controllers/console/datasets/datasets_segments.py

@@ -11,11 +11,11 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
-from controllers.console.setup import setup_required
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_knowledge_limit_check,
     cloud_edition_billing_resource_check,
+    setup_required,
 )
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager

api/controllers/console/datasets/external.py

@@ -6,8 +6,7 @@ from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 import services
 from controllers.console import api
 from controllers.console.datasets.error import DatasetNameDuplicateError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from fields.dataset_fields import dataset_detail_fields
 from libs.login import login_required
 from services.dataset_service import DatasetService

api/controllers/console/datasets/hit_testing.py

@@ -2,8 +2,7 @@ from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 
 

api/controllers/console/datasets/website.py

@@ -2,8 +2,7 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.website_service import WebsiteService
 

api/controllers/console/extension.py

@@ -3,8 +3,7 @@ from flask_restful import Resource, marshal_with, reqparse
 
 from constants import HIDDEN_VALUE
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from fields.api_based_extension_fields import api_based_extension_fields
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension

api/controllers/console/feature.py

@@ -5,8 +5,7 @@ from libs.login import login_required
 from services.feature_service import FeatureService
 
 from . import api
-from .setup import setup_required
-from .wraps import account_initialization_required, cloud_utm_record
+from .wraps import account_initialization_required, cloud_utm_record, setup_required
 
 
 class FeatureApi(Resource):

api/controllers/console/files/__init__.py

@@ -1,25 +1,26 @@
-import urllib.parse
-
 from flask import request
 from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
+from flask_restful import Resource, marshal_with
 
 import services
 from configs import dify_config
 from constants import DOCUMENT_EXTENSIONS
-from controllers.console import api
-from controllers.console.datasets.error import (
+from controllers.common.errors import FilenameNotExistsError
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
+from fields.file_fields import file_fields, upload_config_fields
+from libs.login import login_required
+from services.file_service import FileService
+
+from .errors import (
     FileTooLargeError,
     NoFileUploadedError,
     TooManyFilesError,
     UnsupportedFileTypeError,
 )
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.helper import ssrf_proxy
-from fields.file_fields import file_fields, remote_file_info_fields, upload_config_fields
-from libs.login import login_required
-from services.file_service import FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -44,21 +45,29 @@ class FileApi(Resource):
     @marshal_with(file_fields)
     @cloud_edition_billing_resource_check("documents")
     def post(self):
-        # get file from request
         file = request.files["file"]
+        source = request.form.get("source")
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("source", type=str, required=False, location="args")
-        source = parser.parse_args().get("source")
-
-        # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
+
+        if not file.filename:
+            raise FilenameNotExistsError
+
+        if source not in ("datasets", None):
+            source = None
+
         try:
-            upload_file = FileService.upload_file(file=file, user=current_user, source=source)
+            upload_file = FileService.upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=current_user,
+                source=source,
+            )
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
@@ -83,23 +92,3 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         return {"allowed_extensions": DOCUMENT_EXTENSIONS}
-
-
-class RemoteFileInfoApi(Resource):
-    @marshal_with(remote_file_info_fields)
-    def get(self, url):
-        decoded_url = urllib.parse.unquote(url)
-        try:
-            response = ssrf_proxy.head(decoded_url)
-            return {
-                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
-                "file_length": int(response.headers.get("Content-Length", 0)),
-            }
-        except Exception as e:
-            return {"error": str(e)}, 400
-
-
-api.add_resource(FileApi, "/files/upload")
-api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
-api.add_resource(FileSupportTypeApi, "/files/support-type")
-api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")

api/controllers/console/files/errors.py

@@ -0,0 +1,25 @@
+from libs.exception import BaseHTTPException
+
+
+class FileTooLargeError(BaseHTTPException):
+    error_code = "file_too_large"
+    description = "File size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = "unsupported_file_type"
+    description = "File type not allowed."
+    code = 415
+
+
+class TooManyFilesError(BaseHTTPException):
+    error_code = "too_many_files"
+    description = "Only one file is allowed."
+    code = 400
+
+
+class NoFileUploadedError(BaseHTTPException):
+    error_code = "no_file_uploaded"
+    description = "Please upload your file."
+    code = 400

api/controllers/console/remote_files.py

@@ -0,0 +1,71 @@
+import urllib.parse
+from typing import cast
+
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+
+from controllers.common import helpers
+from core.file import helpers as file_helpers
+from core.helper import ssrf_proxy
+from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
+from models.account import Account
+from services.file_service import FileService
+
+
+class RemoteFileInfoApi(Resource):
+    @marshal_with(remote_file_info_fields)
+    def get(self, url):
+        decoded_url = urllib.parse.unquote(url)
+        try:
+            response = ssrf_proxy.head(decoded_url)
+            return {
+                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
+                "file_length": int(response.headers.get("Content-Length", 0)),
+            }
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+
+class RemoteFileUploadApi(Resource):
+    @marshal_with(file_fields_with_signed_url)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("url", type=str, required=True, help="URL is required")
+        args = parser.parse_args()
+
+        url = args["url"]
+
+        response = ssrf_proxy.head(url)
+        response.raise_for_status()
+
+        file_info = helpers.guess_file_info_from_response(response)
+
+        if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
+            return {"error": "File size exceeded"}, 400
+
+        response = ssrf_proxy.get(url)
+        response.raise_for_status()
+        content = response.content
+
+        try:
+            user = cast(Account, current_user)
+            upload_file = FileService.upload_file(
+                filename=file_info.filename,
+                content=content,
+                mimetype=file_info.mimetype,
+                user=user,
+                source_url=url,
+            )
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+        return {
+            "id": upload_file.id,
+            "name": upload_file.name,
+            "size": upload_file.size,
+            "extension": upload_file.extension,
+            "url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
+            "mime_type": upload_file.mime_type,
+            "created_by": upload_file.created_by,
+            "created_at": upload_file.created_at,
+        }, 201

api/controllers/console/setup.py

@@ -1,5 +1,3 @@
-from functools import wraps
-
 from flask import request
 from flask_restful import Resource, reqparse
 
@@ -10,7 +8,7 @@ from models.model import DifySetup
 from services.account_service import RegisterService, TenantService
 
 from . import api
-from .error import AlreadySetupError, NotInitValidateError, NotSetupError
+from .error import AlreadySetupError, NotInitValidateError
 from .init_validate import get_init_validate_status
 from .wraps import only_edition_self_hosted
 
@@ -52,26 +50,10 @@ class SetupApi(Resource):
         return {"result": "success"}, 201
 
 
-def setup_required(view):
-    @wraps(view)
-    def decorated(*args, **kwargs):
-        # check setup
-        if not get_init_validate_status():
-            raise NotInitValidateError()
-
-        elif not get_setup_status():
-            raise NotSetupError()
-
-        return view(*args, **kwargs)
-
-    return decorated
-
-
 def get_setup_status():
     if dify_config.EDITION == "SELF_HOSTED":
         return DifySetup.query.first()
-    else:
-        return True
+    return True
 
 
 api.add_resource(SetupApi, "/setup")

api/controllers/console/tag/tags.py

@@ -4,8 +4,7 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from fields.tag_fields import tag_fields
 from libs.login import login_required
 from models.model import Tag

api/controllers/console/workspace/account.py

@@ -8,14 +8,13 @@ from flask_restful import Resource, fields, marshal_with, reqparse
 from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
-from controllers.console.setup import setup_required
 from controllers.console.workspace.error import (
     AccountAlreadyInitedError,
     CurrentPasswordIncorrectError,
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.helper import TimestampField, timezone

api/controllers/console/workspace/load_balancing_config.py

@@ -2,8 +2,7 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from libs.login import current_user, login_required

api/controllers/console/workspace/members.py

@@ -4,8 +4,11 @@ from flask_restful import Resource, abort, marshal_with, reqparse
 import services
 from configs import dify_config
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from extensions.ext_database import db
 from fields.member_fields import account_with_role_list_fields
 from libs.login import login_required

api/controllers/console/workspace/model_providers.py

@@ -6,8 +6,7 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/models.py

@@ -5,8 +5,7 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/tool_providers.py

@@ -7,8 +7,7 @@ from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
+from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import alphanumeric, uuid_value
 from libs.login import login_required

api/controllers/console/workspace/workspace.py

@@ -6,6 +6,7 @@ from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqpa
 from werkzeug.exceptions import Unauthorized
 
 import services
+from controllers.common.errors import FilenameNotExistsError
 from controllers.console import api
 from controllers.console.admin import admin_required
 from controllers.console.datasets.error import (
@@ -15,8 +16,11 @@ from controllers.console.datasets.error import (
     UnsupportedFileTypeError,
 )
 from controllers.console.error import AccountNotLinkTenantError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
 from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
@@ -193,12 +197,20 @@ class WebappLogoWorkspaceApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        if not file.filename:
+            raise FilenameNotExistsError
+
         extension = file.filename.split(".")[-1]
         if extension.lower() not in {"svg", "png"}:
             raise UnsupportedFileTypeError()
 
         try:
-            upload_file = FileService.upload_file(file=file, user=current_user)
+            upload_file = FileService.upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=current_user,
+            )
 
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)

api/controllers/console/wraps.py

@@ -1,4 +1,5 @@
 import json
+import os
 from functools import wraps
 
 from flask import abort, request
@@ -6,9 +7,12 @@ from flask_login import current_user
 
 from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError
+from models.model import DifySetup
 from services.feature_service import FeatureService
 from services.operation_service import OperationService
 
+from .error import NotInitValidateError, NotSetupError
+
 
 def account_initialization_required(view):
     @wraps(view)
@@ -124,3 +128,17 @@ def cloud_utm_record(view):
         return view(*args, **kwargs)
 
     return decorated
+
+
+def setup_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        # check setup
+        if dify_config.EDITION == "SELF_HOSTED" and os.environ.get("INIT_PASSWORD") and not DifySetup.query.first():
+            raise NotInitValidateError()
+        elif dify_config.EDITION == "SELF_HOSTED" and not DifySetup.query.first():
+            raise NotSetupError()
+
+        return view(*args, **kwargs)
+
+    return decorated

api/controllers/inner_api/workspace/workspace.py

@@ -1,6 +1,6 @@
 from flask_restful import Resource, reqparse
 
-from controllers.console.setup import setup_required
+from controllers.console.wraps import setup_required
 from controllers.inner_api import api
 from controllers.inner_api.wraps import inner_api_only
 from events.tenant_event import tenant_was_created

api/controllers/service_api/app/file.py

@@ -2,6 +2,7 @@ from flask import request
 from flask_restful import Resource, marshal_with
 
 import services
+from controllers.common.errors import FilenameNotExistsError
 from controllers.service_api import api
 from controllers.service_api.app.error import (
     FileTooLargeError,
@@ -31,8 +32,17 @@ class FileApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        if not file.filename:
+            raise FilenameNotExistsError
+
         try:
-            upload_file = FileService.upload_file(file, end_user)
+            upload_file = FileService.upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=end_user,
+                source="datasets",
+            )
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:

api/controllers/service_api/dataset/document.py

@@ -6,6 +6,7 @@ from sqlalchemy import desc
 from werkzeug.exceptions import NotFound
 
 import services.dataset_service
+from controllers.common.errors import FilenameNotExistsError
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
 from controllers.service_api.dataset.error import (
@@ -55,7 +56,12 @@ class DocumentAddByTextApi(DatasetApiResource):
         if not dataset.indexing_technique and not args["indexing_technique"]:
             raise ValueError("indexing_technique is required.")
 
-        upload_file = FileService.upload_text(args.get("text"), args.get("name"))
+        text = args.get("text")
+        name = args.get("name")
+        if text is None or name is None:
+            raise ValueError("Both 'text' and 'name' must be non-null values.")
+
+        upload_file = FileService.upload_text(text=str(text), text_name=str(name))
         data_source = {
             "type": "upload_file",
             "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
@@ -104,7 +110,11 @@ class DocumentUpdateByTextApi(DatasetApiResource):
             raise ValueError("Dataset is not exist.")
 
         if args["text"]:
-            upload_file = FileService.upload_text(args.get("text"), args.get("name"))
+            text = args.get("text")
+            name = args.get("name")
+            if text is None or name is None:
+                raise ValueError("Both text and name must be strings.")
+            upload_file = FileService.upload_text(text=str(text), text_name=str(name))
             data_source = {
                 "type": "upload_file",
                 "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
@@ -163,7 +173,16 @@ class DocumentAddByFileApi(DatasetApiResource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        upload_file = FileService.upload_file(file, current_user)
+        if not file.filename:
+            raise FilenameNotExistsError
+
+        upload_file = FileService.upload_file(
+            filename=file.filename,
+            content=file.read(),
+            mimetype=file.mimetype,
+            user=current_user,
+            source="datasets",
+        )
         data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
         args["data_source"] = data_source
         # validate args
@@ -212,7 +231,16 @@ class DocumentUpdateByFileApi(DatasetApiResource):
             if len(request.files) > 1:
                 raise TooManyFilesError()
 
-            upload_file = FileService.upload_file(file, current_user)
+            if not file.filename:
+                raise FilenameNotExistsError
+
+            upload_file = FileService.upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=current_user,
+                source="datasets",
+            )
             data_source = {"type": "upload_file", "info_list": {"file_info_list": {"file_ids": [upload_file.id]}}}
             args["data_source"] = data_source
         # validate args

api/controllers/web/__init__.py

@@ -2,8 +2,17 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
+from .files import FileApi
+from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
+
 bp = Blueprint("web", __name__, url_prefix="/api")
 api = ExternalApi(bp)
 
+# Files
+api.add_resource(FileApi, "/files/upload")
 
-from . import app, audio, completion, conversation, feature, file, message, passport, saved_message, site, workflow
+# Remote files
+api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")
+api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
+
+from . import app, audio, completion, conversation, feature, message, passport, saved_message, site, workflow

api/controllers/web/file.py

@@ -1,56 +0,0 @@
-import urllib.parse
-
-from flask import request
-from flask_restful import marshal_with, reqparse
-
-import services
-from controllers.web import api
-from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
-from controllers.web.wraps import WebApiResource
-from core.helper import ssrf_proxy
-from fields.file_fields import file_fields, remote_file_info_fields
-from services.file_service import FileService
-
-
-class FileApi(WebApiResource):
-    @marshal_with(file_fields)
-    def post(self, app_model, end_user):
-        # get file from request
-        file = request.files["file"]
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("source", type=str, required=False, location="args")
-        source = parser.parse_args().get("source")
-
-        # check file
-        if "file" not in request.files:
-            raise NoFileUploadedError()
-
-        if len(request.files) > 1:
-            raise TooManyFilesError()
-        try:
-            upload_file = FileService.upload_file(file=file, user=end_user, source=source)
-        except services.errors.file.FileTooLargeError as file_too_large_error:
-            raise FileTooLargeError(file_too_large_error.description)
-        except services.errors.file.UnsupportedFileTypeError:
-            raise UnsupportedFileTypeError()
-
-        return upload_file, 201
-
-
-class RemoteFileInfoApi(WebApiResource):
-    @marshal_with(remote_file_info_fields)
-    def get(self, url):
-        decoded_url = urllib.parse.unquote(url)
-        try:
-            response = ssrf_proxy.head(decoded_url)
-            return {
-                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
-                "file_length": int(response.headers.get("Content-Length", -1)),
-            }
-        except Exception as e:
-            return {"error": str(e)}, 400
-
-
-api.add_resource(FileApi, "/files/upload")
-api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")

api/controllers/web/files.py

@@ -0,0 +1,43 @@
+from flask import request
+from flask_restful import marshal_with
+
+import services
+from controllers.common.errors import FilenameNotExistsError
+from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
+from controllers.web.wraps import WebApiResource
+from fields.file_fields import file_fields
+from services.file_service import FileService
+
+
+class FileApi(WebApiResource):
+    @marshal_with(file_fields)
+    def post(self, app_model, end_user):
+        file = request.files["file"]
+        source = request.form.get("source")
+
+        if "file" not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+
+        if not file.filename:
+            raise FilenameNotExistsError
+
+        if source not in ("datasets", None):
+            source = None
+
+        try:
+            upload_file = FileService.upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=end_user,
+                source=source,
+            )
+        except services.errors.file.FileTooLargeError as file_too_large_error:
+            raise FileTooLargeError(file_too_large_error.description)
+        except services.errors.file.UnsupportedFileTypeError:
+            raise UnsupportedFileTypeError()
+
+        return upload_file, 201

api/controllers/web/remote_files.py

@@ -0,0 +1,68 @@
+import urllib.parse
+
+from flask_restful import marshal_with, reqparse
+
+from controllers.common import helpers
+from controllers.web.wraps import WebApiResource
+from core.file import helpers as file_helpers
+from core.helper import ssrf_proxy
+from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
+from services.file_service import FileService
+
+
+class RemoteFileInfoApi(WebApiResource):
+    @marshal_with(remote_file_info_fields)
+    def get(self, url):
+        decoded_url = urllib.parse.unquote(url)
+        try:
+            response = ssrf_proxy.head(decoded_url)
+            return {
+                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
+                "file_length": int(response.headers.get("Content-Length", -1)),
+            }
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+
+class RemoteFileUploadApi(WebApiResource):
+    @marshal_with(file_fields_with_signed_url)
+    def post(self, app_model, end_user):  # Add app_model and end_user parameters
+        parser = reqparse.RequestParser()
+        parser.add_argument("url", type=str, required=True, help="URL is required")
+        args = parser.parse_args()
+
+        url = args["url"]
+
+        response = ssrf_proxy.head(url)
+        response.raise_for_status()
+
+        file_info = helpers.guess_file_info_from_response(response)
+
+        if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
+            return {"error": "File size exceeded"}, 400
+
+        response = ssrf_proxy.get(url)
+        response.raise_for_status()
+        content = response.content
+
+        try:
+            upload_file = FileService.upload_file(
+                filename=file_info.filename,
+                content=content,
+                mimetype=file_info.mimetype,
+                user=end_user,  # Use end_user instead of current_user
+                source_url=url,
+            )
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+        return {
+            "id": upload_file.id,
+            "name": upload_file.name,
+            "size": upload_file.size,
+            "extension": upload_file.extension,
+            "url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
+            "mime_type": upload_file.mime_type,
+            "created_by": upload_file.created_by,
+            "created_at": upload_file.created_at,
+        }, 201

api/core/app/apps/base_app_generator.py

@@ -76,6 +76,7 @@ class BaseAppGenerator:
 
     def _validate_input(self, *, inputs: Mapping[str, Any], var: "VariableEntity"):
         user_input_value = inputs.get(var.variable)
+
         if not user_input_value:
             if var.required:
                 raise ValueError(f"{var.variable} is required in input form")
@@ -88,6 +89,7 @@ class BaseAppGenerator:
             VariableEntityType.PARAGRAPH,
         } and not isinstance(user_input_value, str):
             raise ValueError(f"(type '{var.type}') {var.variable} in input form must be a string")
+
         if var.type == VariableEntityType.NUMBER and isinstance(user_input_value, str):
             # may raise ValueError if user_input_value is not a valid number
             try:
@@ -97,25 +99,30 @@ class BaseAppGenerator:
                     return int(user_input_value)
             except ValueError:
                 raise ValueError(f"{var.variable} in input form must be a valid number")
-        if var.type == VariableEntityType.SELECT:
-            options = var.options
-            if user_input_value not in options:
-                raise ValueError(f"{var.variable} in input form must be one of the following: {options}")
-        elif var.type in {VariableEntityType.TEXT_INPUT, VariableEntityType.PARAGRAPH}:
-            if var.max_length and len(user_input_value) > var.max_length:
-                raise ValueError(f"{var.variable} in input form must be less than {var.max_length} characters")
-        elif var.type == VariableEntityType.FILE:
-            if not isinstance(user_input_value, dict) and not isinstance(user_input_value, File):
-                raise ValueError(f"{var.variable} in input form must be a file")
-        elif var.type == VariableEntityType.FILE_LIST:
-            if not (
-                isinstance(user_input_value, list)
-                and (
-                    all(isinstance(item, dict) for item in user_input_value)
-                    or all(isinstance(item, File) for item in user_input_value)
-                )
-            ):
-                raise ValueError(f"{var.variable} in input form must be a list of files")
+
+        match var.type:
+            case VariableEntityType.SELECT:
+                if user_input_value not in var.options:
+                    raise ValueError(f"{var.variable} in input form must be one of the following: {var.options}")
+            case VariableEntityType.TEXT_INPUT | VariableEntityType.PARAGRAPH:
+                if var.max_length and len(user_input_value) > var.max_length:
+                    raise ValueError(f"{var.variable} in input form must be less than {var.max_length} characters")
+            case VariableEntityType.FILE:
+                if not isinstance(user_input_value, dict) and not isinstance(user_input_value, File):
+                    raise ValueError(f"{var.variable} in input form must be a file")
+            case VariableEntityType.FILE_LIST:
+                # if number of files exceeds the limit, raise ValueError
+                if not (
+                    isinstance(user_input_value, list)
+                    and (
+                        all(isinstance(item, dict) for item in user_input_value)
+                        or all(isinstance(item, File) for item in user_input_value)
+                    )
+                ):
+                    raise ValueError(f"{var.variable} in input form must be a list of files")
+
+                if var.max_length and len(user_input_value) > var.max_length:
+                    raise ValueError(f"{var.variable} in input form must be less than {var.max_length} files")
 
         return user_input_value
 

api/core/indexing_runner.py

@@ -17,6 +17,7 @@ from core.errors.error import ProviderTokenNotInitError
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_manager import ModelInstance, ModelManager
 from core.model_runtime.entities.model_entities import ModelType
+from core.rag.cleaner.clean_processor import CleanProcessor
 from core.rag.datasource.keyword.keyword_factory import Keyword
 from core.rag.docstore.dataset_docstore import DatasetDocumentStore
 from core.rag.extractor.entity.extract_setting import ExtractSetting
@@ -597,26 +598,9 @@ class IndexingRunner:
             rules = DatasetProcessRule.AUTOMATIC_RULES
         else:
             rules = json.loads(processing_rule.rules) if processing_rule.rules else {}
+        document_text = CleanProcessor.clean(text, rules)
 
-        if "pre_processing_rules" in rules:
-            pre_processing_rules = rules["pre_processing_rules"]
-            for pre_processing_rule in pre_processing_rules:
-                if pre_processing_rule["id"] == "remove_extra_spaces" and pre_processing_rule["enabled"] is True:
-                    # Remove extra spaces
-                    pattern = r"\n{3,}"
-                    text = re.sub(pattern, "\n\n", text)
-                    pattern = r"[\t\f\r\x20\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000]{2,}"
-                    text = re.sub(pattern, " ", text)
-                elif pre_processing_rule["id"] == "remove_urls_emails" and pre_processing_rule["enabled"] is True:
-                    # Remove email
-                    pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
-                    text = re.sub(pattern, "", text)
-
-                    # Remove URL
-                    pattern = r"https?://[^\s]+"
-                    text = re.sub(pattern, "", text)
-
-        return text
+        return document_text
 
     @staticmethod
     def format_split_text(text):

api/core/model_runtime/model_providers/gpustack/_assets/icon_l_en.svg

@@ -0,0 +1,15 @@
+<svg width="68" height="24" viewBox="0 0 68 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Gemini">
+<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M50.6875 4.37014C48.3498 4.59292 46.5349 6.41319 46.3337 8.72764C46.1446 6.44662 44.2677 4.56074 41.9805 4.3737C44.2762 4.1997 46.152 2.28299 46.3373 0C46.4882 2.28911 48.405 4.20047 50.6875 4.37014ZM15.4567 9.41141L13.9579 10.9076C9.92941 6.64892 2.69298 9.97287 3.17317 15.8112C3.22394 23.108 14.5012 24.4317 15.3628 16.8809H9.52096L9.50061 14.9149H17.3595C18.8163 23.1364 8.44367 27.0292 3.19453 21.238C0.847044 18.7556 0.363651 14.7682 1.83717 11.7212C4.1129 6.62089 11.6505 5.29845 15.4567 9.41141ZM45.5915 23.5989H47.6945C47.6944 22.9155 47.6945 22.2307 47.6946 21.5452V21.5325C47.6948 19.8907 47.695 18.2453 47.6924 16.6072C47.6914 15.9407 47.6161 15.2823 47.4024 14.647C46.4188 11.2828 41.4255 11.4067 39.8332 14.214C38.5637 11.4171 34.4009 11.5236 32.8538 14.0084L32.8082 13.9976V12.4806L32.4233 12.4804H32.4224C31.8687 12.4801 31.3324 12.4798 30.7949 12.4811V23.5848L32.8977 23.5672C32.8981 22.9411 32.8979 22.3122 32.8977 21.6822V21.6812V21.6802V21.6791V21.6781V21.6771V21.676V21.676V21.6759V21.6758V21.6757V21.6757V21.6756C32.8973 20.204 32.8969 18.7261 32.904 17.2614C32.8889 15.3646 34.5674 13.5687 36.5358 14.124C37.7794 14.3298 38.1851 15.6148 38.1761 16.7257C38.1821 17.7019 38.18 18.6824 38.178 19.6633V19.6638C38.1752 20.9756 38.1724 22.2881 38.1891 23.5919L40.2846 23.5731C40.2929 22.7511 40.2881 21.9245 40.2832 21.0966C40.2753 19.7402 40.2674 18.3805 40.317 17.0328C40.4418 15.2122 42.0141 13.6186 43.9064 14.1168C45.2685 14.3231 45.6136 15.7748 45.5882 16.9545C45.5938 18.4959 45.5929 20.0492 45.5921 21.5968V21.5991V21.6014V21.6037V21.606V21.6083V21.6106C45.5917 22.2749 45.5913 22.9382 45.5915 23.5989ZM20.6167 18.4408C20.5625 21.9486 25.2121 23.6996 27.2993 20.0558L29.1566 20.9592C27.8157 23.7067 24.2337 24.7424 21.5381 23.4213C18.0052 21.7253 17.41 16.5007 20.0334 13.7517C21.4609 12.1752 23.7291 11.7901 25.7206 12.3653C28.3408 13.1257 29.4974 15.8937 29.326 18.4399C27.5547 18.4415 25.7971 18.4412 24.0364 18.4409C22.8993 18.4407 21.7609 18.4405 20.6167 18.4408ZM27.1041 16.6957C26.7048 13.1033 21.2867 13.2256 20.7494 16.6957H27.1041ZM53.543 23.5999H55.6206L55.6206 22.4361C55.6205 20.7877 55.6205 19.1443 55.6207 17.4939C55.6208 16.8853 55.7234 16.297 56.0063 15.7531C56.6115 14.3862 58.1745 13.7002 59.5927 14.1774C60.7512 14.4455 61.2852 15.6069 61.2762 16.7154C61.2774 18.3497 61.2771 19.9826 61.2769 21.6162V21.6166V21.617V21.6174V21.6179L61.2766 23.6007H63.3698C63.3913 22.0924 63.3869 20.584 63.3826 19.0755V19.0754V19.0753V19.0753V19.0752C63.3799 18.1682 63.3773 17.2612 63.3803 16.3541C63.3796 15.8622 63.3103 15.3765 63.1698 14.9052C62.3248 11.5142 57.3558 11.2385 55.5828 14.0038L55.5336 13.9905V12.4917H53.539C53.4898 12.7313 53.4934 23.4113 53.543 23.5999ZM49.6211 12.4944H51.7065V23.5994H49.6211V12.4944ZM65.1035 23.5991H67.1831C67.2367 23.2198 67.2133 12.6566 67.1634 12.4983H65.1035V23.5991ZM52.1504 8.67829C52.1709 10.4847 49.2418 10.7058 49.1816 8.65714C49.2189 6.5948 52.2437 6.81331 52.1504 8.67829ZM66.1387 10.1324C64.2712 10.1609 64.1316 7.19881 66.1559 7.17114C68.1709 7.19817 68.0215 10.2087 66.1387 10.1324Z" fill="url(#paint0_linear_14286_118464)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_14286_118464" x1="-2" y1="0.999998" x2="67.9999" y2="27.5002" gradientUnits="userSpaceOnUse">
+<stop stop-color="#7798E0"/>
+<stop offset="0.210002" stop-color="#086FFF"/>
+<stop offset="0.345945" stop-color="#086FFF"/>
+<stop offset="0.591777" stop-color="#479AFF"/>
+<stop offset="0.895892" stop-color="#B7C4FA"/>
+<stop offset="1" stop-color="#B5C5F9"/>
+</linearGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/gpustack/_assets/icon_s_en.svg

@@ -0,0 +1,11 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="24" height="24" rx="6" fill="url(#paint0_linear_7301_16076)"/>
+<path d="M20 12.0116C15.7043 12.42 12.3692 15.757 11.9995 20C11.652 15.8183 8.20301 12.361 4 12.0181C8.21855 11.6991 11.6656 8.1853 12.006 4C12.2833 8.19653 15.8057 11.7005 20 12.0116Z" fill="white" fill-opacity="0.88"/>
+<defs>
+<linearGradient id="paint0_linear_7301_16076" x1="-9" y1="29.5" x2="19.4387" y2="1.43791" gradientUnits="userSpaceOnUse">
+<stop offset="0.192878" stop-color="#1C7DFF"/>
+<stop offset="0.520213" stop-color="#1C69FF"/>
+<stop offset="1" stop-color="#F0DCD6"/>
+</linearGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/gpustack/gpustack.py

@@ -0,0 +1,10 @@
+import logging
+
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class GPUStackProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        pass

api/core/model_runtime/model_providers/gpustack/gpustack.yaml

@@ -0,0 +1,120 @@
+provider: gpustack
+label:
+  en_US: GPUStack
+icon_small:
+  en_US: icon_s_en.png
+icon_large:
+  en_US: icon_l_en.png
+supported_model_types:
+  - llm
+  - text-embedding
+  - rerank
+configurate_methods:
+  - customizable-model
+model_credential_schema:
+  model:
+    label:
+      en_US: Model Name
+      zh_Hans: 模型名称
+    placeholder:
+      en_US: Enter your model name
+      zh_Hans: 输入模型名称
+  credential_form_schemas:
+    - variable: endpoint_url
+      label:
+        zh_Hans: 服务器地址
+        en_US: Server URL
+      type: text-input
+      required: true
+      placeholder:
+        zh_Hans: 输入 GPUStack 的服务器地址，如 http://192.168.1.100
+        en_US: Enter the GPUStack server URL, e.g. http://192.168.1.100
+    - variable: api_key
+      label:
+        en_US: API Key
+      type: secret-input
+      required: true
+      placeholder:
+        zh_Hans: 输入您的 API Key
+        en_US: Enter your API Key
+    - variable: mode
+      show_on:
+        - variable: __model_type
+          value: llm
+      label:
+        en_US: Completion mode
+      type: select
+      required: false
+      default: chat
+      placeholder:
+        zh_Hans: 选择补全类型
+        en_US: Select completion type
+      options:
+        - value: completion
+          label:
+            en_US: Completion
+            zh_Hans: 补全
+        - value: chat
+          label:
+            en_US: Chat
+            zh_Hans: 对话
+    - variable: context_size
+      label:
+        zh_Hans: 模型上下文长度
+        en_US: Model context size
+      required: true
+      type: text-input
+      default: "8192"
+      placeholder:
+        zh_Hans: 输入您的模型上下文长度
+        en_US: Enter your Model context size
+    - variable: max_tokens_to_sample
+      label:
+        zh_Hans: 最大 token 上限
+        en_US: Upper bound for max tokens
+      show_on:
+        - variable: __model_type
+          value: llm
+      default: "8192"
+      type: text-input
+    - variable: function_calling_type
+      show_on:
+        - variable: __model_type
+          value: llm
+      label:
+        en_US: Function calling
+      type: select
+      required: false
+      default: no_call
+      options:
+        - value: function_call
+          label:
+            en_US: Function Call
+            zh_Hans: Function Call
+        - value: tool_call
+          label:
+            en_US: Tool Call
+            zh_Hans: Tool Call
+        - value: no_call
+          label:
+            en_US: Not Support
+            zh_Hans: 不支持
+    - variable: vision_support
+      show_on:
+        - variable: __model_type
+          value: llm
+      label:
+        zh_Hans: Vision 支持
+        en_US: Vision Support
+      type: select
+      required: false
+      default: no_support
+      options:
+        - value: support
+          label:
+            en_US: Support
+            zh_Hans: 支持
+        - value: no_support
+          label:
+            en_US: Not Support
+            zh_Hans: 不支持

api/core/model_runtime/model_providers/gpustack/llm/llm.py

@@ -0,0 +1,45 @@
+from collections.abc import Generator
+
+from yarl import URL
+
+from core.model_runtime.entities.llm_entities import LLMResult
+from core.model_runtime.entities.message_entities import (
+    PromptMessage,
+    PromptMessageTool,
+)
+from core.model_runtime.model_providers.openai_api_compatible.llm.llm import (
+    OAIAPICompatLargeLanguageModel,
+)
+
+
+class GPUStackLanguageModel(OAIAPICompatLargeLanguageModel):
+    def _invoke(
+        self,
+        model: str,
+        credentials: dict,
+        prompt_messages: list[PromptMessage],
+        model_parameters: dict,
+        tools: list[PromptMessageTool] | None = None,
+        stop: list[str] | None = None,
+        stream: bool = True,
+        user: str | None = None,
+    ) -> LLMResult | Generator:
+        return super()._invoke(
+            model,
+            credentials,
+            prompt_messages,
+            model_parameters,
+            tools,
+            stop,
+            stream,
+            user,
+        )
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        self._add_custom_parameters(credentials)
+        super().validate_credentials(model, credentials)
+
+    @staticmethod
+    def _add_custom_parameters(credentials: dict) -> None:
+        credentials["endpoint_url"] = str(URL(credentials["endpoint_url"]) / "v1-openai")
+        credentials["mode"] = "chat"

api/core/model_runtime/model_providers/gpustack/rerank/rerank.py

@@ -0,0 +1,146 @@
+from json import dumps
+from typing import Optional
+
+import httpx
+from requests import post
+from yarl import URL
+
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.model_entities import (
+    AIModelEntity,
+    FetchFrom,
+    ModelPropertyKey,
+    ModelType,
+)
+from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult
+from core.model_runtime.errors.invoke import (
+    InvokeAuthorizationError,
+    InvokeBadRequestError,
+    InvokeConnectionError,
+    InvokeError,
+    InvokeRateLimitError,
+    InvokeServerUnavailableError,
+)
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.rerank_model import RerankModel
+
+
+class GPUStackRerankModel(RerankModel):
+    """
+    Model class for GPUStack rerank model.
+    """
+
+    def _invoke(
+        self,
+        model: str,
+        credentials: dict,
+        query: str,
+        docs: list[str],
+        score_threshold: Optional[float] = None,
+        top_n: Optional[int] = None,
+        user: Optional[str] = None,
+    ) -> RerankResult:
+        """
+        Invoke rerank model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param query: search query
+        :param docs: docs for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n documents to return
+        :param user: unique user id
+        :return: rerank result
+        """
+        if len(docs) == 0:
+            return RerankResult(model=model, docs=[])
+
+        endpoint_url = credentials["endpoint_url"]
+        headers = {
+            "Authorization": f"Bearer {credentials.get('api_key')}",
+            "Content-Type": "application/json",
+        }
+
+        data = {"model": model, "query": query, "documents": docs, "top_n": top_n}
+
+        try:
+            response = post(
+                str(URL(endpoint_url) / "v1" / "rerank"),
+                headers=headers,
+                data=dumps(data),
+                timeout=10,
+            )
+            response.raise_for_status()
+            results = response.json()
+
+            rerank_documents = []
+            for result in results["results"]:
+                index = result["index"]
+                if "document" in result:
+                    text = result["document"]["text"]
+                else:
+                    text = docs[index]
+
+                rerank_document = RerankDocument(
+                    index=index,
+                    text=text,
+                    score=result["relevance_score"],
+                )
+
+                if score_threshold is None or result["relevance_score"] >= score_threshold:
+                    rerank_documents.append(rerank_document)
+
+            return RerankResult(model=model, docs=rerank_documents)
+        except httpx.HTTPStatusError as e:
+            raise InvokeServerUnavailableError(str(e))
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._invoke(
+                model=model,
+                credentials=credentials,
+                query="What is the capital of the United States?",
+                docs=[
+                    "Carson City is the capital city of the American state of Nevada. At the 2010 United States "
+                    "Census, Carson City had a population of 55,274.",
+                    "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that "
+                    "are a political division controlled by the United States. Its capital is Saipan.",
+                ],
+                score_threshold=0.8,
+            )
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        """
+        return {
+            InvokeConnectionError: [httpx.ConnectError],
+            InvokeServerUnavailableError: [httpx.RemoteProtocolError],
+            InvokeRateLimitError: [],
+            InvokeAuthorizationError: [httpx.HTTPStatusError],
+            InvokeBadRequestError: [httpx.RequestError],
+        }
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity:
+        """
+        generate custom model entities from credentials
+        """
+        entity = AIModelEntity(
+            model=model,
+            label=I18nObject(en_US=model),
+            model_type=ModelType.RERANK,
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
+        )
+
+        return entity

api/core/model_runtime/model_providers/gpustack/text_embedding/text_embedding.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from yarl import URL
+
+from core.entities.embedding_type import EmbeddingInputType
+from core.model_runtime.entities.text_embedding_entities import (
+    TextEmbeddingResult,
+)
+from core.model_runtime.model_providers.openai_api_compatible.text_embedding.text_embedding import (
+    OAICompatEmbeddingModel,
+)
+
+
+class GPUStackTextEmbeddingModel(OAICompatEmbeddingModel):
+    """
+    Model class for GPUStack text embedding model.
+    """
+
+    def _invoke(
+        self,
+        model: str,
+        credentials: dict,
+        texts: list[str],
+        user: Optional[str] = None,
+        input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT,
+    ) -> TextEmbeddingResult:
+        return super()._invoke(model, credentials, texts, user, input_type)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        self._add_custom_parameters(credentials)
+        super().validate_credentials(model, credentials)
+
+    @staticmethod
+    def _add_custom_parameters(credentials: dict) -> None:
+        credentials["endpoint_url"] = str(URL(credentials["endpoint_url"]) / "v1-openai")

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-11b-vision-instruct.yaml

@@ -5,6 +5,7 @@ label:
 model_type: llm
 features:
   - agent-thought
+  - vision
 model_properties:
   mode: chat
   context_size: 131072

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-90b-vision-instruct.yaml

@@ -5,6 +5,7 @@ label:
 model_type: llm
 features:
   - agent-thought
+  - vision
 model_properties:
   mode: chat
   context_size: 131072

api/core/rag/datasource/vdb/lindorm/lindorm_vector.py

@@ -0,0 +1,498 @@
+import copy
+import json
+import logging
+from collections.abc import Iterable
+from typing import Any, Optional
+
+from opensearchpy import OpenSearch
+from opensearchpy.helpers import bulk
+from pydantic import BaseModel, model_validator
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+from configs import dify_config
+from core.rag.datasource.vdb.field import Field
+from core.rag.datasource.vdb.vector_base import BaseVector
+from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory
+from core.rag.datasource.vdb.vector_type import VectorType
+from core.rag.embedding.embedding_base import Embeddings
+from core.rag.models.document import Document
+from extensions.ext_redis import redis_client
+from models.dataset import Dataset
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logging.getLogger("lindorm").setLevel(logging.WARN)
+
+
+class LindormVectorStoreConfig(BaseModel):
+    hosts: str
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_config(cls, values: dict) -> dict:
+        if not values["hosts"]:
+            raise ValueError("config URL is required")
+        if not values["username"]:
+            raise ValueError("config USERNAME is required")
+        if not values["password"]:
+            raise ValueError("config PASSWORD is required")
+        return values
+
+    def to_opensearch_params(self) -> dict[str, Any]:
+        params = {
+            "hosts": self.hosts,
+        }
+        if self.username and self.password:
+            params["http_auth"] = (self.username, self.password)
+        return params
+
+
+class LindormVectorStore(BaseVector):
+    def __init__(self, collection_name: str, config: LindormVectorStoreConfig, **kwargs):
+        super().__init__(collection_name.lower())
+        self._client_config = config
+        self._client = OpenSearch(**config.to_opensearch_params())
+        self.kwargs = kwargs
+
+    def get_type(self) -> str:
+        return VectorType.LINDORM
+
+    def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
+        self.create_collection(len(embeddings[0]), **kwargs)
+        self.add_texts(texts, embeddings)
+
+    def refresh(self):
+        self._client.indices.refresh(index=self._collection_name)
+
+    def __filter_existed_ids(
+        self,
+        texts: list[str],
+        metadatas: list[dict],
+        ids: list[str],
+        bulk_size: int = 1024,
+    ) -> tuple[Iterable[str], Optional[list[dict]], Optional[list[str]]]:
+        @retry(stop=stop_after_attempt(3), wait=wait_fixed(60))
+        def __fetch_existing_ids(batch_ids: list[str]) -> set[str]:
+            try:
+                existing_docs = self._client.mget(index=self._collection_name, body={"ids": batch_ids}, _source=False)
+                return {doc["_id"] for doc in existing_docs["docs"] if doc["found"]}
+            except Exception as e:
+                logger.error(f"Error fetching batch {batch_ids}: {e}")
+                return set()
+
+        @retry(stop=stop_after_attempt(3), wait=wait_fixed(60))
+        def __fetch_existing_routing_ids(batch_ids: list[str], route_ids: list[str]) -> set[str]:
+            try:
+                existing_docs = self._client.mget(
+                    body={
+                        "docs": [
+                            {"_index": self._collection_name, "_id": id, "routing": routing}
+                            for id, routing in zip(batch_ids, route_ids)
+                        ]
+                    },
+                    _source=False,
+                )
+                return {doc["_id"] for doc in existing_docs["docs"] if doc["found"]}
+            except Exception as e:
+                logger.error(f"Error fetching batch {batch_ids}: {e}")
+                return set()
+
+        if ids is None:
+            return texts, metadatas, ids
+
+        if len(texts) != len(ids):
+            raise RuntimeError(f"texts {len(texts)} != {ids}")
+
+        filtered_texts = []
+        filtered_metadatas = []
+        filtered_ids = []
+
+        def batch(iterable, n):
+            length = len(iterable)
+            for idx in range(0, length, n):
+                yield iterable[idx : min(idx + n, length)]
+
+        for ids_batch, texts_batch, metadatas_batch in zip(
+            batch(ids, bulk_size),
+            batch(texts, bulk_size),
+            batch(metadatas, bulk_size) if metadatas is not None else batch([None] * len(ids), bulk_size),
+        ):
+            existing_ids_set = __fetch_existing_ids(ids_batch)
+            for text, metadata, doc_id in zip(texts_batch, metadatas_batch, ids_batch):
+                if doc_id not in existing_ids_set:
+                    filtered_texts.append(text)
+                    filtered_ids.append(doc_id)
+                    if metadatas is not None:
+                        filtered_metadatas.append(metadata)
+
+        return filtered_texts, metadatas if metadatas is None else filtered_metadatas, filtered_ids
+
+    def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
+        actions = []
+        uuids = self._get_uuids(documents)
+        for i in range(len(documents)):
+            action = {
+                "_op_type": "index",
+                "_index": self._collection_name.lower(),
+                "_id": uuids[i],
+                "_source": {
+                    Field.CONTENT_KEY.value: documents[i].page_content,
+                    Field.VECTOR.value: embeddings[i],  # Make sure you pass an array here
+                    Field.METADATA_KEY.value: documents[i].metadata,
+                },
+            }
+            actions.append(action)
+        bulk(self._client, actions)
+        self.refresh()
+
+    def get_ids_by_metadata_field(self, key: str, value: str):
+        query = {"query": {"term": {f"{Field.METADATA_KEY.value}.{key}.keyword": value}}}
+        response = self._client.search(index=self._collection_name, body=query)
+        if response["hits"]["hits"]:
+            return [hit["_id"] for hit in response["hits"]["hits"]]
+        else:
+            return None
+
+    def delete_by_metadata_field(self, key: str, value: str):
+        query_str = {"query": {"match": {f"metadata.{key}": f"{value}"}}}
+        results = self._client.search(index=self._collection_name, body=query_str)
+        ids = [hit["_id"] for hit in results["hits"]["hits"]]
+        if ids:
+            self.delete_by_ids(ids)
+
+    def delete_by_ids(self, ids: list[str]) -> None:
+        for id in ids:
+            if self._client.exists(index=self._collection_name, id=id):
+                self._client.delete(index=self._collection_name, id=id)
+            else:
+                logger.warning(f"DELETE BY ID: ID {id} does not exist in the index.")
+
+    def delete(self) -> None:
+        try:
+            if self._client.indices.exists(index=self._collection_name):
+                self._client.indices.delete(index=self._collection_name, params={"timeout": 60})
+                logger.info("Delete index success")
+            else:
+                logger.warning(f"Index '{self._collection_name}' does not exist. No deletion performed.")
+        except Exception as e:
+            logger.error(f"Error occurred while deleting the index: {e}")
+            raise e
+
+    def text_exists(self, id: str) -> bool:
+        try:
+            self._client.get(index=self._collection_name, id=id)
+            return True
+        except:
+            return False
+
+    def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
+        # Make sure query_vector is a list
+        if not isinstance(query_vector, list):
+            raise ValueError("query_vector should be a list of floats")
+
+        # Check whether query_vector is a floating-point number list
+        if not all(isinstance(x, float) for x in query_vector):
+            raise ValueError("All elements in query_vector should be floats")
+
+        top_k = kwargs.get("top_k", 10)
+        query = default_vector_search_query(query_vector=query_vector, k=top_k, **kwargs)
+        try:
+            response = self._client.search(index=self._collection_name, body=query)
+        except Exception as e:
+            logger.error(f"Error executing search: {e}")
+            raise
+
+        docs_and_scores = []
+        for hit in response["hits"]["hits"]:
+            docs_and_scores.append(
+                (
+                    Document(
+                        page_content=hit["_source"][Field.CONTENT_KEY.value],
+                        vector=hit["_source"][Field.VECTOR.value],
+                        metadata=hit["_source"][Field.METADATA_KEY.value],
+                    ),
+                    hit["_score"],
+                )
+            )
+        docs = []
+        for doc, score in docs_and_scores:
+            score_threshold = kwargs.get("score_threshold", 0.0) or 0.0
+            if score > score_threshold:
+                doc.metadata["score"] = score
+                docs.append(doc)
+
+        return docs
+
+    def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
+        must = kwargs.get("must")
+        must_not = kwargs.get("must_not")
+        should = kwargs.get("should")
+        minimum_should_match = kwargs.get("minimum_should_match", 0)
+        top_k = kwargs.get("top_k", 10)
+        filters = kwargs.get("filter")
+        routing = kwargs.get("routing")
+        full_text_query = default_text_search_query(
+            query_text=query,
+            k=top_k,
+            text_field=Field.CONTENT_KEY.value,
+            must=must,
+            must_not=must_not,
+            should=should,
+            minimum_should_match=minimum_should_match,
+            filters=filters,
+            routing=routing,
+        )
+        response = self._client.search(index=self._collection_name, body=full_text_query)
+        docs = []
+        for hit in response["hits"]["hits"]:
+            docs.append(
+                Document(
+                    page_content=hit["_source"][Field.CONTENT_KEY.value],
+                    vector=hit["_source"][Field.VECTOR.value],
+                    metadata=hit["_source"][Field.METADATA_KEY.value],
+                )
+            )
+
+        return docs
+
+    def create_collection(self, dimension: int, **kwargs):
+        lock_name = f"vector_indexing_lock_{self._collection_name}"
+        with redis_client.lock(lock_name, timeout=20):
+            collection_exist_cache_key = f"vector_indexing_{self._collection_name}"
+            if redis_client.get(collection_exist_cache_key):
+                logger.info(f"Collection {self._collection_name} already exists.")
+                return
+            if self._client.indices.exists(index=self._collection_name):
+                logger.info("{self._collection_name.lower()} already exists.")
+                return
+            if len(self.kwargs) == 0 and len(kwargs) != 0:
+                self.kwargs = copy.deepcopy(kwargs)
+            vector_field = kwargs.pop("vector_field", Field.VECTOR.value)
+            shards = kwargs.pop("shards", 2)
+
+            engine = kwargs.pop("engine", "lvector")
+            method_name = kwargs.pop("method_name", "hnsw")
+            data_type = kwargs.pop("data_type", "float")
+            space_type = kwargs.pop("space_type", "cosinesimil")
+
+            hnsw_m = kwargs.pop("hnsw_m", 24)
+            hnsw_ef_construction = kwargs.pop("hnsw_ef_construction", 500)
+            ivfpq_m = kwargs.pop("ivfpq_m", dimension)
+            nlist = kwargs.pop("nlist", 1000)
+            centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", True if nlist >= 5000 else False)
+            centroids_hnsw_m = kwargs.pop("centroids_hnsw_m", 24)
+            centroids_hnsw_ef_construct = kwargs.pop("centroids_hnsw_ef_construct", 500)
+            centroids_hnsw_ef_search = kwargs.pop("centroids_hnsw_ef_search", 100)
+            mapping = default_text_mapping(
+                dimension,
+                method_name,
+                shards=shards,
+                engine=engine,
+                data_type=data_type,
+                space_type=space_type,
+                vector_field=vector_field,
+                hnsw_m=hnsw_m,
+                hnsw_ef_construction=hnsw_ef_construction,
+                nlist=nlist,
+                ivfpq_m=ivfpq_m,
+                centroids_use_hnsw=centroids_use_hnsw,
+                centroids_hnsw_m=centroids_hnsw_m,
+                centroids_hnsw_ef_construct=centroids_hnsw_ef_construct,
+                centroids_hnsw_ef_search=centroids_hnsw_ef_search,
+                **kwargs,
+            )
+            self._client.indices.create(index=self._collection_name.lower(), body=mapping)
+            redis_client.set(collection_exist_cache_key, 1, ex=3600)
+            # logger.info(f"create index success: {self._collection_name}")
+
+
+def default_text_mapping(dimension: int, method_name: str, **kwargs: Any) -> dict:
+    routing_field = kwargs.get("routing_field")
+    excludes_from_source = kwargs.get("excludes_from_source")
+    analyzer = kwargs.get("analyzer", "ik_max_word")
+    text_field = kwargs.get("text_field", Field.CONTENT_KEY.value)
+    engine = kwargs["engine"]
+    shard = kwargs["shards"]
+    space_type = kwargs["space_type"]
+    data_type = kwargs["data_type"]
+    vector_field = kwargs.get("vector_field", Field.VECTOR.value)
+
+    if method_name == "ivfpq":
+        ivfpq_m = kwargs["ivfpq_m"]
+        nlist = kwargs["nlist"]
+        centroids_use_hnsw = True if nlist > 10000 else False
+        centroids_hnsw_m = 24
+        centroids_hnsw_ef_construct = 500
+        centroids_hnsw_ef_search = 100
+        parameters = {
+            "m": ivfpq_m,
+            "nlist": nlist,
+            "centroids_use_hnsw": centroids_use_hnsw,
+            "centroids_hnsw_m": centroids_hnsw_m,
+            "centroids_hnsw_ef_construct": centroids_hnsw_ef_construct,
+            "centroids_hnsw_ef_search": centroids_hnsw_ef_search,
+        }
+    elif method_name == "hnsw":
+        neighbor = kwargs["hnsw_m"]
+        ef_construction = kwargs["hnsw_ef_construction"]
+        parameters = {"m": neighbor, "ef_construction": ef_construction}
+    elif method_name == "flat":
+        parameters = {}
+    else:
+        raise RuntimeError(f"unexpected method_name: {method_name}")
+
+    mapping = {
+        "settings": {"index": {"number_of_shards": shard, "knn": True}},
+        "mappings": {
+            "properties": {
+                vector_field: {
+                    "type": "knn_vector",
+                    "dimension": dimension,
+                    "data_type": data_type,
+                    "method": {
+                        "engine": engine,
+                        "name": method_name,
+                        "space_type": space_type,
+                        "parameters": parameters,
+                    },
+                },
+                text_field: {"type": "text", "analyzer": analyzer},
+            }
+        },
+    }
+
+    if excludes_from_source:
+        mapping["mappings"]["_source"] = {"excludes": excludes_from_source}  # e.g. {"excludes": ["vector_field"]}
+
+    if method_name == "ivfpq" and routing_field is not None:
+        mapping["settings"]["index"]["knn_routing"] = True
+        mapping["settings"]["index"]["knn.offline.construction"] = True
+
+    if method_name == "flat" and routing_field is not None:
+        mapping["settings"]["index"]["knn_routing"] = True
+
+    return mapping
+
+
+def default_text_search_query(
+    query_text: str,
+    k: int = 4,
+    text_field: str = Field.CONTENT_KEY.value,
+    must: Optional[list[dict]] = None,
+    must_not: Optional[list[dict]] = None,
+    should: Optional[list[dict]] = None,
+    minimum_should_match: int = 0,
+    filters: Optional[list[dict]] = None,
+    routing: Optional[str] = None,
+    **kwargs,
+) -> dict:
+    if routing is not None:
+        routing_field = kwargs.get("routing_field", "routing_field")
+        query_clause = {
+            "bool": {
+                "must": [{"match": {text_field: query_text}}, {"term": {f"metadata.{routing_field}.keyword": routing}}]
+            }
+        }
+    else:
+        query_clause = {"match": {text_field: query_text}}
+    # build the simplest search_query when only query_text is specified
+    if not must and not must_not and not should and not filters:
+        search_query = {"size": k, "query": query_clause}
+        return search_query
+
+    # build complex search_query when either of must/must_not/should/filter is specified
+    if must:
+        if not isinstance(must, list):
+            raise RuntimeError(f"unexpected [must] clause with {type(filters)}")
+        if query_clause not in must:
+            must.append(query_clause)
+    else:
+        must = [query_clause]
+
+    boolean_query = {"must": must}
+
+    if must_not:
+        if not isinstance(must_not, list):
+            raise RuntimeError(f"unexpected [must_not] clause with {type(filters)}")
+        boolean_query["must_not"] = must_not
+
+    if should:
+        if not isinstance(should, list):
+            raise RuntimeError(f"unexpected [should] clause with {type(filters)}")
+        boolean_query["should"] = should
+        if minimum_should_match != 0:
+            boolean_query["minimum_should_match"] = minimum_should_match
+
+    if filters:
+        if not isinstance(filters, list):
+            raise RuntimeError(f"unexpected [filter] clause with {type(filters)}")
+        boolean_query["filter"] = filters
+
+    search_query = {"size": k, "query": {"bool": boolean_query}}
+    return search_query
+
+
+def default_vector_search_query(
+    query_vector: list[float],
+    k: int = 4,
+    min_score: str = "0.0",
+    ef_search: Optional[str] = None,  # only for hnsw
+    nprobe: Optional[str] = None,  # "2000"
+    reorder_factor: Optional[str] = None,  # "20"
+    client_refactor: Optional[str] = None,  # "true"
+    vector_field: str = Field.VECTOR.value,
+    filters: Optional[list[dict]] = None,
+    filter_type: Optional[str] = None,
+    **kwargs,
+) -> dict:
+    if filters is not None:
+        filter_type = "post_filter" if filter_type is None else filter_type
+        if not isinstance(filter, list):
+            raise RuntimeError(f"unexpected filter with {type(filters)}")
+    final_ext = {"lvector": {}}
+    if min_score != "0.0":
+        final_ext["lvector"]["min_score"] = min_score
+    if ef_search:
+        final_ext["lvector"]["ef_search"] = ef_search
+    if nprobe:
+        final_ext["lvector"]["nprobe"] = nprobe
+    if reorder_factor:
+        final_ext["lvector"]["reorder_factor"] = reorder_factor
+    if client_refactor:
+        final_ext["lvector"]["client_refactor"] = client_refactor
+
+    search_query = {
+        "size": k,
+        "_source": True,  # force return '_source'
+        "query": {"knn": {vector_field: {"vector": query_vector, "k": k}}},
+    }
+
+    if filters is not None:
+        # when using filter, transform filter from List[Dict] to Dict as valid format
+        filters = {"bool": {"must": filters}} if len(filters) > 1 else filters[0]
+        search_query["query"]["knn"][vector_field]["filter"] = filters  # filter should be Dict
+        if filter_type:
+            final_ext["lvector"]["filter_type"] = filter_type
+
+    if final_ext != {"lvector": {}}:
+        search_query["ext"] = final_ext
+    return search_query
+
+
+class LindormVectorStoreFactory(AbstractVectorFactory):
+    def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> LindormVectorStore:
+        if dataset.index_struct_dict:
+            class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"]
+            collection_name = class_prefix
+        else:
+            dataset_id = dataset.id
+            collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+            dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.LINDORM, collection_name))
+        lindorm_config = LindormVectorStoreConfig(
+            hosts=dify_config.LINDORM_URL,
+            username=dify_config.LINDORM_USERNAME,
+            password=dify_config.LINDORM_PASSWORD,
+        )
+        return LindormVectorStore(collection_name, lindorm_config)

api/core/rag/datasource/vdb/vector_factory.py

@@ -134,6 +134,10 @@ class Vector:
                 from core.rag.datasource.vdb.tidb_on_qdrant.tidb_on_qdrant_vector import TidbOnQdrantVectorFactory
 
                 return TidbOnQdrantVectorFactory
+            case VectorType.LINDORM:
+                from core.rag.datasource.vdb.lindorm.lindorm_vector import LindormVectorStoreFactory
+
+                return LindormVectorStoreFactory
             case VectorType.OCEANBASE:
                 from core.rag.datasource.vdb.oceanbase.oceanbase_vector import OceanBaseVectorFactory
 

api/core/rag/datasource/vdb/vector_type.py

@@ -16,6 +16,7 @@ class VectorType(str, Enum):
     TENCENT = "tencent"
     ORACLE = "oracle"
     ELASTICSEARCH = "elasticsearch"
+    LINDORM = "lindorm"
     COUCHBASE = "couchbase"
     BAIDU = "baidu"
     VIKINGDB = "vikingdb"

api/core/tools/provider/builtin/chart/chart.py

@@ -1,5 +1,5 @@
 import matplotlib.pyplot as plt
-from matplotlib.font_manager import FontProperties
+from matplotlib.font_manager import FontProperties, fontManager
 
 from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
 
@@ -17,9 +17,10 @@ def set_chinese_font():
     ]
 
     for font in font_list:
-        chinese_font = FontProperties(font)
-        if chinese_font.get_name() == font:
-            return chinese_font
+        if font in fontManager.ttflist:
+            chinese_font = FontProperties(font)
+            if chinese_font.get_name() == font:
+                return chinese_font
 
     return FontProperties()
 

api/core/tools/provider/builtin/podcast_generator/tools/podcast_audio_generator.py

@@ -1,15 +1,19 @@
 import concurrent.futures
 import io
 import random
+import warnings
 from typing import Any, Literal, Optional, Union
 
 import openai
-from pydub import AudioSegment
 
 from core.tools.entities.tool_entities import ToolInvokeMessage
 from core.tools.errors import ToolParameterValidationError, ToolProviderCredentialValidationError
 from core.tools.tool.builtin_tool import BuiltinTool
 
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    from pydub import AudioSegment
+
 
 class PodcastAudioGeneratorTool(BuiltinTool):
     @staticmethod

api/core/workflow/nodes/document_extractor/exc.py

@@ -1,4 +1,4 @@
-class DocumentExtractorError(Exception):
+class DocumentExtractorError(ValueError):
     """Base exception for errors related to the DocumentExtractorNode."""
 
 

api/core/workflow/nodes/document_extractor/node.py

@@ -6,12 +6,14 @@ import docx
 import pandas as pd
 import pypdfium2
 import yaml
+from unstructured.partition.api import partition_via_api
 from unstructured.partition.email import partition_email
 from unstructured.partition.epub import partition_epub
 from unstructured.partition.msg import partition_msg
 from unstructured.partition.ppt import partition_ppt
 from unstructured.partition.pptx import partition_pptx
 
+from configs import dify_config
 from core.file import File, FileTransferMethod, file_manager
 from core.helper import ssrf_proxy
 from core.variables import ArrayFileSegment
@@ -263,7 +265,14 @@ def _extract_text_from_ppt(file_content: bytes) -> str:
 def _extract_text_from_pptx(file_content: bytes) -> str:
     try:
         with io.BytesIO(file_content) as file:
-            elements = partition_pptx(file=file)
+            if dify_config.UNSTRUCTURED_API_URL and dify_config.UNSTRUCTURED_API_KEY:
+                elements = partition_via_api(
+                    file=file,
+                    api_url=dify_config.UNSTRUCTURED_API_URL,
+                    api_key=dify_config.UNSTRUCTURED_API_KEY,
+                )
+            else:
+                elements = partition_pptx(file=file)
         return "\n".join([getattr(element, "text", "") for element in elements])
     except Exception as e:
         raise TextExtractionError(f"Failed to extract text from PPTX: {str(e)}") from e

api/core/workflow/nodes/list_operator/exc.py

@@ -0,0 +1,16 @@
+class ListOperatorError(ValueError):
+    """Base class for all ListOperator errors."""
+
+    pass
+
+
+class InvalidFilterValueError(ListOperatorError):
+    pass
+
+
+class InvalidKeyError(ListOperatorError):
+    pass
+
+
+class InvalidConditionError(ListOperatorError):
+    pass

api/core/workflow/nodes/list_operator/node.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable, Sequence
-from typing import Literal
+from typing import Literal, Union
 
 from core.file import File
 from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment
@@ -9,6 +9,7 @@ from core.workflow.nodes.enums import NodeType
 from models.workflow import WorkflowNodeExecutionStatus
 
 from .entities import ListOperatorNodeData
+from .exc import InvalidConditionError, InvalidFilterValueError, InvalidKeyError, ListOperatorError
 
 
 class ListOperatorNode(BaseNode[ListOperatorNodeData]):
@@ -26,7 +27,17 @@ class ListOperatorNode(BaseNode[ListOperatorNodeData]):
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED, error=error_message, inputs=inputs, outputs=outputs
             )
-        if variable.value and not isinstance(variable, ArrayFileSegment | ArrayNumberSegment | ArrayStringSegment):
+        if not variable.value:
+            inputs = {"variable": []}
+            process_data = {"variable": []}
+            outputs = {"result": [], "first_record": None, "last_record": None}
+            return NodeRunResult(
+                status=WorkflowNodeExecutionStatus.SUCCEEDED,
+                inputs=inputs,
+                process_data=process_data,
+                outputs=outputs,
+            )
+        if not isinstance(variable, ArrayFileSegment | ArrayNumberSegment | ArrayStringSegment):
             error_message = (
                 f"Variable {self.node_data.variable} is not an ArrayFileSegment, ArrayNumberSegment "
                 "or ArrayStringSegment"
@@ -36,70 +47,98 @@ class ListOperatorNode(BaseNode[ListOperatorNodeData]):
             )
 
         if isinstance(variable, ArrayFileSegment):
+            inputs = {"variable": [item.to_dict() for item in variable.value]}
             process_data["variable"] = [item.to_dict() for item in variable.value]
         else:
+            inputs = {"variable": variable.value}
             process_data["variable"] = variable.value
 
-        # Filter
-        if self.node_data.filter_by.enabled:
-            for condition in self.node_data.filter_by.conditions:
-                if isinstance(variable, ArrayStringSegment):
-                    if not isinstance(condition.value, str):
-                        raise ValueError(f"Invalid filter value: {condition.value}")
-                    value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
-                    filter_func = _get_string_filter_func(condition=condition.comparison_operator, value=value)
-                    result = list(filter(filter_func, variable.value))
-                    variable = variable.model_copy(update={"value": result})
-                elif isinstance(variable, ArrayNumberSegment):
-                    if not isinstance(condition.value, str):
-                        raise ValueError(f"Invalid filter value: {condition.value}")
-                    value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
-                    filter_func = _get_number_filter_func(condition=condition.comparison_operator, value=float(value))
-                    result = list(filter(filter_func, variable.value))
-                    variable = variable.model_copy(update={"value": result})
-                elif isinstance(variable, ArrayFileSegment):
-                    if isinstance(condition.value, str):
-                        value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
-                    else:
-                        value = condition.value
-                    filter_func = _get_file_filter_func(
-                        key=condition.key,
-                        condition=condition.comparison_operator,
-                        value=value,
-                    )
-                    result = list(filter(filter_func, variable.value))
-                    variable = variable.model_copy(update={"value": result})
+        try:
+            # Filter
+            if self.node_data.filter_by.enabled:
+                variable = self._apply_filter(variable)
 
-        # Order
-        if self.node_data.order_by.enabled:
+            # Order
+            if self.node_data.order_by.enabled:
+                variable = self._apply_order(variable)
+
+            # Slice
+            if self.node_data.limit.enabled:
+                variable = self._apply_slice(variable)
+
+            outputs = {
+                "result": variable.value,
+                "first_record": variable.value[0] if variable.value else None,
+                "last_record": variable.value[-1] if variable.value else None,
+            }
+            return NodeRunResult(
+                status=WorkflowNodeExecutionStatus.SUCCEEDED,
+                inputs=inputs,
+                process_data=process_data,
+                outputs=outputs,
+            )
+        except ListOperatorError as e:
+            return NodeRunResult(
+                status=WorkflowNodeExecutionStatus.FAILED,
+                error=str(e),
+                inputs=inputs,
+                process_data=process_data,
+                outputs=outputs,
+            )
+
+    def _apply_filter(
+        self, variable: Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]
+    ) -> Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]:
+        for condition in self.node_data.filter_by.conditions:
             if isinstance(variable, ArrayStringSegment):
-                result = _order_string(order=self.node_data.order_by.value, array=variable.value)
+                if not isinstance(condition.value, str):
+                    raise InvalidFilterValueError(f"Invalid filter value: {condition.value}")
+                value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
+                filter_func = _get_string_filter_func(condition=condition.comparison_operator, value=value)
+                result = list(filter(filter_func, variable.value))
                 variable = variable.model_copy(update={"value": result})
             elif isinstance(variable, ArrayNumberSegment):
-                result = _order_number(order=self.node_data.order_by.value, array=variable.value)
+                if not isinstance(condition.value, str):
+                    raise InvalidFilterValueError(f"Invalid filter value: {condition.value}")
+                value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
+                filter_func = _get_number_filter_func(condition=condition.comparison_operator, value=float(value))
+                result = list(filter(filter_func, variable.value))
                 variable = variable.model_copy(update={"value": result})
             elif isinstance(variable, ArrayFileSegment):
-                result = _order_file(
-                    order=self.node_data.order_by.value, order_by=self.node_data.order_by.key, array=variable.value
+                if isinstance(condition.value, str):
+                    value = self.graph_runtime_state.variable_pool.convert_template(condition.value).text
+                else:
+                    value = condition.value
+                filter_func = _get_file_filter_func(
+                    key=condition.key,
+                    condition=condition.comparison_operator,
+                    value=value,
                 )
+                result = list(filter(filter_func, variable.value))
                 variable = variable.model_copy(update={"value": result})
+        return variable
 
-        # Slice
-        if self.node_data.limit.enabled:
-            result = variable.value[: self.node_data.limit.size]
+    def _apply_order(
+        self, variable: Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]
+    ) -> Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]:
+        if isinstance(variable, ArrayStringSegment):
+            result = _order_string(order=self.node_data.order_by.value, array=variable.value)
             variable = variable.model_copy(update={"value": result})
+        elif isinstance(variable, ArrayNumberSegment):
+            result = _order_number(order=self.node_data.order_by.value, array=variable.value)
+            variable = variable.model_copy(update={"value": result})
+        elif isinstance(variable, ArrayFileSegment):
+            result = _order_file(
+                order=self.node_data.order_by.value, order_by=self.node_data.order_by.key, array=variable.value
+            )
+            variable = variable.model_copy(update={"value": result})
+        return variable
 
-        outputs = {
-            "result": variable.value,
-            "first_record": variable.value[0] if variable.value else None,
-            "last_record": variable.value[-1] if variable.value else None,
-        }
-        return NodeRunResult(
-            status=WorkflowNodeExecutionStatus.SUCCEEDED,
-            inputs=inputs,
-            process_data=process_data,
-            outputs=outputs,
-        )
+    def _apply_slice(
+        self, variable: Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]
+    ) -> Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]:
+        result = variable.value[: self.node_data.limit.size]
+        return variable.model_copy(update={"value": result})
 
 
 def _get_file_extract_number_func(*, key: str) -> Callable[[File], int]:
@@ -107,7 +146,7 @@ def _get_file_extract_number_func(*, key: str) -> Callable[[File], int]:
         case "size":
             return lambda x: x.size
         case _:
-            raise ValueError(f"Invalid key: {key}")
+            raise InvalidKeyError(f"Invalid key: {key}")
 
 
 def _get_file_extract_string_func(*, key: str) -> Callable[[File], str]:
@@ -125,7 +164,7 @@ def _get_file_extract_string_func(*, key: str) -> Callable[[File], str]:
         case "url":
             return lambda x: x.remote_url or ""
         case _:
-            raise ValueError(f"Invalid key: {key}")
+            raise InvalidKeyError(f"Invalid key: {key}")
 
 
 def _get_string_filter_func(*, condition: str, value: str) -> Callable[[str], bool]:
@@ -151,7 +190,7 @@ def _get_string_filter_func(*, condition: str, value: str) -> Callable[[str], bo
         case "not empty":
             return lambda x: x != ""
         case _:
-            raise ValueError(f"Invalid condition: {condition}")
+            raise InvalidConditionError(f"Invalid condition: {condition}")
 
 
 def _get_sequence_filter_func(*, condition: str, value: Sequence[str]) -> Callable[[str], bool]:
@@ -161,7 +200,7 @@ def _get_sequence_filter_func(*, condition: str, value: Sequence[str]) -> Callab
         case "not in":
             return lambda x: not _in(value)(x)
         case _:
-            raise ValueError(f"Invalid condition: {condition}")
+            raise InvalidConditionError(f"Invalid condition: {condition}")
 
 
 def _get_number_filter_func(*, condition: str, value: int | float) -> Callable[[int | float], bool]:
@@ -179,7 +218,7 @@ def _get_number_filter_func(*, condition: str, value: int | float) -> Callable[[
         case "≥":
             return _ge(value)
         case _:
-            raise ValueError(f"Invalid condition: {condition}")
+            raise InvalidConditionError(f"Invalid condition: {condition}")
 
 
 def _get_file_filter_func(*, key: str, condition: str, value: str | Sequence[str]) -> Callable[[File], bool]:
@@ -193,7 +232,7 @@ def _get_file_filter_func(*, key: str, condition: str, value: str | Sequence[str
         extract_func = _get_file_extract_number_func(key=key)
         return lambda x: _get_number_filter_func(condition=condition, value=float(value))(extract_func(x))
     else:
-        raise ValueError(f"Invalid key: {key}")
+        raise InvalidKeyError(f"Invalid key: {key}")
 
 
 def _contains(value: str):

api/factories/file_factory.py

@@ -160,7 +160,7 @@ def _build_from_local_file(
         tenant_id=tenant_id,
         type=file_type,
         transfer_method=transfer_method,
-        remote_url=None,
+        remote_url=row.source_url,
         related_id=mapping.get("upload_file_id"),
         _extra_config=config,
         size=row.size,

api/fields/file_fields.py

@@ -24,3 +24,15 @@ remote_file_info_fields = {
     "file_type": fields.String(attribute="file_type"),
     "file_length": fields.Integer(attribute="file_length"),
 }
+
+
+file_fields_with_signed_url = {
+    "id": fields.String,
+    "name": fields.String,
+    "size": fields.Integer,
+    "extension": fields.String,
+    "url": fields.String,
+    "mime_type": fields.String,
+    "created_by": fields.String,
+    "created_at": TimestampField,
+}

api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py

@@ -28,16 +28,12 @@ def upgrade():
     sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
     sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey')
     )
-    with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op:
-        batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False)
 
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ##
-    with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op:
-        batch_op.drop_index('tracing_app_config_app_id_idx')
-
     op.drop_table('tracing_app_configs')
+
     # ### end Alembic commands ###

api/migrations/versions/2024_11_01_0434-d3f6769a94a3_add_upload_files_source_url.py

@@ -0,0 +1,31 @@
+"""Add upload_files.source_url
+
+Revision ID: d3f6769a94a3
+Revises: 43fa78bc3b7d
+Create Date: 2024-11-01 04:34:23.816198
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'd3f6769a94a3'
+down_revision = '43fa78bc3b7d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('upload_files', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('source_url', sa.String(length=255), server_default='', nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('upload_files', schema=None) as batch_op:
+        batch_op.drop_column('source_url')
+    # ### end Alembic commands ###

api/migrations/versions/2024_11_01_0449-93ad8c19c40b_rename_conversation_variables_index_name.py

@@ -0,0 +1,52 @@
+"""rename conversation variables index name
+
+Revision ID: 93ad8c19c40b
+Revises: d3f6769a94a3
+Create Date: 2024-11-01 04:49:53.100250
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '93ad8c19c40b'
+down_revision = 'd3f6769a94a3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    if conn.dialect.name == 'postgresql':
+        # Rename indexes for PostgreSQL
+        op.execute('ALTER INDEX workflow__conversation_variables_app_id_idx RENAME TO workflow_conversation_variables_app_id_idx')
+        op.execute('ALTER INDEX workflow__conversation_variables_created_at_idx RENAME TO workflow_conversation_variables_created_at_idx')
+    else:
+        # For other databases, use the original drop and create method
+        with op.batch_alter_table('workflow_conversation_variables', schema=None) as batch_op:
+            batch_op.drop_index('workflow__conversation_variables_app_id_idx')
+            batch_op.drop_index('workflow__conversation_variables_created_at_idx')
+            batch_op.create_index(batch_op.f('workflow_conversation_variables_app_id_idx'), ['app_id'], unique=False)
+            batch_op.create_index(batch_op.f('workflow_conversation_variables_created_at_idx'), ['created_at'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    if conn.dialect.name == 'postgresql':
+        # Rename indexes back for PostgreSQL
+        op.execute('ALTER INDEX workflow_conversation_variables_app_id_idx RENAME TO workflow__conversation_variables_app_id_idx')
+        op.execute('ALTER INDEX workflow_conversation_variables_created_at_idx RENAME TO workflow__conversation_variables_created_at_idx')
+    else:
+        # For other databases, use the original drop and create method
+        with op.batch_alter_table('workflow_conversation_variables', schema=None) as batch_op:
+            batch_op.drop_index(batch_op.f('workflow_conversation_variables_created_at_idx'))
+            batch_op.drop_index(batch_op.f('workflow_conversation_variables_app_id_idx'))
+            batch_op.create_index('workflow__conversation_variables_created_at_idx', ['created_at'], unique=False)
+            batch_op.create_index('workflow__conversation_variables_app_id_idx', ['app_id'], unique=False)
+
+    # ### end Alembic commands ###

api/migrations/versions/2024_11_01_0540-f4d7ce70a7ca_update_upload_files_source_url.py

@@ -0,0 +1,41 @@
+"""update upload_files.source_url
+
+Revision ID: f4d7ce70a7ca
+Revises: 93ad8c19c40b
+Create Date: 2024-11-01 05:40:03.531751
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f4d7ce70a7ca'
+down_revision = '93ad8c19c40b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('upload_files', schema=None) as batch_op:
+        batch_op.alter_column('source_url',
+               existing_type=sa.VARCHAR(length=255),
+               type_=sa.TEXT(),
+               existing_nullable=False,
+               existing_server_default=sa.text("''::character varying"))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('upload_files', schema=None) as batch_op:
+        batch_op.alter_column('source_url',
+               existing_type=sa.TEXT(),
+               type_=sa.VARCHAR(length=255),
+               existing_nullable=False,
+               existing_server_default=sa.text("''::character varying"))
+
+    # ### end Alembic commands ###

api/migrations/versions/2024_11_01_0622-d07474999927_update_type_of_custom_disclaimer_to_text.py

@@ -0,0 +1,67 @@
+"""update type of custom_disclaimer to TEXT
+
+Revision ID: d07474999927
+Revises: f4d7ce70a7ca
+Create Date: 2024-11-01 06:22:27.981398
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'd07474999927'
+down_revision = 'f4d7ce70a7ca'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("UPDATE recommended_apps SET custom_disclaimer = '' WHERE custom_disclaimer IS NULL")
+    op.execute("UPDATE sites SET custom_disclaimer = '' WHERE custom_disclaimer IS NULL")
+    op.execute("UPDATE tool_api_providers SET custom_disclaimer = '' WHERE custom_disclaimer IS NULL")
+
+    with op.batch_alter_table('recommended_apps', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.VARCHAR(length=255),
+               type_=sa.TEXT(),
+               nullable=False)
+
+    with op.batch_alter_table('sites', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.VARCHAR(length=255),
+               type_=sa.TEXT(),
+               nullable=False)
+
+    with op.batch_alter_table('tool_api_providers', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.VARCHAR(length=255),
+               type_=sa.TEXT(),
+               nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tool_api_providers', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.TEXT(),
+               type_=sa.VARCHAR(length=255),
+               nullable=True)
+
+    with op.batch_alter_table('sites', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.TEXT(),
+               type_=sa.VARCHAR(length=255),
+               nullable=True)
+
+    with op.batch_alter_table('recommended_apps', schema=None) as batch_op:
+        batch_op.alter_column('custom_disclaimer',
+               existing_type=sa.TEXT(),
+               type_=sa.VARCHAR(length=255),
+               nullable=True)
+
+    # ### end Alembic commands ###

api/migrations/versions/2024_11_01_0623-09a8d1878d9b_update_workflows_graph_features_and_.py

@@ -0,0 +1,73 @@
+"""update workflows graph, features and updated_at
+
+Revision ID: 09a8d1878d9b
+Revises: d07474999927
+Create Date: 2024-11-01 06:23:59.579186
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '09a8d1878d9b'
+down_revision = 'd07474999927'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('conversations', schema=None) as batch_op:
+        batch_op.alter_column('inputs',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               nullable=False)
+
+    with op.batch_alter_table('messages', schema=None) as batch_op:
+        batch_op.alter_column('inputs',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               nullable=False)
+
+    op.execute("UPDATE workflows SET updated_at = created_at WHERE updated_at IS NULL")
+    op.execute("UPDATE workflows SET graph = '' WHERE graph IS NULL")
+    op.execute("UPDATE workflows SET features = '' WHERE features IS NULL")
+
+    with op.batch_alter_table('workflows', schema=None) as batch_op:
+        batch_op.alter_column('graph',
+               existing_type=sa.TEXT(),
+               nullable=False)
+        batch_op.alter_column('features',
+               existing_type=sa.TEXT(),
+               nullable=False)
+        batch_op.alter_column('updated_at',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('workflows', schema=None) as batch_op:
+        batch_op.alter_column('updated_at',
+               existing_type=postgresql.TIMESTAMP(),
+               nullable=True)
+        batch_op.alter_column('features',
+               existing_type=sa.TEXT(),
+               nullable=True)
+        batch_op.alter_column('graph',
+               existing_type=sa.TEXT(),
+               nullable=True)
+
+    with op.batch_alter_table('messages', schema=None) as batch_op:
+        batch_op.alter_column('inputs',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               nullable=True)
+
+    with op.batch_alter_table('conversations', schema=None) as batch_op:
+        batch_op.alter_column('inputs',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               nullable=True)
+
+    # ### end Alembic commands ###

api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py

@@ -22,17 +22,11 @@ def upgrade():
     with op.batch_alter_table('apps', schema=None) as batch_op:
         batch_op.add_column(sa.Column('tracing', sa.Text(), nullable=True))
 
-    with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
-        batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False)
-
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
-        batch_op.drop_index('tracing_app_config_app_id_idx')
-
     with op.batch_alter_table('apps', schema=None) as batch_op:
         batch_op.drop_column('tracing')
 

api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py

@@ -30,30 +30,15 @@ def upgrade():
     sa.Column('is_active', sa.Boolean(), server_default=sa.text('true'), nullable=False),
                     sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey')
     )
+
     with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
         batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False)
 
-    with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op:
-        batch_op.drop_index('tracing_app_config_app_id_idx')
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('tracing_app_configs',
-    sa.Column('id', sa.UUID(), server_default=sa.text('uuid_generate_v4()'), autoincrement=False, nullable=False),
-    sa.Column('app_id', sa.UUID(), autoincrement=False, nullable=False),
-    sa.Column('tracing_provider', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
-    sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True),
-    sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False),
-    sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False),
-                    sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey')
-    )
-    with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op:
-        batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False)
-
-    with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
-        batch_op.drop_index('trace_app_config_app_id_idx')
-
     op.drop_table('trace_app_config')
+
     # ### end Alembic commands ###

api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py

@@ -20,12 +20,10 @@ def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('tracing_app_configs')
 
-    with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
-        batch_op.drop_index('tracing_app_config_app_id_idx')
-
     # idx_dataset_permissions_tenant_id
     with op.batch_alter_table('dataset_permissions', schema=None) as batch_op:
         batch_op.create_index('idx_dataset_permissions_tenant_id', ['tenant_id'])
+
     # ### end Alembic commands ###
 
 
@@ -46,9 +44,7 @@ def downgrade():
         sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey')
     )
 
-    with op.batch_alter_table('trace_app_config', schema=None) as batch_op:
-        batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'])
-
     with op.batch_alter_table('dataset_permissions', schema=None) as batch_op:
         batch_op.drop_index('idx_dataset_permissions_tenant_id')
+
     # ### end Alembic commands ###

api/models/model.py

@@ -6,6 +6,7 @@ from datetime import datetime
 from enum import Enum
 from typing import Any, Literal, Optional
 
+import sqlalchemy as sa
 from flask import request
 from flask_login import UserMixin
 from pydantic import BaseModel, Field
@@ -483,7 +484,7 @@ class RecommendedApp(db.Model):
     description = db.Column(db.JSON, nullable=False)
     copyright = db.Column(db.String(255), nullable=False)
     privacy_policy = db.Column(db.String(255), nullable=False)
-    custom_disclaimer = db.Column(db.String(255), nullable=True)
+    custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")
     category = db.Column(db.String(255), nullable=False)
     position = db.Column(db.Integer, nullable=False, default=0)
     is_listed = db.Column(db.Boolean, nullable=False, default=True)
@@ -1306,7 +1307,7 @@ class Site(db.Model):
     privacy_policy = db.Column(db.String(255))
     show_workflow_steps = db.Column(db.Boolean, nullable=False, server_default=db.text("true"))
     use_icon_as_answer_icon = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))
-    custom_disclaimer = db.Column(db.String(255), nullable=True)
+    custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")
     customize_domain = db.Column(db.String(255))
     customize_token_strategy = db.Column(db.String(255), nullable=False)
     prompt_public = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))
@@ -1384,6 +1385,7 @@ class UploadFile(db.Model):
     used_by: Mapped[str | None] = db.Column(StringUUID, nullable=True)
     used_at: Mapped[datetime | None] = db.Column(db.DateTime, nullable=True)
     hash: Mapped[str | None] = db.Column(db.String(255), nullable=True)
+    source_url: Mapped[str] = mapped_column(sa.TEXT, default="")
 
     def __init__(
         self,
@@ -1402,7 +1404,8 @@ class UploadFile(db.Model):
         used_by: str | None = None,
         used_at: datetime | None = None,
         hash: str | None = None,
-    ) -> None:
+        source_url: str = "",
+    ):
         self.tenant_id = tenant_id
         self.storage_type = storage_type
         self.key = key
@@ -1417,6 +1420,7 @@ class UploadFile(db.Model):
         self.used_by = used_by
         self.used_at = used_at
         self.hash = hash
+        self.source_url = source_url
 
 
 class ApiRequest(db.Model):

api/models/tools.py

@@ -1,6 +1,7 @@
 import json
 from typing import Optional
 
+import sqlalchemy as sa
 from sqlalchemy import ForeignKey
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -117,7 +118,7 @@ class ApiToolProvider(db.Model):
     # privacy policy
     privacy_policy = db.Column(db.String(255), nullable=True)
     # custom_disclaimer
-    custom_disclaimer = db.Column(db.String(255), nullable=True)
+    custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")
 
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))

api/models/workflow.py

@@ -1,9 +1,10 @@
 import json
 from collections.abc import Mapping, Sequence
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Any, Optional, Union
 
+import sqlalchemy as sa
 from sqlalchemy import func
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -99,14 +100,16 @@ class Workflow(db.Model):
     app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     type: Mapped[str] = mapped_column(db.String(255), nullable=False)
     version: Mapped[str] = mapped_column(db.String(255), nullable=False)
-    graph: Mapped[str] = mapped_column(db.Text)
-    _features: Mapped[str] = mapped_column("features")
+    graph: Mapped[str] = mapped_column(sa.Text)
+    _features: Mapped[str] = mapped_column("features", sa.TEXT)
     created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(
         db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
     )
-    updated_by: Mapped[str] = mapped_column(StringUUID)
-    updated_at: Mapped[datetime] = mapped_column(db.DateTime)
+    updated_by: Mapped[Optional[str]] = mapped_column(StringUUID)
+    updated_at: Mapped[datetime] = mapped_column(
+        sa.DateTime, nullable=False, default=datetime.now(tz=timezone.utc), server_onupdate=func.current_timestamp()
+    )
     _environment_variables: Mapped[str] = mapped_column(
         "environment_variables", db.Text, nullable=False, server_default="{}"
     )

api/schedule/clean_embedding_cache_task.py

@@ -14,7 +14,7 @@ from models.dataset import Embedding
 @app.celery.task(queue="dataset")
 def clean_embedding_cache_task():
     click.echo(click.style("Start clean embedding cache.", fg="green"))
-    clean_days = int(dify_config.CLEAN_DAY_SETTING)
+    clean_days = int(dify_config.PLAN_SANDBOX_CLEAN_DAY_SETTING)
     start_at = time.perf_counter()
     thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days)
     while True: