diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py index 8d7523e46f..2edc0d6470 100644 --- a/api/controllers/console/workspace/plugin.py +++ b/api/controllers/console/workspace/plugin.py @@ -11,6 +11,7 @@ from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.model_runtime.utils.encoders import jsonable_encoder +from core.plugin.entities.plugin_daemon import InstallPluginMessage from libs.login import login_required from services.plugin.plugin_service import PluginService @@ -27,7 +28,7 @@ class PluginDebuggingKeyApi(Resource): tenant_id = user.current_tenant_id return { - "key": PluginService.get_plugin_debugging_key(tenant_id), + "key": PluginService.get_debugging_key(tenant_id), "host": dify_config.PLUGIN_REMOTE_INSTALL_HOST, "port": dify_config.PLUGIN_REMOTE_INSTALL_PORT, } @@ -40,7 +41,7 @@ class PluginListApi(Resource): def get(self): user = current_user tenant_id = user.current_tenant_id - plugins = PluginService.list_plugins(tenant_id) + plugins = PluginService.list(tenant_id) return jsonable_encoder({"plugins": plugins}) @@ -88,9 +89,7 @@ class PluginInstallFromUniqueIdentifierApi(Resource): tenant_id = user.current_tenant_id - return { - "success": PluginService.install_plugin_from_unique_identifier(tenant_id, args["plugin_unique_identifier"]) - } + return {"success": PluginService.install_from_unique_identifier(tenant_id, args["plugin_unique_identifier"])} class PluginInstallFromPkgApi(Resource): @@ -108,9 +107,71 @@ class PluginInstallFromPkgApi(Resource): content = file.read() def generator(): - response = PluginService.install_plugin_from_pkg(tenant_id, content) - for message in response: - yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" + try: + response = PluginService.install_from_local_pkg(tenant_id, content) + for message in response: + yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" + except ValueError as e: + error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) + yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" + + return Response(generator(), mimetype="text/event-stream") + + +class PluginInstallFromGithubApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + + tenant_id = user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument("repo", type=str, required=True, location="json") + parser.add_argument("version", type=str, required=True, location="json") + parser.add_argument("package", type=str, required=True, location="json") + args = parser.parse_args() + + def generator(): + try: + response = PluginService.install_from_github_pkg( + tenant_id, args["repo"], args["version"], args["package"] + ) + for message in response: + yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" + except ValueError as e: + error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) + yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" + + return Response(generator(), mimetype="text/event-stream") + + +class PluginInstallFromMarketplaceApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + user = current_user + if not user.is_admin_or_owner: + raise Forbidden() + + tenant_id = user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument("plugin_unique_identifier", type=str, required=True, location="json") + args = parser.parse_args() + + def generator(): + try: + response = PluginService.install_from_marketplace_pkg(tenant_id, args["plugin_unique_identifier"]) + for message in response: + yield f"data: {json.dumps(jsonable_encoder(message))}\n\n" + except ValueError as e: + error_message = InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e)) + yield f"data: {json.dumps(jsonable_encoder(error_message))}\n\n" return Response(generator(), mimetype="text/event-stream") @@ -130,7 +191,7 @@ class PluginUninstallApi(Resource): tenant_id = user.current_tenant_id - return {"success": PluginService.uninstall_plugin(tenant_id, args["plugin_installation_id"])} + return {"success": PluginService.uninstall(tenant_id, args["plugin_installation_id"])} api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key") @@ -139,4 +200,6 @@ api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon") api.add_resource(PluginInstallCheckUniqueIdentifierApi, "/workspaces/current/plugin/install/check_unique_identifier") api.add_resource(PluginInstallFromUniqueIdentifierApi, "/workspaces/current/plugin/install/from_unique_identifier") api.add_resource(PluginInstallFromPkgApi, "/workspaces/current/plugin/install/from_pkg") +api.add_resource(PluginInstallFromGithubApi, "/workspaces/current/plugin/install/from_github") +api.add_resource(PluginInstallFromMarketplaceApi, "/workspaces/current/plugin/install/from_marketplace") api.add_resource(PluginUninstallApi, "/workspaces/current/plugin/uninstall") diff --git a/api/core/helper/download.py b/api/core/helper/download.py new file mode 100644 index 0000000000..d54229c420 --- /dev/null +++ b/api/core/helper/download.py @@ -0,0 +1,17 @@ +from core.helper import ssrf_proxy + + +def download_with_size_limit(url, max_download_size: int, **kwargs): + response = ssrf_proxy.get(url, **kwargs) + if response.status_code == 404: + raise ValueError("file not found") + + total_size = 0 + chunks = [] + for chunk in response.iter_bytes(): + total_size += len(chunk) + if total_size > max_download_size: + raise ValueError("Max file size reached") + chunks.append(chunk) + content = b"".join(chunks) + return content diff --git a/api/core/plugin/entities/plugin.py b/api/core/plugin/entities/plugin.py index 72ecd5ba23..9afde0fef8 100644 --- a/api/core/plugin/entities/plugin.py +++ b/api/core/plugin/entities/plugin.py @@ -1,4 +1,5 @@ import datetime +from enum import Enum from typing import Optional from pydantic import BaseModel, Field @@ -10,6 +11,13 @@ from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolProviderEntity +class PluginInstallationSource(str, Enum): + Github = "github" + Marketplace = "marketplace" + Package = "package" + Remote = "remote" + + class PluginResourceRequirements(BaseModel): memory: int @@ -75,3 +83,14 @@ class PluginEntity(BasePluginEntity): endpoints_active: int runtime_type: str version: str + + +class GithubPackage(BaseModel): + repo: str + version: str + package: str + + +class GithubVersion(BaseModel): + repo: str + version: str diff --git a/api/core/plugin/manager/plugin.py b/api/core/plugin/manager/plugin.py index 6e11f95d91..0e89c257e8 100644 --- a/api/core/plugin/manager/plugin.py +++ b/api/core/plugin/manager/plugin.py @@ -1,6 +1,8 @@ -from collections.abc import Generator +import json +from collections.abc import Generator, Mapping +from typing import Any -from core.plugin.entities.plugin import PluginEntity +from core.plugin.entities.plugin import PluginEntity, PluginInstallationSource from core.plugin.entities.plugin_daemon import InstallPluginMessage from core.plugin.manager.base import BasePluginManager @@ -25,7 +27,12 @@ class PluginInstallationManager(BasePluginManager): ) def install_from_pkg( - self, tenant_id: str, pkg: bytes, verify_signature: bool = False + self, + tenant_id: str, + pkg: bytes, + source: PluginInstallationSource, + meta: Mapping[str, Any], + verify_signature: bool = False, ) -> Generator[InstallPluginMessage, None, None]: """ Install a plugin from a package. @@ -33,7 +40,12 @@ class PluginInstallationManager(BasePluginManager): # using multipart/form-data to encode body body = { "dify_pkg": ("dify_pkg", pkg, "application/octet-stream"), + } + + data = { "verify_signature": "true" if verify_signature else "false", + "source": source.value, + "meta": json.dumps(meta), } return self._request_with_plugin_daemon_response_stream( @@ -41,6 +53,7 @@ class PluginInstallationManager(BasePluginManager): f"plugin/{tenant_id}/management/install/pkg", InstallPluginMessage, files=body, + data=data, ) def install_from_identifier(self, tenant_id: str, identifier: str) -> bool: diff --git a/api/services/plugin/plugin_service.py b/api/services/plugin/plugin_service.py index 25c62d18e4..92ea3caa3f 100644 --- a/api/services/plugin/plugin_service.py +++ b/api/services/plugin/plugin_service.py @@ -1,7 +1,8 @@ from collections.abc import Generator from mimetypes import guess_type -from core.plugin.entities.plugin import PluginEntity +from core.helper.download import download_with_size_limit +from core.plugin.entities.plugin import PluginEntity, PluginInstallationSource from core.plugin.entities.plugin_daemon import InstallPluginMessage, PluginDaemonInnerError from core.plugin.manager.asset import PluginAssetManager from core.plugin.manager.debugging import PluginDebuggingManager @@ -10,12 +11,12 @@ from core.plugin.manager.plugin import PluginInstallationManager class PluginService: @staticmethod - def get_plugin_debugging_key(tenant_id: str) -> str: + def get_debugging_key(tenant_id: str) -> str: manager = PluginDebuggingManager() return manager.get_debugging_key(tenant_id) @staticmethod - def list_plugins(tenant_id: str) -> list[PluginEntity]: + def list(tenant_id: str) -> list[PluginEntity]: manager = PluginInstallationManager() return manager.list_plugins(tenant_id) @@ -32,19 +33,71 @@ class PluginService: return manager.fetch_plugin_by_identifier(tenant_id, plugin_unique_identifier) @staticmethod - def install_plugin_from_unique_identifier(tenant_id: str, plugin_unique_identifier: str) -> bool: + def install_from_unique_identifier(tenant_id: str, plugin_unique_identifier: str) -> bool: manager = PluginInstallationManager() return manager.install_from_identifier(tenant_id, plugin_unique_identifier) @staticmethod - def install_plugin_from_pkg(tenant_id: str, pkg: bytes) -> Generator[InstallPluginMessage, None, None]: + def install_from_local_pkg(tenant_id: str, pkg: bytes) -> Generator[InstallPluginMessage, None, None]: + """ + Install plugin from uploaded package files + """ manager = PluginInstallationManager() try: - yield from manager.install_from_pkg(tenant_id, pkg) + yield from manager.install_from_pkg(tenant_id, pkg, PluginInstallationSource.Package, {}) except PluginDaemonInnerError as e: yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) @staticmethod - def uninstall_plugin(tenant_id: str, plugin_installation_id: str) -> bool: + def install_from_github_pkg( + tenant_id: str, repo: str, version: str, package: str + ) -> Generator[InstallPluginMessage, None, None]: + """ + Install plugin from github release package files + """ + pkg = download_with_size_limit( + f"https://github.com/{repo}/releases/download/{version}/{package}", 15 * 1024 * 1024 + ) + + manager = PluginInstallationManager() + try: + yield from manager.install_from_pkg( + tenant_id, + pkg, + PluginInstallationSource.Github, + { + "repo": repo, + "version": version, + "package": package, + }, + ) + except PluginDaemonInnerError as e: + yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) + + @staticmethod + def install_from_marketplace_pkg( + tenant_id: str, plugin_unique_identifier: str + ) -> Generator[InstallPluginMessage, None, None]: + """ + TODO: wait for marketplace api + """ + manager = PluginInstallationManager() + + pkg = b"" + + try: + yield from manager.install_from_pkg( + tenant_id, + pkg, + PluginInstallationSource.Marketplace, + { + "plugin_unique_identifier": plugin_unique_identifier, + }, + ) + except PluginDaemonInnerError as e: + yield InstallPluginMessage(event=InstallPluginMessage.Event.Error, data=str(e.message)) + + @staticmethod + def uninstall(tenant_id: str, plugin_installation_id: str) -> bool: manager = PluginInstallationManager() return manager.uninstall(tenant_id, plugin_installation_id)