diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx index e691cc05f6..a58027bcd1 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC, SVGProps } from 'react' -import React, { useEffect } from 'react' +import React, { useEffect, useMemo } from 'react' import { usePathname } from 'next/navigation' import useSWR from 'swr' import { useTranslation } from 'react-i18next' @@ -203,12 +203,23 @@ const DatasetDetailLayout: FC = (props) => { datasetId, }, apiParams => fetchDatasetRelatedApps(apiParams.datasetId)) - const navigation = [ - { name: t('common.datasetMenus.documents'), href: `/datasets/${datasetId}/documents`, icon: DocumentTextIcon, selectedIcon: DocumentTextSolidIcon }, - { name: t('common.datasetMenus.hitTesting'), href: `/datasets/${datasetId}/hitTesting`, icon: TargetIcon, selectedIcon: TargetSolidIcon }, - // { name: 'api & webhook', href: `/datasets/${datasetId}/api`, icon: CommandLineIcon, selectedIcon: CommandLineSolidIcon }, - { name: t('common.datasetMenus.settings'), href: `/datasets/${datasetId}/settings`, icon: Cog8ToothIcon, selectedIcon: Cog8ToothSolidIcon }, - ] + const navigation = useMemo(() => { + const baseNavigation = [ + { name: t('common.datasetMenus.hitTesting'), href: `/datasets/${datasetId}/hitTesting`, icon: TargetIcon, selectedIcon: TargetSolidIcon }, + // { name: 'api & webhook', href: `/datasets/${datasetId}/api`, icon: CommandLineIcon, selectedIcon: CommandLineSolidIcon }, + { name: t('common.datasetMenus.settings'), href: `/datasets/${datasetId}/settings`, icon: Cog8ToothIcon, selectedIcon: Cog8ToothSolidIcon }, + ] + + if (datasetRes?.provider !== 'external') { + baseNavigation.unshift({ + name: t('common.datasetMenus.documents'), + href: `/datasets/${datasetId}/documents`, + icon: DocumentTextIcon, + selectedIcon: DocumentTextSolidIcon, + }) + } + return baseNavigation + }, [datasetRes?.provider, datasetId, t]) useEffect(() => { if (datasetRes) @@ -233,6 +244,7 @@ const DatasetDetailLayout: FC = (props) => { icon={datasetRes?.icon || 'https://static.dify.ai/images/dataset-default-icon.png'} icon_background={datasetRes?.icon_background || '#F5F5F5'} desc={datasetRes?.description || '--'} + isExternal={datasetRes?.provider === 'external'} navigation={navigation} extraInfo={!isCurrentWorkspaceDatasetOperator ? mode => : undefined} iconType={datasetRes?.data_source_type === DataSourceType.NOTION ? 'notion' : 'dataset'} diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index f532ca416f..3fbbf12d4e 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -8,6 +8,7 @@ import { useDebounceFn } from 'ahooks' import useSWR from 'swr' // Components +import ExternalAPIPanel from '../../components/datasets/external-api/external-api-panel' import Datasets from './Datasets' import DatasetFooter from './DatasetFooter' import ApiServer from './ApiServer' @@ -16,6 +17,9 @@ import TabSliderNew from '@/app/components/base/tab-slider-new' import SearchInput from '@/app/components/base/search-input' import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' +import Button from '@/app/components/base/button' +import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' +import { ExternalKnowledgeApiProvider } from '@/context/external-knowledge-api-context' // Services import { fetchDatasetApiBaseUrl } from '@/service/datasets' @@ -30,6 +34,7 @@ const Container = () => { const router = useRouter() const { currentWorkspace } = useAppContext() const showTagManagementModal = useTagStore(s => s.showTagManagementModal) + const [showExternalApiPanel, setShowExternalApiPanel] = useState(false) const options = useMemo(() => { return [ @@ -66,38 +71,49 @@ const Container = () => { useEffect(() => { if (currentWorkspace.role === 'normal') return router.replace('/apps') - }, [currentWorkspace]) + }, [currentWorkspace, router]) return ( -
-
- setActiveTab(newActiveTab)} - options={options} - /> - {activeTab === 'dataset' && ( -
- - -
- )} - {activeTab === 'api' && data && } -
- - {activeTab === 'dataset' && ( - <> - - - {showTagManagementModal && ( - + +
+
+ setActiveTab(newActiveTab)} + options={options} + /> + {activeTab === 'dataset' && ( +
+ + +
+ +
)} - - )} + {activeTab === 'api' && data && } +
- {activeTab === 'api' && data && } -
+ {activeTab === 'dataset' && ( + <> + + + {showTagManagementModal && ( + + )} + + )} + {activeTab === 'api' && data && } + + {showExternalApiPanel && setShowExternalApiPanel(false)} isShow={showExternalApiPanel} datasetBindings={[]} />} +
+
) } diff --git a/web/app/(commonLayout)/datasets/DatasetCard.tsx b/web/app/(commonLayout)/datasets/DatasetCard.tsx index f66c6bccf4..fa0625a528 100644 --- a/web/app/(commonLayout)/datasets/DatasetCard.tsx +++ b/web/app/(commonLayout)/datasets/DatasetCard.tsx @@ -18,6 +18,7 @@ import Divider from '@/app/components/base/divider' import RenameDatasetModal from '@/app/components/datasets/rename-modal' import type { Tag } from '@/app/components/base/tag-management/constant' import TagSelector from '@/app/components/base/tag-management/selector' +import CornerLabel from '@/app/components/base/corner-label' import { useAppContext } from '@/context/app-context' export type DatasetCardProps = { @@ -32,6 +33,7 @@ const DatasetCard = ({ const { t } = useTranslation() const { notify } = useContext(ToastContext) const { push } = useRouter() + const EXTERNAL_PROVIDER = 'external' as const const { isCurrentWorkspaceDatasetOperator } = useAppContext() const [tags, setTags] = useState(dataset.tags) @@ -39,6 +41,7 @@ const DatasetCard = ({ const [showRenameModal, setShowRenameModal] = useState(false) const [showConfirmDelete, setShowConfirmDelete] = useState(false) const [confirmMessage, setConfirmMessage] = useState('') + const isExternalProvider = (provider: string): boolean => provider === EXTERNAL_PROVIDER const detectIsUsedByApp = useCallback(async () => { try { const { is_using: isUsedByApp } = await checkIsUsedInApp(dataset.id) @@ -108,13 +111,16 @@ const DatasetCard = ({ return ( <>
{ e.preventDefault() - push(`/datasets/${dataset.id}/documents`) + isExternalProvider(dataset.provider) + ? push(`/datasets/${dataset.id}/hitTesting`) + : push(`/datasets/${dataset.id}/documents`) }} > + {isExternalProvider(dataset.provider) && }
- {dataset.document_count}{t('dataset.documentCount')} - · - {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')} - · - {dataset.app_count}{t('dataset.appCount')} + {dataset.provider === 'external' + ? <> + {dataset.app_count}{t('dataset.appCount')} + + : <> + {dataset.document_count}{t('dataset.documentCount')} + · + {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')} + · + {dataset.app_count}{t('dataset.appCount')} + + }
diff --git a/web/app/(commonLayout)/datasets/NewDatasetCard.tsx b/web/app/(commonLayout)/datasets/NewDatasetCard.tsx index f76efa5769..19a7ba2ab6 100644 --- a/web/app/(commonLayout)/datasets/NewDatasetCard.tsx +++ b/web/app/(commonLayout)/datasets/NewDatasetCard.tsx @@ -4,21 +4,31 @@ import { forwardRef } from 'react' import { useTranslation } from 'react-i18next' import { RiAddLine, + RiArrowRightLine, } from '@remixicon/react' const CreateAppCard = forwardRef((_, ref) => { const { t } = useTranslation() return ( - -
- ) }) diff --git a/web/app/(commonLayout)/datasets/connect/page.tsx b/web/app/(commonLayout)/datasets/connect/page.tsx new file mode 100644 index 0000000000..2a66837585 --- /dev/null +++ b/web/app/(commonLayout)/datasets/connect/page.tsx @@ -0,0 +1,13 @@ +import React from 'react' +import ExternalKnowledgeBaseConnector from '@/app/components/datasets/external-knowledge-base/connector' +import { ExternalKnowledgeApiProvider } from '@/context/external-knowledge-api-context' + +const ExternalKnowledgeBaseCreation = async () => { + return ( + + + + ) +} + +export default ExternalKnowledgeBaseCreation diff --git a/web/app/components/app-sidebar/basic.tsx b/web/app/components/app-sidebar/basic.tsx index c939cb7bb3..51fc10721e 100644 --- a/web/app/components/app-sidebar/basic.tsx +++ b/web/app/components/app-sidebar/basic.tsx @@ -1,4 +1,5 @@ import React from 'react' +import { useTranslation } from 'react-i18next' import AppIcon from '../base/app-icon' import Tooltip from '@/app/components/base/tooltip' @@ -6,6 +7,7 @@ export type IAppBasicProps = { iconType?: 'app' | 'api' | 'dataset' | 'webapp' | 'notion' icon?: string icon_background?: string | null + isExternal?: boolean name: string type: string | React.ReactNode hoverTip?: string @@ -52,7 +54,9 @@ const ICON_MAP = { notion: , } -export default function AppBasic({ icon, icon_background, name, type, hoverTip, textStyle, mode = 'expand', iconType = 'app' }: IAppBasicProps) { +export default function AppBasic({ icon, icon_background, name, isExternal, type, hoverTip, textStyle, mode = 'expand', iconType = 'app' }: IAppBasicProps) { + const { t } = useTranslation() + return (
{icon && icon_background && iconType === 'app' && ( @@ -83,6 +87,7 @@ export default function AppBasic({ icon, icon_background, name, type, hoverTip, }
{type}
+
{isExternal ? t('dataset.externalTag') : ''}
}
) diff --git a/web/app/components/app-sidebar/index.tsx b/web/app/components/app-sidebar/index.tsx index 5d5d407dc0..5ee063ad64 100644 --- a/web/app/components/app-sidebar/index.tsx +++ b/web/app/components/app-sidebar/index.tsx @@ -15,6 +15,7 @@ export type IAppDetailNavProps = { iconType?: 'app' | 'dataset' | 'notion' title: string desc: string + isExternal?: boolean icon: string icon_background: string navigation: Array<{ @@ -26,7 +27,7 @@ export type IAppDetailNavProps = { extraInfo?: (modeState: string) => React.ReactNode } -const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInfo, iconType = 'app' }: IAppDetailNavProps) => { +const AppDetailNav = ({ title, desc, isExternal, icon, icon_background, navigation, extraInfo, iconType = 'app' }: IAppDetailNavProps) => { const { appSidebarExpand, setAppSiderbarExpand } = useAppStore(useShallow(state => ({ appSidebarExpand: state.appSidebarExpand, setAppSiderbarExpand: state.setAppSiderbarExpand, @@ -70,6 +71,7 @@ const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInf icon_background={icon_background} name={title} type={desc} + isExternal={isExternal} /> )} diff --git a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx index 91cae54bb8..f99496b14f 100644 --- a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx +++ b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx @@ -174,6 +174,20 @@ const ConfigContent: FC = ({ ) } + { + selectedDatasetsMode.mixtureInternalAndExternal && ( +
+ {t('dataset.mixtureInternalAndExternalTip')} +
+ ) + } + { + selectedDatasetsMode.allExternal && ( +
+ {t('dataset.allExternalTip')} +
+ ) + } { selectedDatasetsMode.mixtureHighQualityAndEconomic && ( @@ -229,15 +243,15 @@ const ConfigContent: FC = ({ /> ) } -
{t('common.modelProvider.rerankModel.key')}
+
{t('common.modelProvider.rerankModel.key')}
{t('common.modelProvider.rerankModel.tip')} } - popupClassName='ml-0.5' - triggerClassName='ml-0.5 w-3.5 h-3.5' + popupClassName='ml-1' + triggerClassName='ml-1 w-4 h-4' />
diff --git a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx index 4493755ba0..0d94e599b4 100644 --- a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx +++ b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx @@ -47,7 +47,7 @@ const SelectDataSet: FC = ({ const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } }) setPage(getPage() + 1) setIsNoMore(!has_more) - const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)] + const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')] setDataSets(newList) setLoaded(true) if (!selected.find(item => !item.name)) @@ -145,6 +145,11 @@ const SelectDataSet: FC = ({ /> ) } + { + item.provider === 'external' && ( + + ) + }
))} diff --git a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx index e538c347d9..05e7b46ced 100644 --- a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx +++ b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx @@ -5,8 +5,10 @@ import { useTranslation } from 'react-i18next' import { isEqual } from 'lodash-es' import { RiCloseLine } from '@remixicon/react' import { BookOpenIcon } from '@heroicons/react/24/outline' +import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' import cn from '@/utils/classnames' import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio' +import Divider from '@/app/components/base/divider' import Button from '@/app/components/base/button' import type { DataSet } from '@/models/datasets' import { useToastContext } from '@/app/components/base/toast' @@ -14,6 +16,7 @@ import { updateDatasetSetting } from '@/service/datasets' import { useAppContext } from '@/context/app-context' import { useModalContext } from '@/context/modal-context' import type { RetrievalConfig } from '@/types/app' +import RetrievalSettings from '@/app/components/datasets/external-knowledge-base/create/RetrievalSettings' import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config' import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model' @@ -56,6 +59,8 @@ const SettingsModal: FC = ({ const { t } = useTranslation() const { notify } = useToastContext() const ref = useRef(null) + const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2) + const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5) const { setShowAccountSettingModal } = useModalContext() const [loading, setLoading] = useState(false) @@ -73,6 +78,13 @@ const SettingsModal: FC = ({ const [isHideChangedTip, setIsHideChangedTip] = useState(false) const isRetrievalChanged = !isEqual(retrievalConfig, localeCurrentDataset?.retrieval_model_dict) || indexMethod !== localeCurrentDataset?.indexing_technique + const handleSettingsChange = (data: { top_k?: number; score_threshold?: number }) => { + if (data.top_k !== undefined) + setTopK(data.top_k) + if (data.score_threshold !== undefined) + setScoreThreshold(data.score_threshold) + } + const handleSave = async () => { if (loading) return @@ -107,10 +119,16 @@ const SettingsModal: FC = ({ description, permission, indexing_technique: indexMethod, + external_retrieval_model: { + top_k: topK, + score_threshold: scoreThreshold, + }, retrieval_model: { ...postRetrievalConfig, score_threshold: postRetrievalConfig.score_threshold_enabled ? postRetrievalConfig.score_threshold : 0, }, + external_knowledge_id: currentDataset!.external_knowledge_info.external_knowledge_id, + external_knowledge_api_id: currentDataset!.external_knowledge_info.external_knowledge_api_id, embedding_model: localeCurrentDataset.embedding_model, embedding_model_provider: localeCurrentDataset.embedding_model_provider, }, @@ -178,7 +196,7 @@ const SettingsModal: FC = ({ }}>
- {t('datasetSettings.form.name')} +
{t('datasetSettings.form.name')}
= ({
- {t('datasetSettings.form.desc')} +
{t('datasetSettings.form.desc')}