feat: connect knowledge base to app

This commit is contained in:
Yi 2024-09-27 15:50:22 +08:00
parent 1597f34471
commit 5554cf7b20
23 changed files with 284 additions and 260 deletions

View File

@ -111,7 +111,7 @@ const DatasetCard = ({
return (
<>
<div
className='group relative col-span-1 bg-white border-2 border-solid border-transparent rounded-xl shadow-sm min-h-[160px] flex flex-col transition-all duration-200 ease-in-out cursor-pointer hover:shadow-lg'
className='group relative col-span-1 bg-white border-[0.5px] border-solid border-transparent rounded-xl shadow-sm min-h-[160px] flex flex-col transition-all duration-200 ease-in-out cursor-pointer hover:shadow-lg'
data-disable-nprogress={true}
onClick={(e) => {
e.preventDefault()
@ -144,11 +144,18 @@ const DatasetCard = ({
className={cn('truncate', (!dataset.embedding_available || !dataset.document_count) && 'opacity-50')}
title={`${dataset.document_count}${t('dataset.documentCount')} · ${Math.round(dataset.word_count / 1000)}${t('dataset.wordCount')} · ${dataset.app_count}${t('dataset.appCount')}`}
>
<span>{dataset.document_count}{t('dataset.documentCount')}</span>
<span className='shrink-0 mx-0.5 w-1 text-gray-400'>·</span>
<span>{Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}</span>
<span className='shrink-0 mx-0.5 w-1 text-gray-400'>·</span>
<span>{dataset.app_count}{t('dataset.appCount')}</span>
{dataset.provider === 'external'
? <>
<span>{dataset.app_count}{t('dataset.appCount')}</span>
</>
: <>
<span>{dataset.document_count}{t('dataset.documentCount')}</span>
<span className='shrink-0 mx-0.5 w-1 text-gray-400'>·</span>
<span>{Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}</span>
<span className='shrink-0 mx-0.5 w-1 text-gray-400'>·</span>
<span>{dataset.app_count}{t('dataset.appCount')}</span>
</>
}
</div>
</div>
</div>

View File

@ -1,4 +1,5 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import AppIcon from '../base/app-icon'
import Tooltip from '@/app/components/base/tooltip'
@ -54,6 +55,8 @@ const ICON_MAP = {
}
export default function AppBasic({ icon, icon_background, name, isExternal, type, hoverTip, textStyle, mode = 'expand', iconType = 'app' }: IAppBasicProps) {
const { t } = useTranslation()
return (
<div className="flex items-start p-1">
{icon && icon_background && iconType === 'app' && (
@ -84,7 +87,7 @@ export default function AppBasic({ icon, icon_background, name, isExternal, type
}
</div>
<div className={`text-xs font-normal text-gray-500 group-hover:text-gray-700 break-all ${textStyle?.extra ?? ''}`}>{type}</div>
<div className='text-text-tertiary system-2xs-medium-uppercase'>{isExternal ? 'External' : ''}</div>
<div className='text-text-tertiary system-2xs-medium-uppercase'>{isExternal ? t('dataset.externalTag') : ''}</div>
</div>}
</div>
)

View File

@ -174,6 +174,20 @@ const ConfigContent: FC<Props> = ({
</div>
)
}
{
selectedDatasetsMode.mixtureInternalAndExternal && (
<div className='mt-4 system-xs-medium text-text-warning'>
{t('dataset.mixtureInternalAndExternalTip')}
</div>
)
}
{
selectedDatasetsMode.allExternal && (
<div className='mt-4 system-xs-medium text-text-warning'>
{t('dataset.allExternalTip')}
</div>
)
}
{
selectedDatasetsMode.mixtureHighQualityAndEconomic
&& (
@ -229,15 +243,15 @@ const ConfigContent: FC<Props> = ({
/>
)
}
<div className='ml-2 leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.rerankModel.key')}</div>
<div className='leading-[32px] text-text-secondary system-sm-semibold'>{t('common.modelProvider.rerankModel.key')}</div>
<Tooltip
popupContent={
<div className="w-[200px]">
{t('common.modelProvider.rerankModel.tip')}
</div>
}
popupClassName='ml-0.5'
triggerClassName='ml-0.5 w-3.5 h-3.5'
popupClassName='ml-1'
triggerClassName='ml-1 w-4 h-4'
/>
</div>
<div>

View File

@ -47,7 +47,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } })
setPage(getPage() + 1)
setIsNoMore(!has_more)
const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)]
const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')]
setDataSets(newList)
setLoaded(true)
if (!selected.find(item => !item.name))
@ -145,6 +145,11 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
/>
)
}
{
item.provider === 'external' && (
<Badge text={t('dataset.externalTag')} />
)
}
</div>
))}
</div>

View File

@ -5,8 +5,10 @@ import { useTranslation } from 'react-i18next'
import { isEqual } from 'lodash-es'
import { RiCloseLine } from '@remixicon/react'
import { BookOpenIcon } from '@heroicons/react/24/outline'
import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
import cn from '@/utils/classnames'
import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio'
import Divider from '@/app/components/base/divider'
import Button from '@/app/components/base/button'
import type { DataSet } from '@/models/datasets'
import { useToastContext } from '@/app/components/base/toast'
@ -14,6 +16,7 @@ import { updateDatasetSetting } from '@/service/datasets'
import { useAppContext } from '@/context/app-context'
import { useModalContext } from '@/context/modal-context'
import type { RetrievalConfig } from '@/types/app'
import RetrievalSettings from '@/app/components/datasets/external-knowledge-base/create/RetrievalSettings'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
@ -56,6 +59,8 @@ const SettingsModal: FC<SettingsModalProps> = ({
const { t } = useTranslation()
const { notify } = useToastContext()
const ref = useRef(null)
const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2)
const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5)
const { setShowAccountSettingModal } = useModalContext()
const [loading, setLoading] = useState(false)
@ -73,6 +78,13 @@ const SettingsModal: FC<SettingsModalProps> = ({
const [isHideChangedTip, setIsHideChangedTip] = useState(false)
const isRetrievalChanged = !isEqual(retrievalConfig, localeCurrentDataset?.retrieval_model_dict) || indexMethod !== localeCurrentDataset?.indexing_technique
const handleSettingsChange = (data: { top_k?: number; score_threshold?: number }) => {
if (data.top_k !== undefined)
setTopK(data.top_k)
if (data.score_threshold !== undefined)
setScoreThreshold(data.score_threshold)
}
const handleSave = async () => {
if (loading)
return
@ -107,10 +119,16 @@ const SettingsModal: FC<SettingsModalProps> = ({
description,
permission,
indexing_technique: indexMethod,
external_retrieval_model: {
top_k: topK,
score_threshold: scoreThreshold,
},
retrieval_model: {
...postRetrievalConfig,
score_threshold: postRetrievalConfig.score_threshold_enabled ? postRetrievalConfig.score_threshold : 0,
},
external_knowledge_id: currentDataset!.external_knowledge_info.external_knowledge_id,
external_knowledge_api_id: currentDataset!.external_knowledge_info.external_knowledge_api_id,
embedding_model: localeCurrentDataset.embedding_model,
embedding_model_provider: localeCurrentDataset.embedding_model_provider,
},
@ -178,7 +196,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
}}>
<div className={cn(rowClass, 'items-center')}>
<div className={labelClass}>
{t('datasetSettings.form.name')}
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.name')}</div>
</div>
<input
value={localeCurrentDataset.name}
@ -189,7 +207,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
</div>
<div className={cn(rowClass)}>
<div className={labelClass}>
{t('datasetSettings.form.desc')}
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.desc')}</div>
</div>
<div className='w-full'>
<textarea
@ -206,7 +224,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
</div>
<div className={rowClass}>
<div className={labelClass}>
<div>{t('datasetSettings.form.permissions')}</div>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.permissions')}</div>
</div>
<div className='w-full'>
<PermissionSelector
@ -219,24 +237,25 @@ const SettingsModal: FC<SettingsModalProps> = ({
/>
</div>
</div>
<div className="w-full h-0 border-b-[0.5px] border-b-gray-200 my-2"></div>
<div className={cn(rowClass)}>
<div className={labelClass}>
{t('datasetSettings.form.indexMethod')}
{currentDataset && currentDataset.indexing_technique && (
<div className={cn(rowClass)}>
<div className={labelClass}>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.indexMethod')}</div>
</div>
<div className='grow'>
<IndexMethodRadio
disable={!localeCurrentDataset?.embedding_available}
value={indexMethod}
onChange={v => setIndexMethod(v!)}
itemClassName='sm:!w-[280px]'
/>
</div>
</div>
<div className='grow'>
<IndexMethodRadio
disable={!localeCurrentDataset?.embedding_available}
value={indexMethod}
onChange={v => setIndexMethod(v!)}
itemClassName='sm:!w-[280px]'
/>
</div>
</div>
)}
{indexMethod === 'high_quality' && (
<div className={cn(rowClass)}>
<div className={labelClass}>
{t('datasetSettings.form.embeddingModel')}
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.embeddingModel')}</div>
</div>
<div className='w-full'>
<div className='w-full h-9 rounded-lg bg-gray-100 opacity-60'>
@ -258,32 +277,74 @@ const SettingsModal: FC<SettingsModalProps> = ({
)}
{/* Retrieval Method Config */}
<div className={rowClass}>
<div className={cn(labelClass, 'w-auto min-w-[168px]')}>
<div>
<div>{t('datasetSettings.form.retrievalSetting.title')}</div>
<div className='leading-[18px] text-xs font-normal text-gray-500'>
<a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
{t('datasetSettings.form.retrievalSetting.description')}
{currentDataset?.provider === 'external'
? <>
<div className={rowClass}><Divider/></div>
<div className={rowClass}>
<div className={labelClass}>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
</div>
<RetrievalSettings
topK={topK}
scoreThreshold={scoreThreshold}
onChange={handleSettingsChange}
isInRetrievalSetting={true}
/>
</div>
<div className={rowClass}><Divider/></div>
<div className={rowClass}>
<div className={labelClass}>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeAPI')}</div>
</div>
<div className='w-full max-w-[480px]'>
<div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
<ApiConnectionMod className='w-4 h-4 text-text-secondary' />
<div className='overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>
{currentDataset?.external_knowledge_info.external_knowledge_api_name}
</div>
<div className='text-text-tertiary system-xs-regular'>·</div>
<div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_api_endpoint}</div>
</div>
</div>
</div>
</div>
<div>
{indexMethod === 'high_quality'
? (
<RetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
: (
<EconomicalRetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)}
</div>
</div>
<div className={rowClass}>
<div className={labelClass}>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeID')}</div>
</div>
<div className='w-full max-w-[480px]'>
<div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
<div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_id}</div>
</div>
</div>
</div>
<div className={rowClass}><Divider/></div>
</>
: <div className={rowClass}>
<div className={cn(labelClass, 'w-auto min-w-[168px]')}>
<div>
<div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
<div className='leading-[18px] text-xs font-normal text-gray-500'>
<a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
{t('datasetSettings.form.retrievalSetting.description')}
</div>
</div>
</div>
<div>
{indexMethod === 'high_quality'
? (
<RetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)
: (
<EconomicalRetrievalMethodConfig
value={retrievalConfig}
onChange={setRetrievalConfig}
/>
)}
</div>
</div>}
</div>
{isRetrievalChanged && !isHideChangedTip && (
<div className='absolute z-10 left-[30px] right-[30px] bottom-[76px] flex h-10 items-center px-3 rounded-lg border border-[#FEF0C7] bg-[#FFFAEB] shadow-lg justify-between'>

View File

@ -1,42 +0,0 @@
import type { Dispatch, SetStateAction } from 'react'
export enum ValidatedEndpointStatus {
Success = 'success',
Error = 'error',
}
export type ValidatedStatusState = {
status?: ValidatedEndpointStatus
message?: string
}
export type Status = 'add' | 'fail' | 'success'
export type ValidateValue = string
export type ValidateCallback = {
before: (v?: ValidateValue) => boolean | undefined
run?: (v?: ValidateValue) => Promise<ValidatedStatusState>
}
export type Form = {
key: string
title: string
placeholder: string
value?: string
validate?: ValidateCallback
handleFocus?: (v: ValidateValue, dispatch: Dispatch<SetStateAction<ValidateValue>>) => void
}
export type KeyFrom = {
text: string
link: string
}
export type KeyValidatorProps = {
type: string
title: React.ReactNode
status: Status
forms: Form[]
keyFrom: KeyFrom
}

View File

@ -1,31 +0,0 @@
import { useState } from 'react'
import { useDebounceFn } from 'ahooks'
import type { DebouncedFunc } from 'lodash-es'
import { ValidatedEndpointStatus } from './declarations'
import type { ValidateCallback, ValidateValue, ValidatedStatusState } from './declarations'
export const useValidateEndpoint: (value: ValidateValue) => [DebouncedFunc<(validateCallback: ValidateCallback) => Promise<void>>, boolean, ValidatedStatusState] = (value) => {
const [validating, setValidating] = useState(false)
const [validatedStatus, setValidatedStatus] = useState<ValidatedStatusState>({})
const { run } = useDebounceFn(async (validateCallback: ValidateCallback) => {
if (!validateCallback.before(value)) {
setValidating(false)
setValidatedStatus({})
return
}
setValidating(true)
if (validateCallback.run) {
const res = await validateCallback?.run(value)
setValidatedStatus(
res.status === 'success'
? { status: ValidatedEndpointStatus.Success }
: { status: ValidatedEndpointStatus.Error, message: res.message })
setValidating(false)
}
}, { wait: 1000 })
return [run, validating, validatedStatus]
}

View File

@ -1,6 +1,7 @@
import React, { useState } from 'react'
import type { FC } from 'react'
import { useTranslation } from 'react-i18next'
import { RiBookOpenLine } from '@remixicon/react'
import type { CreateExternalAPIReq, FormSchema } from '../declarations'
import Input from '@/app/components/base/input'
import cn from '@/utils/classnames'
@ -11,10 +12,6 @@ type FormProps = {
fieldLabelClassName?: string
value: CreateExternalAPIReq
onChange: (val: CreateExternalAPIReq) => void
validatingEndpoint: boolean
validatedApiKeySuccess?: boolean
validatingApiKey: boolean
validatedEndpointSuccess?: boolean
formSchemas: FormSchema[]
inputClassName?: string
}
@ -26,10 +23,6 @@ const Form: FC<FormProps> = React.memo(({
value,
onChange,
formSchemas,
validatingEndpoint,
validatingApiKey,
validatedApiKeySuccess,
validatedEndpointSuccess,
inputClassName,
}) => {
const { t, i18n } = useTranslation()
@ -57,10 +50,23 @@ const Form: FC<FormProps> = React.memo(({
return (
<div key={variable} className={cn(itemClassName, 'flex flex-col items-start gap-1 self-stretch')}>
<label className={cn(fieldLabelClassName, 'text-text-secondary system-sm-semibold')} htmlFor={variable}>
{label[i18n.language] || label.en_US}
{required && <span className='ml-1 text-red-500'>*</span>}
</label>
<div className="flex justify-between items-center w-full">
<label className={cn(fieldLabelClassName, 'text-text-secondary system-sm-semibold')} htmlFor={variable}>
{label[i18n.language] || label.en_US}
{required && <span className='ml-1 text-red-500'>*</span>}
</label>
{variable === 'endpoint' && (
<a
href={'https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' || '/'}
target='_blank'
rel='noopener noreferrer'
className='text-text-accent body-xs-regular flex items-center'
>
<RiBookOpenLine className='w-3 h-3 text-text-accent mr-1' />
{t('dataset.externalAPIPanelDocumentation')}
</a>
)}
</div>
<Input
type={type === 'secret' ? 'password' : 'text'}
id={variable}

View File

@ -11,10 +11,6 @@ import {
RiInformation2Line,
RiLock2Fill,
} from '@remixicon/react'
import { useValidateApiKey } from '../key-validator/hooks'
import { ValidatedApiKeyStatus } from '../key-validator/declarations'
import { ValidatedEndpointStatus } from '../endpoint-validator/declarations'
import { useValidateEndpoint } from '../endpoint-validator/hooks'
import type { CreateExternalAPIReq, FormSchema } from '../declarations'
import Form from './Form'
import ActionButton from '@/app/components/base/action-button'
@ -76,18 +72,28 @@ const AddExternalAPIModal: FC<AddExternalAPIModalProps> = ({ data, onSave, onCan
setFormData(data)
}, [isEditMode, data])
const [, validatingApiKey, validatedApiKeyStatusState] = useValidateApiKey(formData.settings.api_key)
const [, validatingEndpoint, validatedEndpointStatusState] = useValidateEndpoint(formData.settings.endpoint)
const hasEmptyInputs = Object.values(formData).includes('')
const hasEmptyInputs = Object.values(formData).some(value =>
typeof value === 'string' ? value.trim() === '' : Object.values(value).some(v => v.trim() === ''),
)
const handleDataChange = (val: CreateExternalAPIReq) => {
setFormData(val)
}
const handleSave = async () => {
if (formData && formData.settings.api_key && formData.settings.api_key?.length < 5) {
notify({ type: 'error', message: t('common.apiBasedExtension.modal.apiKey.lengthError') })
setLoading(false)
return
}
try {
setLoading(true)
if (isEditMode && onEdit) {
await onEdit(formData)
await onEdit(
{
...formData,
settings: { ...formData.settings, api_key: formData.settings.api_key ? '[__HIDDEN__]' : formData.settings.api_key },
},
)
notify({ type: 'success', message: 'External API updated successfully' })
}
else {
@ -154,10 +160,6 @@ const AddExternalAPIModal: FC<AddExternalAPIModalProps> = ({ data, onSave, onCan
<Form
value={formData}
onChange={handleDataChange}
validatingApiKey={validatingApiKey}
validatedApiKeySuccess={validatedApiKeyStatusState?.status === ValidatedApiKeyStatus.Success}
validatingEndpoint={validatingEndpoint}
validatedEndpointSuccess={validatedEndpointStatusState?.status === ValidatedEndpointStatus.Success}
formSchemas={formSchemas}
className='flex px-6 py-3 flex-col justify-center items-start gap-4 self-stretch'
/>

View File

@ -52,7 +52,7 @@ const ExternalAPIPanel: React.FC<ExternalAPIPanelProps> = ({ onClose, isShow, da
<div className='flex flex-col items-start gap-1 flex-grow'>
<div className='self-stretch text-text-primary system-xl-semibold'>{t('dataset.externalAPIPanelTitle')}</div>
<div className='self-stretch text-text-tertiary body-xs-regular'>{t('dataset.externalAPIPanelDescription')}</div>
<a className='flex justify-center items-center gap-1 self-stretch cursor-pointer' href='https://docs.dify.ai/docs/external-api' target='_blank'>
<a className='flex justify-center items-center gap-1 self-stretch cursor-pointer' href='https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' target='_blank'>
<RiBookOpenLine className='w-3 h-3 text-text-accent' />
<div className='flex-grow text-text-accent body-xs-regular'>{t('dataset.externalAPIPanelDocumentation')}</div>
</a>

View File

@ -1,42 +0,0 @@
import type { Dispatch, SetStateAction } from 'react'
export enum ValidatedApiKeyStatus {
Success = 'success',
Error = 'error',
}
export type ValidatedStatusState = {
status?: ValidatedApiKeyStatus
message?: string
}
export type Status = 'add' | 'fail' | 'success'
export type ValidateValue = string
export type ValidateCallback = {
before: (v?: ValidateValue) => boolean | undefined
run?: (v?: ValidateValue) => Promise<ValidatedStatusState>
}
export type Form = {
key: string
title: string
placeholder: string
value?: string
validate?: ValidateCallback
handleFocus?: (v: ValidateValue, dispatch: Dispatch<SetStateAction<ValidateValue>>) => void
}
export type KeyFrom = {
text: string
link: string
}
export type KeyValidatorProps = {
type: string
title: React.ReactNode
status: Status
forms: Form[]
keyFrom: KeyFrom
}

View File

@ -1,31 +0,0 @@
import { useState } from 'react'
import { useDebounceFn } from 'ahooks'
import type { DebouncedFunc } from 'lodash-es'
import { ValidatedApiKeyStatus } from './declarations'
import type { ValidateCallback, ValidateValue, ValidatedStatusState } from './declarations'
export const useValidateApiKey: (value: ValidateValue) => [DebouncedFunc<(validateCallback: ValidateCallback) => Promise<void>>, boolean, ValidatedStatusState] = (value) => {
const [validating, setValidating] = useState(false)
const [validatedStatus, setValidatedStatus] = useState<ValidatedStatusState>({})
const { run } = useDebounceFn(async (validateCallback: ValidateCallback) => {
if (!validateCallback.before(value)) {
setValidating(false)
setValidatedStatus({})
return
}
setValidating(true)
if (validateCallback.run) {
const res = await validateCallback?.run(value)
setValidatedStatus(
res.status === 'success'
? { status: ValidatedApiKeyStatus.Success }
: { status: ValidatedApiKeyStatus.Error, message: res.message })
setValidating(false)
}
}, { wait: 1000 })
return [run, validating, validatedStatus]
}

View File

@ -1,6 +1,7 @@
'use client'
import React, { useState } from 'react'
import { useRouter } from 'next/navigation'
import { useToastContext } from '@/app/components/base/toast'
import ExternalKnowledgeBaseCreate from '@/app/components/datasets/external-knowledge-base/create'
import type { CreateKnowledgeBaseReq } from '@/app/components/datasets/external-knowledge-base/create/declarations'
@ -9,18 +10,21 @@ import { createExternalKnowledgeBase } from '@/service/datasets'
const ExternalKnowledgeBaseConnector = () => {
const { notify } = useToastContext()
const [loading, setLoading] = useState(false)
const router = useRouter()
const handleConnect = async (formValue: CreateKnowledgeBaseReq) => {
try {
setLoading(true)
const result = await createExternalKnowledgeBase({ body: formValue })
if (result && result.id)
if (result && result.id) {
notify({ type: 'success', message: 'External Knowledge Base Connected Successfully' })
else
throw new Error('Failed to create external knowledge base')
router.back()
}
else { throw new Error('Failed to create external knowledge base') }
}
catch (error) {
console.error('Error creating external knowledge base:', error)
notify({ type: 'error', message: 'Failed to connect External Knowledge Base' })
}
setLoading(false)
}

View File

@ -6,18 +6,22 @@ const InfoPanel = () => {
return (
<div className='flex w-[360px] pt-[108px] pb-2 pr-8 flex-col items-start'>
<div className='flex min-w-[240px] p-6 flex-col items-start gap-3 self-stretch rounded-xl bg-background-section'>
<div className='flex p-1 w-10 h-10 justify-center items-center gap-2 flex-grow self-stretch rounded-lg border-0.5 border-components-card-border bg-components-card-bg'>
<div className='flex min-w-[240px] w-full p-6 flex-col items-start gap-3 self-stretch rounded-xl bg-background-section'>
<div className='flex p-1 w-10 h-10 justify-center items-center gap-2 flex-grow self-stretch rounded-lg border-[0.5px] border-components-card-border bg-components-card-bg'>
<RiBookOpenLine className='w-5 h-5 text-text-accent' />
</div>
<p className='flex flex-col items-start gap-2 self-stretch'>
<span className='self-stretch text-text-secondary system-xl-semibold'>
{t('dataset.connectDatasetIntro.title')}
</span>
<span className='self-stretch text-text-tertiary system-sm-regular'>
{t('dataset.connectDatasetIntro.content')}
<span className='text-text-tertiary system-sm-regular'>
{t('dataset.connectDatasetIntro.content.front')}
<a className='text-text-accent system-sm-regular ml-1' href='https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' target='_blank' rel="noopener noreferrer">
{t('dataset.connectDatasetIntro.content.link')}
</a>
{t('dataset.connectDatasetIntro.content.end')}
</span>
<a className='self-stretch text-text-accent system-sm-regular' href='www.google.com' target='_blank' rel="noopener noreferrer">
<a className='self-stretch text-text-accent system-sm-regular' href='https://docs.dify.ai/guides/knowledge-base/connect-external-knowledge' target='_blank' rel="noopener noreferrer">
{t('dataset.connectDatasetIntro.learnMore')}
</a>
</p>

View File

@ -42,14 +42,19 @@ const KnowledgeBaseInfo: React.FC<KnowledgeBaseInfoProps> = ({ name, description
value={description}
onChange={ e => handleDescriptionChange(e)}
placeholder={t('dataset.externalKnowledgeDescriptionPlaceholder') ?? ''}
className='flex h-20 p-2 self-stretch items-start rounded-lg bg-components-input-bg-normal text-components-input-text-placeholder system-sm-regular'
className={`flex h-20 p-2 self-stretch items-start rounded-lg bg-components-input-bg-normal ${description ? 'text-components-input-text-filled' : 'text-components-input-text-placeholder'} system-sm-regular`}
/>
<div className='flex py-0.5 gap-1 self-stretch'>
<a
className='flex py-0.5 gap-1 self-stretch'
href='https://docs.dify.ai/features/datasets#how-to-write-a-good-dataset-description'
target="_blank"
rel="noopener noreferrer"
>
<div className='flex p-0.5 items-center gap-2'>
<RiBookOpenLine className='w-3 h-3 text-text-tertiary' />
</div>
<div className='flex-grow text-text-tertiary body-xs-regular'>{t('dataset.learnHowToWriteGoodKnowledgeDescription')}</div>
</div>
</a>
</div>
</div>
</div>

View File

@ -41,7 +41,7 @@ const ExternalKnowledgeBaseCreate: React.FC<ExternalKnowledgeBaseCreateProps> =
setFormData(newData)
}
const isFormValid = formData.name !== ''
const isFormValid = formData.name.trim() !== ''
&& formData.external_knowledge_api_id !== ''
&& formData.external_knowledge_id !== ''
&& formData.external_retrieval_model.top_k !== undefined
@ -98,7 +98,6 @@ const ExternalKnowledgeBaseCreate: React.FC<ExternalKnowledgeBaseCreateProps> =
variant='primary'
onClick={() => {
onConnect(formData)
navBackHandle()
}}
disabled={!isFormValid}
loading={loading}

View File

@ -26,12 +26,12 @@ const HitDetail: FC<IHitDetailProps> = ({ segInfo }) => {
)
}
return segInfo?.content
return <div className='mb-4 text-md text-gray-800 h-full'>{segInfo?.content}</div>
}
return (
segInfo?.id === 'external'
? <div className='bg-gray-25 p-10'>
? <div className='w-full overflow-x-auto px-2'>
<div className={s.segModalContent}>{renderContent()}</div>
</div>
: <div className='overflow-x-auto'>

View File

@ -216,7 +216,7 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
</div>
</FloatRightContainer>
<Modal
className='w-[520px] px-8 py-6'
className='w-full px-10 py-6'
closable
onClose={() => {
setCurrParagraph({ showModal: false })

View File

@ -1,13 +1,15 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
import React, { useCallback, useState } from 'react'
import { useBoolean } from 'ahooks'
import {
RiDeleteBinLine,
RiEditLine,
} from '@remixicon/react'
import { useTranslation } from 'react-i18next'
import type { DataSet } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
import ActionButton, { ActionButtonState } from '@/app/components/base/action-button'
import FileIcon from '@/app/components/base/file-icon'
import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
import SettingsModal from '@/app/components/app/configuration/dataset-config/settings-modal'
@ -30,8 +32,10 @@ const DatasetItem: FC<Props> = ({
readonly,
}) => {
const media = useBreakpoints()
const { t } = useTranslation()
const isMobile = media === MediaType.mobile
const { formatIndexingTechniqueAndMethod } = useKnowledge()
const [isDeleteHovered, setIsDeleteHovered] = useState(false)
const [isShowSettingsModal, {
setTrue: showSettingsModal,
@ -44,7 +48,12 @@ const DatasetItem: FC<Props> = ({
}, [hideSettingsModal, onChange])
return (
<div className='flex items-center h-10 justify-between rounded-xl px-2 bg-white border border-gray-200 cursor-pointer group/dataset-item'>
<div className={`flex items-center h-10 justify-between rounded-xl px-2 border-[0.5px]
border-components-panel-border-subtle cursor-pointer group/dataset-item
${isDeleteHovered
? 'bg-state-destructive-hover border-state-destructive-border'
: 'bg-components-panel-on-panel-item-bg hover:bg-components-panel-on-panel-item-bg-hover'
}`}>
<div className='w-0 grow flex items-center space-x-1.5'>
{
payload.data_source_type === DataSourceType.NOTION
@ -61,24 +70,33 @@ const DatasetItem: FC<Props> = ({
</div>
{!readonly && (
<div className='hidden group-hover/dataset-item:flex shrink-0 ml-2 items-center space-x-1'>
<div
className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
<ActionButton
onClick={showSettingsModal}
>
<RiEditLine className='w-4 h-4 text-gray-500' />
</div>
<div
className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
<RiEditLine className='w-4 h-4 flex-shrink-0 text-text-tertiary' />
</ActionButton>
<ActionButton
onClick={onRemove}
state={ActionButtonState.Destructive}
onMouseEnter={() => setIsDeleteHovered(true)}
onMouseLeave={() => setIsDeleteHovered(false)}
>
<RiDeleteBinLine className='w-4 h-4 text-gray-500' />
</div>
<RiDeleteBinLine className={`w-4 h-4 flex-shrink-0 ${isDeleteHovered ? 'text-text-destructive' : 'text-text-tertiary'}`} />
</ActionButton>
</div>
)}
<Badge
className='group-hover/dataset-item:hidden shrink-0'
text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
/>
{
payload.indexing_technique && <Badge
className='group-hover/dataset-item:hidden shrink-0'
text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
/>
}
{
payload.provider === 'external' && <Badge
className='group-hover/dataset-item:hidden shrink-0'
text={t('dataset.externalTag')}
/>
}
{isShowSettingsModal && (
<Drawer isOpen={isShowSettingsModal} onClose={hideSettingsModal} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>

View File

@ -21,6 +21,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
let allHighQualityFullTextSearch = true
let allEconomic = true
let mixtureHighQualityAndEconomic = true
let allExternal = true
let allInternal = true
let mixtureInternalAndExternal = true
let inconsistentEmbeddingModel = false
if (!datasets.length) {
allHighQuality = false
@ -29,6 +32,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allEconomic = false
mixtureHighQualityAndEconomic = false
inconsistentEmbeddingModel = false
allExternal = false
allInternal = false
mixtureInternalAndExternal = false
}
datasets.forEach((dataset) => {
if (dataset.indexing_technique === 'economy') {
@ -45,8 +51,21 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText)
allHighQualityFullTextSearch = false
}
if (dataset.provider !== 'external') {
allExternal = false
}
else {
allInternal = false
allHighQuality = false
allHighQualityVectorSearch = false
allHighQualityFullTextSearch = false
mixtureHighQualityAndEconomic = false
}
})
if (allExternal || allInternal)
mixtureInternalAndExternal = false
if (allHighQuality || allEconomic)
mixtureHighQualityAndEconomic = false
@ -59,6 +78,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} as SelectedDatasetsMode
}
@ -70,6 +92,9 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
allHighQualityFullTextSearch,
allEconomic,
mixtureHighQualityAndEconomic,
allInternal,
allExternal,
mixtureInternalAndExternal,
inconsistentEmbeddingModel,
} = getSelectedDatasetsMode(selectedDatasets)
@ -91,13 +116,13 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
reranking_enable: allEconomic ? reranking_enable : true,
}
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || allExternal || mixtureInternalAndExternal)
result.reranking_mode = RerankingModeEnum.RerankingModel
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined)
if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined && allInternal)
result.reranking_mode = RerankingModeEnum.WeightedScore
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) {
if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && allInternal && !weights) {
result.weights = {
vector_setting: {
vector_weight: allHighQualityVectorSearch

View File

@ -1,5 +1,6 @@
const translation = {
knowledge: 'Knowledge',
externalTag: 'External',
externalAPI: 'External API',
externalAPIPanelTitle: 'External Knowledge API',
externalKnowledgeId: 'External Knowledge ID',
@ -10,7 +11,7 @@ const translation = {
externalKnowledgeDescriptionPlaceholder: 'Describe what\'s in this Knowledge Base (optional)',
learnHowToWriteGoodKnowledgeDescription: 'Learn how to write a good knowledge description',
externalAPIPanelDescription: 'The external knowledge API is used to connect to a knowledge base outside of Dify and retrieve knowledge from that knowledge base.',
externalAPIPanelDocumentation: 'Learn how to create an external API',
externalAPIPanelDocumentation: 'Learn how to create an External Knowledge API',
documentCount: ' docs',
wordCount: ' k words',
appCount: ' linked apps',
@ -40,7 +41,11 @@ const translation = {
connectDataset: 'Connect to an External Knowledge Base',
connectDatasetIntro: {
title: 'How to Connect to an External Knowledge Base',
content: 'To connect to an external knowledge base, you need to create an external API first. Please read carefully and refer to learn how to create an external API. Then find the corresponding knowledge ID and fill it in the form on the left. If all the information is correct, it will automatically jump to the retrieval test in the knowledge base after clicking the connect button.',
content: {
front: 'To connect to an external knowledge base, you need to create an external API first. Please read carefully and refer to',
link: 'Learn how to create an external API',
end: '. Then find the corresponding knowledge ID and fill it in the form on the left. If all the information is correct, it will automatically jump to the retrieval test in the knowledge base after clicking the connect button.',
},
learnMore: 'Learn More',
},
createDatasetIntro: 'Import your own text data or write data in real-time via Webhook for LLM context enhancement.',
@ -113,6 +118,8 @@ const translation = {
defaultRetrievalTip: 'Multi-path retrieval is used by default. Knowledge is retrieved from multiple knowledge bases and then re-ranked.',
mixtureHighQualityAndEconomicTip: 'The Rerank model is required for mixture of high quality and economical knowledge bases.',
inconsistentEmbeddingModelTip: 'The Rerank model is required if the Embedding models of the selected knowledge bases are inconsistent.',
mixtureInternalAndExternalTip: 'The Rerank model is required for mixture of internal and external knowledge.',
allExternalTip: 'When using external knowledge only, the user can choose whether to enable the Rerank model. If not enabled, retrieved chunks will be sorted based on scores. When the retrieval strategies of different knowledge bases are inconsistent, it will be inaccurate.',
retrievalSettings: 'Retrieval Setting',
rerankSettings: 'Rerank Setting',
weightedScore: {

View File

@ -1,5 +1,6 @@
const translation = {
knowledge: '知识库',
externalTag: '外部',
externalAPI: '外部 API',
externalAPIPanelTitle: '外部知识库 API',
externalKnowledgeId: '外部知识库 ID',
@ -10,7 +11,7 @@ const translation = {
externalKnowledgeDescriptionPlaceholder: '描述知识库内容(可选)',
learnHowToWriteGoodKnowledgeDescription: '了解如何编写良好的知识库描述',
externalAPIPanelDescription: '外部知识库 API 用于连接到 Dify 之外的知识库并从中检索知识。',
externalAPIPanelDocumentation: '了解如何创建外部 API',
externalAPIPanelDocumentation: '了解如何创建外部知识库 API',
documentCount: ' 文档',
wordCount: ' 千字符',
appCount: ' 关联应用',
@ -39,7 +40,11 @@ const translation = {
},
connectDatasetIntro: {
title: '如何连接到外部知识库',
content: '要连接到外部知识库,您需要先创建一个外部 API。请仔细阅读并参考如何创建外部 API。然后找到相应的知识 ID 并将其填写在左侧表单中。如果所有信息都正确,点击连接按钮后会自动跳转到知识库的检索测试。',
content: {
front: '要连接到外部知识库,您需要先创建一个外部 API。请仔细阅读并参考',
link: '了解如何创建外部 API',
end: '。然后找到相应的知识库 ID 并填写在左侧表单中。如果所有信息正确,点击连接按钮后将自动跳转到知识库中的检索测试。',
},
learnMore: '了解更多',
},
connectDataset: '连接外部知识库',
@ -113,6 +118,8 @@ const translation = {
defaultRetrievalTip: '默认情况下使用多路召回。从多个知识库中检索知识,然后重新排序。',
mixtureHighQualityAndEconomicTip: '混合使用高质量和经济型知识库需要配置 Rerank 模型。',
inconsistentEmbeddingModelTip: '当所选知识库配置的 Embedding 模型不一致时,需要配置 Rerank 模型。',
mixtureInternalAndExternalTip: '混合使用内部和外部知识时需要配置 Rerank 模型。',
allExternalTip: '仅使用外部知识时,用户可以选择是否启用 Rerank 模型。如果不启用,检索到的文本块将根据分数排序。当不同知识库的检索策略不一致时,结果可能不准确。',
retrievalSettings: '召回设置',
rerankSettings: 'Rerank 设置',
weightedScore: {

View File

@ -538,6 +538,9 @@ export type SelectedDatasetsMode = {
allHighQualityFullTextSearch: boolean
allEconomic: boolean
mixtureHighQualityAndEconomic: boolean
allInternal: boolean
allExternal: boolean
mixtureInternalAndExternal: boolean
inconsistentEmbeddingModel: boolean
}