dify/web/app/components/header/account-setting/data-source-page/data-source-website/config-watercrawl-modal.tsx
Amir Mohsen 29a4dec387 feat: Integrate WaterCrawl.dev as a new knowledge base provider
Add WaterCrawl.dev as an alternative provider for website crawling in datasets/knowledge base alongside Firecrawl and Jina Reader. This integration enhances the data source options for knowledge bases, allowing users to configure and use WaterCrawl for their website content extraction needs. Resolved #15950
2025-03-21 00:18:18 +01:00

162 lines
5.4 KiB
TypeScript

'use client'
import type { FC } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import {
PortalToFollowElem,
PortalToFollowElemContent,
} from '@/app/components/base/portal-to-follow-elem'
import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
import Button from '@/app/components/base/button'
import type { WatercrawlConfig } from '@/models/common'
import Field from '@/app/components/datasets/create/website/base/field'
import Toast from '@/app/components/base/toast'
import { createDataSourceApiKeyBinding } from '@/service/datasets'
import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
type Props = {
onCancel: () => void
onSaved: () => void
}
const I18N_PREFIX = 'datasetCreation.watercrawl'
const DEFAULT_BASE_URL = 'https://app.watercrawl.dev'
const ConfigWatercrawlModal: FC<Props> = ({
onCancel,
onSaved,
}) => {
const { t } = useTranslation()
const [isSaving, setIsSaving] = useState(false)
const [config, setConfig] = useState<WatercrawlConfig>({
api_key: '',
base_url: '',
})
const handleConfigChange = useCallback((key: string) => {
return (value: string | number) => {
setConfig(prev => ({ ...prev, [key]: value as string }))
}
}, [])
const handleSave = useCallback(async () => {
if (isSaving)
return
let errorMsg = ''
if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
errorMsg = t('common.errorMsg.urlError')
if (!errorMsg) {
if (!config.api_key) {
errorMsg = t('common.errorMsg.fieldRequired', {
field: 'API Key',
})
}
}
if (errorMsg) {
Toast.notify({
type: 'error',
message: errorMsg,
})
return
}
const postData = {
category: 'website',
provider: 'watercrawl',
credentials: {
auth_type: 'x-api-key',
config: {
api_key: config.api_key,
base_url: config.base_url || DEFAULT_BASE_URL,
},
},
}
try {
setIsSaving(true)
await createDataSourceApiKeyBinding(postData)
Toast.notify({
type: 'success',
message: t('common.api.success'),
})
}
finally {
setIsSaving(false)
}
onSaved()
}, [config.api_key, config.base_url, onSaved, t, isSaving])
return (
<PortalToFollowElem open>
<PortalToFollowElemContent className='w-full h-full z-[60]'>
<div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
<div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-components-panel-bg shadow-xl rounded-2xl overflow-y-auto'>
<div className='px-8 pt-8'>
<div className='flex justify-between items-center mb-4'>
<div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configWatercrawl`)}</div>
</div>
<div className='space-y-4'>
<Field
label='API Key'
labelClassName='!text-sm'
isRequired
value={config.api_key}
onChange={handleConfigChange('api_key')}
placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
/>
<Field
label='Base URL'
labelClassName='!text-sm'
value={config.base_url}
onChange={handleConfigChange('base_url')}
placeholder={DEFAULT_BASE_URL}
/>
</div>
<div className='my-8 flex justify-between items-center h-8'>
<a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-text-accent' target='_blank' href='https://app.watercrawl.dev/'>
<span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
<LinkExternal02 className='w-3 h-3' />
</a>
<div className='flex'>
<Button
size='large'
className='mr-2'
onClick={onCancel}
>
{t('common.operation.cancel')}
</Button>
<Button
variant='primary'
size='large'
onClick={handleSave}
loading={isSaving}
>
{t('common.operation.save')}
</Button>
</div>
</div>
</div>
<div className='border-t-[0.5px] border-t-divider-regular'>
<div className='flex justify-center items-center py-3 bg-background-section-burn text-xs text-text-tertiary'>
<Lock01 className='mr-1 w-3 h-3 text-text-tertiary' />
{t('common.modelProvider.encrypted.front')}
<a
className='text-text-accent mx-1'
target='_blank' rel='noopener noreferrer'
href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
>
PKCS1_OAEP
</a>
{t('common.modelProvider.encrypted.back')}
</div>
</div>
</div>
</div>
</PortalToFollowElemContent>
</PortalToFollowElem>
)
}
export default React.memo(ConfigWatercrawlModal)