
Add WaterCrawl.dev as an alternative provider for website crawling in datasets/knowledge base alongside Firecrawl and Jina Reader. This integration enhances the data source options for knowledge bases, allowing users to configure and use WaterCrawl for their website content extraction needs. Resolved #15950
162 lines
5.4 KiB
TypeScript
162 lines
5.4 KiB
TypeScript
'use client'
|
|
import type { FC } from 'react'
|
|
import React, { useCallback, useState } from 'react'
|
|
import { useTranslation } from 'react-i18next'
|
|
import {
|
|
PortalToFollowElem,
|
|
PortalToFollowElemContent,
|
|
} from '@/app/components/base/portal-to-follow-elem'
|
|
import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
|
|
import Button from '@/app/components/base/button'
|
|
import type { WatercrawlConfig } from '@/models/common'
|
|
import Field from '@/app/components/datasets/create/website/base/field'
|
|
import Toast from '@/app/components/base/toast'
|
|
import { createDataSourceApiKeyBinding } from '@/service/datasets'
|
|
import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
|
|
type Props = {
|
|
onCancel: () => void
|
|
onSaved: () => void
|
|
}
|
|
|
|
const I18N_PREFIX = 'datasetCreation.watercrawl'
|
|
|
|
const DEFAULT_BASE_URL = 'https://app.watercrawl.dev'
|
|
|
|
const ConfigWatercrawlModal: FC<Props> = ({
|
|
onCancel,
|
|
onSaved,
|
|
}) => {
|
|
const { t } = useTranslation()
|
|
const [isSaving, setIsSaving] = useState(false)
|
|
const [config, setConfig] = useState<WatercrawlConfig>({
|
|
api_key: '',
|
|
base_url: '',
|
|
})
|
|
|
|
const handleConfigChange = useCallback((key: string) => {
|
|
return (value: string | number) => {
|
|
setConfig(prev => ({ ...prev, [key]: value as string }))
|
|
}
|
|
}, [])
|
|
|
|
const handleSave = useCallback(async () => {
|
|
if (isSaving)
|
|
return
|
|
let errorMsg = ''
|
|
if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
|
|
errorMsg = t('common.errorMsg.urlError')
|
|
if (!errorMsg) {
|
|
if (!config.api_key) {
|
|
errorMsg = t('common.errorMsg.fieldRequired', {
|
|
field: 'API Key',
|
|
})
|
|
}
|
|
}
|
|
|
|
if (errorMsg) {
|
|
Toast.notify({
|
|
type: 'error',
|
|
message: errorMsg,
|
|
})
|
|
return
|
|
}
|
|
const postData = {
|
|
category: 'website',
|
|
provider: 'watercrawl',
|
|
credentials: {
|
|
auth_type: 'x-api-key',
|
|
config: {
|
|
api_key: config.api_key,
|
|
base_url: config.base_url || DEFAULT_BASE_URL,
|
|
},
|
|
},
|
|
}
|
|
try {
|
|
setIsSaving(true)
|
|
await createDataSourceApiKeyBinding(postData)
|
|
Toast.notify({
|
|
type: 'success',
|
|
message: t('common.api.success'),
|
|
})
|
|
}
|
|
finally {
|
|
setIsSaving(false)
|
|
}
|
|
|
|
onSaved()
|
|
}, [config.api_key, config.base_url, onSaved, t, isSaving])
|
|
|
|
return (
|
|
<PortalToFollowElem open>
|
|
<PortalToFollowElemContent className='w-full h-full z-[60]'>
|
|
<div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
|
|
<div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-components-panel-bg shadow-xl rounded-2xl overflow-y-auto'>
|
|
<div className='px-8 pt-8'>
|
|
<div className='flex justify-between items-center mb-4'>
|
|
<div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configWatercrawl`)}</div>
|
|
</div>
|
|
|
|
<div className='space-y-4'>
|
|
<Field
|
|
label='API Key'
|
|
labelClassName='!text-sm'
|
|
isRequired
|
|
value={config.api_key}
|
|
onChange={handleConfigChange('api_key')}
|
|
placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
|
|
/>
|
|
<Field
|
|
label='Base URL'
|
|
labelClassName='!text-sm'
|
|
value={config.base_url}
|
|
onChange={handleConfigChange('base_url')}
|
|
placeholder={DEFAULT_BASE_URL}
|
|
/>
|
|
</div>
|
|
<div className='my-8 flex justify-between items-center h-8'>
|
|
<a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-text-accent' target='_blank' href='https://app.watercrawl.dev/'>
|
|
<span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
|
|
<LinkExternal02 className='w-3 h-3' />
|
|
</a>
|
|
<div className='flex'>
|
|
<Button
|
|
size='large'
|
|
className='mr-2'
|
|
onClick={onCancel}
|
|
>
|
|
{t('common.operation.cancel')}
|
|
</Button>
|
|
<Button
|
|
variant='primary'
|
|
size='large'
|
|
onClick={handleSave}
|
|
loading={isSaving}
|
|
>
|
|
{t('common.operation.save')}
|
|
</Button>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
<div className='border-t-[0.5px] border-t-divider-regular'>
|
|
<div className='flex justify-center items-center py-3 bg-background-section-burn text-xs text-text-tertiary'>
|
|
<Lock01 className='mr-1 w-3 h-3 text-text-tertiary' />
|
|
{t('common.modelProvider.encrypted.front')}
|
|
<a
|
|
className='text-text-accent mx-1'
|
|
target='_blank' rel='noopener noreferrer'
|
|
href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
|
|
>
|
|
PKCS1_OAEP
|
|
</a>
|
|
{t('common.modelProvider.encrypted.back')}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</PortalToFollowElemContent>
|
|
</PortalToFollowElem>
|
|
)
|
|
}
|
|
export default React.memo(ConfigWatercrawlModal)
|