dify/web/models/common.ts
Amir Mohsen 29a4dec387 feat: Integrate WaterCrawl.dev as a new knowledge base provider
Add WaterCrawl.dev as an alternative provider for website crawling in datasets/knowledge base alongside Firecrawl and Jina Reader. This integration enhances the data source options for knowledge bases, allowing users to configure and use WaterCrawl for their website content extraction needs. Resolved #15950
2025-03-21 00:18:18 +01:00

294 lines
6.0 KiB
TypeScript

import type { I18nText } from '@/i18n/language'
export type CommonResponse = {
result: 'success' | 'fail'
}
export type OauthResponse = {
redirect_url: string
}
export type SetupStatusResponse = {
step: 'finished' | 'not_started'
setup_at?: Date
}
export type InitValidateStatusResponse = {
status: 'finished' | 'not_started'
}
export type UserProfileResponse = {
id: string
name: string
email: string
avatar: string
avatar_url: string | null
is_password_set: boolean
interface_language?: string
interface_theme?: string
timezone?: string
last_login_at?: string
last_active_at?: string
last_login_ip?: string
created_at?: string
}
export type UserProfileOriginResponse = {
json: () => Promise<UserProfileResponse>
bodyUsed: boolean
headers: any
}
export type LangGeniusVersionResponse = {
current_version: string
latest_version: string
version: string
release_date: string
release_notes: string
can_auto_update: boolean
current_env: string
}
export type TenantInfoResponse = {
name: string
created_at: string
providers: Array<{
provider: string
provider_name: string
token_is_set: boolean
is_valid: boolean
token_is_valid: boolean
}>
in_trail: boolean
trial_end_reason: null | 'trial_exceeded' | 'using_custom'
}
export type Member = Pick<UserProfileResponse, 'id' | 'name' | 'email' | 'last_login_at' | 'last_active_at' | 'created_at' | 'avatar_url'> & {
avatar: string
status: 'pending' | 'active' | 'banned' | 'closed'
role: 'owner' | 'admin' | 'editor' | 'normal' | 'dataset_operator'
}
export enum ProviderName {
OPENAI = 'openai',
AZURE_OPENAI = 'azure_openai',
ANTHROPIC = 'anthropic',
Replicate = 'replicate',
HuggingfaceHub = 'huggingface_hub',
MiniMax = 'minimax',
Spark = 'spark',
Tongyi = 'tongyi',
ChatGLM = 'chatglm',
}
export type ProviderAzureToken = {
openai_api_base?: string
openai_api_key?: string
}
export type ProviderAnthropicToken = {
anthropic_api_key?: string
}
export type ProviderTokenType = {
[ProviderName.OPENAI]: string
[ProviderName.AZURE_OPENAI]: ProviderAzureToken
[ProviderName.ANTHROPIC]: ProviderAnthropicToken
}
export type Provider = {
[Name in ProviderName]: {
provider_name: Name
} & {
provider_type: 'custom' | 'system'
is_valid: boolean
is_enabled: boolean
last_used: string
token?: string | ProviderAzureToken | ProviderAnthropicToken
}
}[ProviderName]
export type ProviderHosted = Provider & {
quota_type: string
quota_limit: number
quota_used: number
}
export type AccountIntegrate = {
provider: 'google' | 'github'
created_at: number
is_bound: boolean
link: string
}
export type IWorkspace = {
id: string
name: string
plan: string
status: string
created_at: number
current: boolean
}
export type ICurrentWorkspace = Omit<IWorkspace, 'current'> & {
role: 'owner' | 'admin' | 'editor' | 'dataset_operator' | 'normal'
providers: Provider[]
trial_end_reason?: string
custom_config?: {
remove_webapp_brand?: boolean
replace_webapp_logo?: string
}
}
export type DataSourceNotionPage = {
page_icon: null | {
type: string | null
url: string | null
emoji: string | null
}
page_id: string
page_name: string
parent_id: string
type: string
is_bound: boolean
}
export type NotionPage = DataSourceNotionPage & {
workspace_id: string
}
export type DataSourceNotionPageMap = Record<string, DataSourceNotionPage & { workspace_id: string }>
export type DataSourceNotionWorkspace = {
workspace_name: string
workspace_id: string
workspace_icon: string | null
total?: number
pages: DataSourceNotionPage[]
}
export type DataSourceNotionWorkspaceMap = Record<string, DataSourceNotionWorkspace>
export type DataSourceNotion = {
id: string
provider: string
is_bound: boolean
source_info: DataSourceNotionWorkspace
}
export enum DataSourceCategory {
website = 'website',
}
export enum DataSourceProvider {
fireCrawl = 'firecrawl',
jinaReader = 'jinareader',
waterCrawl = 'watercrawl',
}
export type FirecrawlConfig = {
api_key: string
base_url: string
}
export type WatercrawlConfig = {
api_key: string
base_url: string
}
export type DataSourceItem = {
id: string
category: DataSourceCategory
provider: DataSourceProvider
disabled: boolean
created_at: number
updated_at: number
}
export type DataSources = {
sources: DataSourceItem[]
}
export type GithubRepo = {
stargazers_count: number
}
export type PluginProvider = {
tool_name: string
is_enabled: boolean
credentials: {
api_key: string
} | null
}
export type FileUploadConfigResponse = {
batch_count_limit: number
image_file_size_limit?: number | string // default is 10MB
file_size_limit: number // default is 15MB
audio_file_size_limit?: number // default is 50MB
video_file_size_limit?: number // default is 100MB
workflow_file_upload_limit?: number // default is 10
}
export type InvitationResult = {
status: 'success'
email: string
url: string
} | {
status: 'failed'
email: string
message: string
}
export type InvitationResponse = CommonResponse & {
invitation_results: InvitationResult[]
}
export type ApiBasedExtension = {
id?: string
name?: string
api_endpoint?: string
api_key?: string
}
export type CodeBasedExtensionForm = {
type: string
label: I18nText
variable: string
required: boolean
options: { label: I18nText; value: string }[]
default: string
placeholder: string
max_length?: number
}
export type CodeBasedExtensionItem = {
name: string
label: any
form_schema: CodeBasedExtensionForm[]
}
export type CodeBasedExtension = {
module: string
data: CodeBasedExtensionItem[]
}
export type ExternalDataTool = {
type?: string
label?: string
icon?: string
icon_background?: string
variable?: string
enabled?: boolean
config?: {
api_based_extension_id?: string
} & Partial<Record<string, any>>
}
export type ModerateResponse = {
flagged: boolean
text: string
}
export type ModerationService = (
url: string,
body: {
app_id: string
text: string
}
) => Promise<ModerateResponse>