outman/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove unused codes

Browse Source
This commit is contained in:
takatost 2024-09-29 18:24:20 +08:00
parent d4e007f9db
commit bcdb407be8
6 changed files with 119 additions and 101210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50001
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/merges.txt
View File

File diff suppressed because it is too large Load Diff

23
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/special_tokens_map.json
View File

@ -1,23 +0,0 @@
{
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

33
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/tokenizer_config.json
View File

@ -1,33 +0,0 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"bos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": true,
"eos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"model_max_length": 1024,
"pad_token": null,
"tokenizer_class": "GPT2Tokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

50259
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/vocab.json
View File

File diff suppressed because it is too large Load Diff

1005
api/poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

8
api/pyproject.toml
View File

@ -139,8 +139,6 @@ azure-identity = "1.16.1"
azure-storage-blob = "12.13.0" azure-storage-blob = "12.13.0"
beautifulsoup4 = "4.12.2" beautifulsoup4 = "4.12.2"
boto3 = "1.35.17" boto3 = "1.35.17"
sagemaker = "2.231.0"
bs4 = "~0.0.1"
cachetools = "~5.3.0" cachetools = "~5.3.0"
celery = "~5.3.6" celery = "~5.3.6"
chardet = "~5.1.0" chardet = "~5.1.0"
@ -155,21 +153,17 @@ flask-restful = "~0.3.10"
Flask-SQLAlchemy = "~3.1.1" Flask-SQLAlchemy = "~3.1.1"
gevent = "~23.9.1" gevent = "~23.9.1"
gmpy2 = "~2.2.1" gmpy2 = "~2.2.1"
google-ai-generativelanguage = "0.6.9"
google-api-core = "2.18.0" google-api-core = "2.18.0"
google-api-python-client = "2.90.0" google-api-python-client = "2.90.0"
google-auth = "2.29.0" google-auth = "2.29.0"
google-auth-httplib2 = "0.2.0" google-auth-httplib2 = "0.2.0"
google-cloud-aiplatform = "1.49.0"
google-cloud-storage = "2.16.0" google-cloud-storage = "2.16.0"
google-generativeai = "0.8.1"
googleapis-common-protos = "1.63.0" googleapis-common-protos = "1.63.0"
gunicorn = "~22.0.0" gunicorn = "~22.0.0"
httpx = { version = "~0.27.0", extras = ["socks"] } httpx = { version = "~0.27.0", extras = ["socks"] }
jieba = "0.42.1" jieba = "0.42.1"
langfuse = "^2.48.0" langfuse = "^2.48.0"
langsmith = "^0.1.77" langsmith = "^0.1.77"
mailchimp-transactional = "~1.0.50"
markdown = "~3.5.1" markdown = "~3.5.1"
nltk = "3.8.1" nltk = "3.8.1"
numpy = "~1.26.4" numpy = "~1.26.4"
@ -193,9 +187,7 @@ resend = "~0.7.0"
scikit-learn = "^1.5.1" scikit-learn = "^1.5.1"
sentry-sdk = { version = "~1.44.1", extras = ["flask"] } sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
sqlalchemy = "~2.0.29" sqlalchemy = "~2.0.29"
tencentcloud-sdk-python-hunyuan = "~3.0.1158"
tiktoken = "~0.7.0" tiktoken = "~0.7.0"
tokenizers = "~0.15.0"
unstructured = { version = "~0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] } unstructured = { version = "~0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
werkzeug = "~3.0.1" werkzeug = "~3.0.1"
yarl = "~1.9.4" yarl = "~1.9.4"