From dbe7a7c4fd96330005fc81864a50fe1f84ac8bda Mon Sep 17 00:00:00 2001
From: Gen Sato <52241300+halogen22@users.noreply.github.com>
Date: Thu, 9 Jan 2025 15:37:46 +0900
Subject: [PATCH] Fix: Add a INFO-level log when fallback to gpt2tokenizer
 (#12508)

---
 .../model_providers/__base/tokenizers/gpt2_tokenzier.py       | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
index 7f3c4a61e4..2f6f4fbbef 100644
--- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
+++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py
@@ -1,6 +1,9 @@
+import logging
 from threading import Lock
 from typing import Any
 
+logger = logging.getLogger(__name__)
+
 _tokenizer: Any = None
 _lock = Lock()
 
@@ -43,5 +46,6 @@ class GPT2Tokenizer:
                     base_path = abspath(__file__)
                     gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
                     _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
+                    logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken")
 
             return _tokenizer