Merge branch 'dev' of https://git.97id.com/ly/zjdataai-app into dev-web

backend/app/engine/retriever/CHTokener.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Union, Callable, NamedTuple
 from bm25s.tokenization import *
 
@@ -8,9 +9,12 @@ except ImportError:
     def tqdm(iterable, *args, **kwargs):
         return iterable
 
+import jieba
+jiebapath = os.environ.get("JIEBA_DATA", "")
+jieba.set_dictionary(os.path.join(jiebapath, 'dict.txt')) #设置字典
+jieba.initialize() #初始化jeiba
 
 def chinese_tokenizer(text: str) -> List[str]:
-    import jieba
     from nltk.corpus import stopwords
     tokens = jieba.lcut(text)
     return [token for token in tokens if token not in stopwords.words('chinese')]

backend/app/settings.py

@@ -13,7 +13,7 @@ from util.register import *
 from llama_index.core.callbacks import CallbackManager
 from modelProvide.customDashScope import CustomDashScope
 
-ModelPlateCategory = '模型平台'
+ModelPlateCategory = 'Ä£ÐÍƽ̨'
 
 def get_node_postprocessors():
     rerank_enabled = os.getenv("RERANK_ENABLED").title()