Merge branch 'dev' of https://git.97id.com/ly/zjdataai-app into dev

backend/.env.example

@@ -1,3 +1,8 @@
+JIEBA_DATA=./nltk_data
+NLTK_DATA=./nltk_data
+SQLITE_DATABASE_URL=sqlite:///./source.db
+DATA_SOURCE_CACHE=./restapi
+
 # The Llama Cloud API key.
 # LLAMA_CLOUD_API_KEY=
 SQL_DATABASE_URL=mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1

backend/.env.xinference

@@ -1,3 +1,8 @@
+JIEBA_DATA=./nltk_data
+NLTK_DATA=./nltk_data
+SQLITE_DATABASE_URL=sqlite:///./source.db
+DATA_SOURCE_CACHE=./restapi
+
 # The Llama Cloud API key.
 # LLAMA_CLOUD_API_KEY=
 SQL_DATABASE_URL=mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1

backend/app/engine/retriever/CHTokener.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Union, Callable, NamedTuple
 from bm25s.tokenization import *
 
@@ -8,9 +9,12 @@ except ImportError:
     def tqdm(iterable, *args, **kwargs):
         return iterable
 
+import jieba
+jiebapath = os.environ.get("JIEBA_DATA", "")
+jieba.set_dictionary(os.path.join(jiebapath, 'dict.txt')) #设置字典
+jieba.initialize() #初始化jeiba
 
 def chinese_tokenizer(text: str) -> List[str]:
-    import jieba
     from nltk.corpus import stopwords
     tokens = jieba.lcut(text)
     return [token for token in tokens if token not in stopwords.words('chinese')]

backend/app/settings.py

@@ -3,11 +3,10 @@ from typing import Dict
 
 from llama_index.core.constants import DEFAULT_TEMPERATURE
 from llama_index.core.settings import Settings
+from app.xinference.base import XinferenceEmbedding, XinferenceRerank
 from llama_index.llms.xinference import Xinference
 from llama_index.llms.xinference.base import DEFAULT_XINFERENCE_TEMP
 
-from app.xinference.base import XinferenceEmbedding, XinferenceRerank
-
 
 def get_node_postprocessors():
     rerank_enabled = os.getenv("RERANK_ENABLED").title()
@@ -232,4 +231,4 @@ def init_mistral():
     #
     # Settings.llm = MistralAI(model=os.getenv("MODEL"))
     # Settings.embed_model = MistralAIEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
-    pass
+    pass

backend/main.py

@@ -1,7 +1,5 @@
 
 from dotenv import load_dotenv
-from llama_index.core.node_parser import SentenceSplitter
-
 load_dotenv()
 
 import logging

phoenixserver/phoenixserver.py

@@ -1,9 +1,10 @@
 import os
+from dotenv import load_dotenv
+load_dotenv()
+
 import phoenix as px
 
 
-os.environ['PHOENIX_HOST'] = "0.0.0.0"
-
 session = px.launch_app(use_temp_dir=False)
 
 import msvcrt