优化模型初始化代码
This commit is contained in:
wanyaokun
+37
-23
@@ -4,34 +4,48 @@ SQL_DATABASE_URL=mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zj
|
||||
#SQL_DATABASE_URL=mysql+pymysql://zjinfo2:GSKcziSdBixDXwcd@110.42.234.166:3306/zjinfo2
|
||||
SQLITE_DATABASE_URL=sqlite:///./source.db
|
||||
|
||||
DASHSCOPE_API_KEY=sk-02c8540e86d84b7ca0e6f4f51bac6e60
|
||||
# The provider for the AI models to use.
|
||||
MODEL_PROVIDER=dashscope
|
||||
# The name of LLM model to use.
|
||||
MODEL=qwen-max
|
||||
# The number of similar embeddings to return when retrieving documents.
|
||||
TOP_K=10
|
||||
#--------------------------
|
||||
# 是否启用混合检索
|
||||
HYBRID_ENABLED = false
|
||||
# 混合检索阈值
|
||||
HYBRID_ALPHA = 0.6
|
||||
# 是否启用检索重排功能
|
||||
ENABLE_RERANK=true
|
||||
# Name of the embedding model to use.
|
||||
EMBEDDING_MODEL=text-embedding-v2
|
||||
RERANK_ENABLED=true
|
||||
|
||||
# Dimension of the embedding model to use.
|
||||
#---------- rerank- Xinference ----------------
|
||||
RERANK_PROVIDER=xinference
|
||||
RERANK_MODEL=bge-reranker-v2-m3
|
||||
RERANK_BASE_URL=http://10.1.16.39:9995
|
||||
RERANK_TOP_N=5
|
||||
RERANK_THRESHOLD=0.3
|
||||
|
||||
#---------- model - Xinference ----------------
|
||||
#MODEL_PROVIDER=xinference
|
||||
#OPENAI_API_KEY=xinference
|
||||
#BASE_URL=http://172.20.0.145:9995
|
||||
#MODEL=Qwen2-72B-Instruct-GPTQ-Int8
|
||||
## Temperature for sampling from the model.
|
||||
#LLM_TEMPERATURE=0.1
|
||||
|
||||
#---------- model - dashscope ----------------
|
||||
MODEL_PROVIDER=dashscope
|
||||
DASHSCOPE_API_KEY=sk-221d2d202e104618a56002ce2e7dc0d0
|
||||
MODEL=qwen-max
|
||||
|
||||
|
||||
|
||||
#---------- embedding - Xinference ----------------
|
||||
EMBEDDING_PROVIDER=xinference
|
||||
EMBEDDING_MODEL=bge-m3
|
||||
EMBEDDING_BASE_URL=http://10.1.16.39:9995
|
||||
EMBEDDING_DIM=1024
|
||||
|
||||
|
||||
# The questions to help users get started (multi-line).
|
||||
CONVERSATION_STARTERS=本工程指什么？\n总算表有哪些费用？\n项目划分哪些内容构成？\n其他费用表有哪些内容？
|
||||
|
||||
# The OpenAI API key to use.
|
||||
# OPENAI_API_KEY=
|
||||
|
||||
# Temperature for sampling from the model.
|
||||
# LLM_TEMPERATURE=
|
||||
|
||||
# Maximum number of tokens to generate.
|
||||
# LLM_MAX_TOKENS=
|
||||
|
||||
# The number of similar embeddings to return when retrieving documents.
|
||||
TOP_K=5
|
||||
|
||||
# The time in milliseconds to wait for the stream to return a response.
|
||||
STREAM_TIMEOUT=60000
|
||||
|
||||
@@ -53,9 +67,8 @@ VECTOR_STORE_PATH=./storage_vector
|
||||
BM_RETRIEVER_PATH =./storage_bm
|
||||
|
||||
|
||||
|
||||
PHOENIX_API_KEY=123456
|
||||
PHOENIX_URL=http://localhost:6006/v1/traces
|
||||
PHOENIX_URL=http://10.1.6.103:6006/v1/traces
|
||||
PHOENIX_PROJECT_NAME=ly_zjapp
|
||||
#OTEL_SERVICE_NAME=ly_zjapp
|
||||
#OTEL_RESOURCE_ATTRIBUTES=openinference.project.name=ly_zjapp
|
||||
@@ -82,4 +95,5 @@ SYSTEM_PROMPT="You are a weather forecast agent. You help users to get the weath
|
||||
|
||||
PRJTOJSON_URL = 'http://10.1.6.60:8092'
|
||||
PROJECT_TITLE = "您好，我是博微工程理解小助手，您可以问我有关[线路工程]工程数据的相关问题！"
|
||||
|
||||
CHAT_UPLOAD_FILECACHE = "./output/uploaded"
|
||||
+14
-13
@@ -14,27 +14,28 @@ HYBRID_ALPHA = 0.6
|
||||
#--------------------------
|
||||
# 是否启用检索重排功能
|
||||
RERANK_ENABLED=true
|
||||
# Rerank model
|
||||
|
||||
#---------- rerank- Xinference ----------------
|
||||
RERANK_PROVIDER=xinference
|
||||
RERANK_MODEL=bge-reranker-v2-m3
|
||||
RERANK_BASE_URL=http://10.1.16.39:9995
|
||||
RERANK_TOP_N=5
|
||||
RERANK_THRESHOLD=0.3
|
||||
#---------- Xinference ----------------
|
||||
# The provider for the AI models to use.
|
||||
MODEL_PROVIDER=xinference
|
||||
# The OpenAI API key to use.
|
||||
OPENAI_API_KEY=xinference
|
||||
|
||||
#---------- model - Xinference ----------------
|
||||
MODEL_PROVIDER=xinference # The provider for the AI models to use.
|
||||
OPENAI_API_KEY=xinference # The OpenAI API key to use.
|
||||
BASE_URL=http://10.1.0.142:9995
|
||||
MODEL=Qwen2-72B-Instruct-GPTQ-Int8
|
||||
# Temperature for sampling from the model.
|
||||
LLM_TEMPERATURE=0.1
|
||||
# Maximum number of tokens to generate.
|
||||
#LLM_MAX_TOKENS=
|
||||
# Name of the embedding model to use.
|
||||
LLM_TEMPERATURE=0.1 # Temperature for sampling from the model.
|
||||
#LLM_MAX_TOKENS= # Maximum number of tokens to generate.
|
||||
|
||||
|
||||
#---------- embedding - Xinference ----------------
|
||||
EMBEDDING_PROVIDER=xinference
|
||||
EMBEDDING_MODEL=bge-m3
|
||||
EMBEDDING_BASE_URL=http://10.1.16.39:9995
|
||||
# Dimension of the embedding model to use.
|
||||
EMBEDDING_DIM=1024
|
||||
EMBEDDING_DIM=1024 # Dimension of the embedding model to use.
|
||||
|
||||
##---------- OpenAI ----------------
|
||||
## The provider for the AI models to use.
|
||||
|
||||
@@ -24,14 +24,11 @@ from app.api.routers.services.fileServices import PrjFileLoadService,ChatFileSer
|
||||
from app.api.routers.services.suggestion import NextQuestionSuggestion
|
||||
import time
|
||||
from llama_index.core.settings import Settings
|
||||
from llama_index.core.callbacks import CallbackManager
|
||||
|
||||
logger = logging.getLogger("uvicorn")
|
||||
|
||||
v1_router = v = APIRouter()
|
||||
|
||||
Settings.llm.callback_manager = CallbackManager()
|
||||
|
||||
gEvent_handler = None
|
||||
|
||||
|
||||
|
||||
+286
-190
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from typing import Dict
|
||||
|
||||
from abc import abstractmethod
|
||||
from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
from llama_index.core.settings import Settings
|
||||
from llama_index.llms.xinference import Xinference
|
||||
@@ -9,229 +9,322 @@ from llama_index.llms.xinference.base import DEFAULT_XINFERENCE_TEMP
|
||||
from app.xinference.base import XinferenceEmbedding, XinferenceRerank
|
||||
from app.engine.loaders import getProjectInfos
|
||||
from app.api.routers.request.base import ProjectInfo
|
||||
from util.register import *
|
||||
from llama_index.core.callbacks import CallbackManager
|
||||
from modelProvide.customDashScope import CustomDashScope
|
||||
|
||||
ModelPlateCategory = '模型平台'
|
||||
|
||||
def get_node_postprocessors():
|
||||
rerank_enabled = os.getenv("RERANK_ENABLED").title()
|
||||
if rerank_enabled is None or rerank_enabled == 'False':
|
||||
return []
|
||||
|
||||
rerank_model = os.getenv("RERANK_MODEL")
|
||||
rerank_url = os.getenv("RERANK_BASE_URL")
|
||||
rerank_top_n = os.getenv("RERANK_TOP_N")
|
||||
rerank_threshold = os.getenv("RERANK_THRESHOLD")
|
||||
|
||||
Rerank_provider = os.getenv("RERANK_PROVIDER")
|
||||
modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,Rerank_provider)
|
||||
postprocess = None
|
||||
if rerank_model is not None:
|
||||
postprocess = [XinferenceRerank(rerank_model, rerank_url, top_n=rerank_top_n, threshold=rerank_threshold)]
|
||||
if modelPaltCls is not None:
|
||||
modelPalt:ModelPlatform = modelPaltCls()
|
||||
postprocess = modelPalt.rerank()
|
||||
else:
|
||||
raise ValueError(f"Invalid rerank provider: {Rerank_provider}")
|
||||
return postprocess
|
||||
|
||||
def init_settings():
|
||||
model_provider = os.getenv("MODEL_PROVIDER")
|
||||
match model_provider:
|
||||
case "openai":
|
||||
init_openai()
|
||||
case "dashscope":
|
||||
init_dashscope()
|
||||
case "groq":
|
||||
init_groq()
|
||||
case "ollama":
|
||||
init_ollama()
|
||||
case "anthropic":
|
||||
init_anthropic()
|
||||
case "gemini":
|
||||
init_gemini()
|
||||
case "mistral":
|
||||
init_mistral()
|
||||
case "azure-openai":
|
||||
init_azure_openai()
|
||||
case "t-systems":
|
||||
from .llmhub import init_llmhub
|
||||
init_llmhub()
|
||||
case "xinference":
|
||||
init_xinference()
|
||||
case _:
|
||||
raise ValueError(f"Invalid model provider: {model_provider}")
|
||||
modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,model_provider)
|
||||
if modelPaltCls is not None:
|
||||
modelPalt:ModelPlatform = modelPaltCls()
|
||||
Settings.llm = modelPalt.model()
|
||||
else:
|
||||
raise ValueError(f"Invalid model provider: {model_provider}")
|
||||
|
||||
embedding_provider = os.getenv("EMBEDDING_PROVIDER")
|
||||
modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,embedding_provider)
|
||||
if modelPalt is not None:
|
||||
modelPalt:ModelPlatform = modelPaltCls()
|
||||
Settings.embed_model = modelPalt.embedding()
|
||||
else:
|
||||
raise ValueError(f"Invalid embedding provider: {embedding_provider}")
|
||||
|
||||
Settings.llm.callback_manager = CallbackManager()
|
||||
Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
|
||||
Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))
|
||||
|
||||
def init_ollama():
|
||||
# from llama_index.embeddings.ollama import OllamaEmbedding
|
||||
# from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
|
||||
#
|
||||
# base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
|
||||
# request_timeout = float(
|
||||
# os.getenv("OLLAMA_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
|
||||
# )
|
||||
# Settings.embed_model = OllamaEmbedding(
|
||||
# base_url=base_url,
|
||||
# model_name=os.getenv("EMBEDDING_MODEL"),
|
||||
# )
|
||||
# Settings.llm = Ollama(
|
||||
# base_url=base_url, model=os.getenv("MODEL"), request_timeout=request_timeout
|
||||
# )
|
||||
pass
|
||||
class ModelPlatform:
|
||||
@abstractmethod
|
||||
def model(self):
|
||||
pass
|
||||
|
||||
def init_xinference():
|
||||
base_url = os.getenv("BASE_URL")
|
||||
model = os.getenv("MODEL")
|
||||
max_tokens = int(os.getenv("LLM_MAX_TOKENS")) if os.getenv("LLM_MAX_TOKENS") is not None else None
|
||||
temperature = float(os.getenv("LLM_TEMPERATURE", DEFAULT_XINFERENCE_TEMP))
|
||||
@abstractmethod
|
||||
def embedding(self):
|
||||
pass
|
||||
|
||||
Settings.llm = Xinference(model, base_url, temperature, max_tokens)
|
||||
@abstractmethod
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
embedding_base_url = os.getenv("EMBEDDING_BASE_URL")
|
||||
embedding_base_url = embedding_base_url if embedding_base_url != None and embedding_base_url != "" else base_url
|
||||
@register(ModelPlateCategory,'ollama')
|
||||
class OllamaPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
#from llama_index.embeddings.ollama import OllamaEmbedding
|
||||
#from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
|
||||
#
|
||||
# base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
|
||||
# request_timeout = float(
|
||||
# os.getenv("OLLAMA_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
|
||||
# )
|
||||
# Settings.llm = Ollama(
|
||||
# base_url=base_url, model=os.getenv("MODEL"), request_timeout=request_timeout
|
||||
# )
|
||||
pass
|
||||
|
||||
embed_model_name = os.getenv("EMBEDDING_MODEL")
|
||||
dimensions = os.getenv("EMBEDDING_DIM")
|
||||
dimensions = int(dimensions) if dimensions is not None else None
|
||||
Settings.embed_model = XinferenceEmbedding(embed_model_name, embedding_base_url, dimensions=dimensions)
|
||||
def embedding(self):
|
||||
#from llama_index.embeddings.ollama import OllamaEmbedding
|
||||
# base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
|
||||
# Settings.embed_model = OllamaEmbedding(
|
||||
# base_url=base_url,
|
||||
# model_name=os.getenv("EMBEDDING_MODEL"),
|
||||
# )
|
||||
pass
|
||||
|
||||
def init_openai():
|
||||
from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
from llama_index.embeddings.openai import OpenAIEmbedding
|
||||
from llama_index.llms.openai import OpenAI
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
config = {
|
||||
"model": os.getenv("MODEL"),
|
||||
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
|
||||
"max_tokens": int(max_tokens) if max_tokens is not None else None,
|
||||
}
|
||||
Settings.llm = OpenAI(**config)
|
||||
@register(ModelPlateCategory,'xinference')
|
||||
class XinferencePlatform(ModelPlatform):
|
||||
def model(self):
|
||||
base_url = os.getenv("BASE_URL")
|
||||
model = os.getenv("MODEL")
|
||||
max_tokens = int(os.getenv("LLM_MAX_TOKENS")) if os.getenv("LLM_MAX_TOKENS") is not None else None
|
||||
temperature = float(os.getenv("LLM_TEMPERATURE", DEFAULT_XINFERENCE_TEMP))
|
||||
return Xinference(model, base_url, temperature, max_tokens)
|
||||
|
||||
def embedding(self):
|
||||
base_url = os.getenv("BASE_URL")
|
||||
embedding_base_url = os.getenv("EMBEDDING_BASE_URL")
|
||||
embedding_base_url = embedding_base_url if embedding_base_url != None and embedding_base_url != "" else base_url
|
||||
|
||||
dimensions = os.getenv("EMBEDDING_DIM")
|
||||
config = {
|
||||
"model": os.getenv("EMBEDDING_MODEL"),
|
||||
"dimensions": int(dimensions) if dimensions is not None else None,
|
||||
}
|
||||
Settings.embed_model = OpenAIEmbedding(**config)
|
||||
embed_model_name = os.getenv("EMBEDDING_MODEL")
|
||||
dimensions = os.getenv("EMBEDDING_DIM")
|
||||
dimensions = int(dimensions) if dimensions is not None else None
|
||||
return XinferenceEmbedding(embed_model_name, embedding_base_url, dimensions=dimensions)
|
||||
|
||||
def rerank(self):
|
||||
rerank_model = os.getenv("RERANK_MODEL")
|
||||
rerank_url = os.getenv("RERANK_BASE_URL")
|
||||
rerank_top_n = os.getenv("RERANK_TOP_N")
|
||||
rerank_threshold = os.getenv("RERANK_THRESHOLD")
|
||||
postprocess = None
|
||||
if rerank_model is not None:
|
||||
postprocess = [XinferenceRerank(rerank_model, rerank_url, top_n=rerank_top_n, threshold=rerank_threshold)]
|
||||
return postprocess
|
||||
|
||||
def init_dashscope():
|
||||
from llama_index.llms.dashscope import DashScope,DashScopeGenerationModels
|
||||
from llama_index.embeddings.dashscope import DashScopeEmbedding,DashScopeBatchTextEmbeddingModels,DashScopeTextEmbeddingType,DashScopeTextEmbeddingModels
|
||||
@register(ModelPlateCategory,'openai')
|
||||
class OpenAIPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
from llama_index.llms.openai import OpenAI
|
||||
|
||||
max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
config = {
|
||||
"model": os.getenv("MODEL"),
|
||||
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
|
||||
"max_tokens": int(max_tokens) if max_tokens is not None else None,
|
||||
}
|
||||
Settings.llm = llm = DashScope(model_name=DashScopeGenerationModels.QWEN_MAX)
|
||||
max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
config = {
|
||||
"model": os.getenv("MODEL"),
|
||||
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
|
||||
"max_tokens": int(max_tokens) if max_tokens is not None else None,
|
||||
}
|
||||
return OpenAI(**config)
|
||||
|
||||
def embedding(self):
|
||||
from llama_index.embeddings.openai import OpenAIEmbedding
|
||||
dimensions = os.getenv("EMBEDDING_DIM")
|
||||
config = {
|
||||
"model": os.getenv("EMBEDDING_MODEL"),
|
||||
"dimensions": int(dimensions) if dimensions is not None else None,
|
||||
}
|
||||
return OpenAIEmbedding(**config)
|
||||
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
dimensions = os.getenv("EMBEDDING_DIM")
|
||||
config = {
|
||||
"model": os.getenv("EMBEDDING_MODEL"),
|
||||
"dimensions": int(dimensions) if dimensions is not None else None,
|
||||
}
|
||||
Settings.embed_model = DashScopeEmbedding(model_name=DashScopeTextEmbeddingModels.TEXT_EMBEDDING_V2,
|
||||
text_type=DashScopeTextEmbeddingType.TEXT_TYPE_QUERY)
|
||||
@register(ModelPlateCategory,'dashscope')
|
||||
class DashscopePlatform(ModelPlatform):
|
||||
def model(self):
|
||||
apikey = os.getenv('DASHSCOPE_API_KEY')
|
||||
modelName = os.getenv('MODEL')
|
||||
return CustomDashScope(model_name=modelName,api_key = apikey)
|
||||
|
||||
def init_azure_openai():
|
||||
# from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
# from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
|
||||
# from llama_index.llms.azure_openai import AzureOpenAI
|
||||
#
|
||||
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
|
||||
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
|
||||
# max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
|
||||
# dimensions = os.getenv("EMBEDDING_DIM")
|
||||
#
|
||||
# azure_config = {
|
||||
# "api_key": os.environ["AZURE_OPENAI_KEY"],
|
||||
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
|
||||
# or os.getenv("OPENAI_API_VERSION"),
|
||||
# }
|
||||
#
|
||||
# Settings.llm = AzureOpenAI(
|
||||
# model=os.getenv("MODEL"),
|
||||
# max_tokens=int(max_tokens) if max_tokens is not None else None,
|
||||
# temperature=float(temperature),
|
||||
# deployment_name=llm_deployment,
|
||||
# **azure_config,
|
||||
# )
|
||||
#
|
||||
# Settings.embed_model = AzureOpenAIEmbedding(
|
||||
# model=os.getenv("EMBEDDING_MODEL"),
|
||||
# dimensions=int(dimensions) if dimensions is not None else None,
|
||||
# deployment_name=embedding_deployment,
|
||||
# **azure_config,
|
||||
# )
|
||||
pass
|
||||
def embedding(self):
|
||||
from llama_index.embeddings.dashscope import DashScopeEmbedding,DashScopeTextEmbeddingType,DashScopeTextEmbeddingModels
|
||||
api_key = os.getenv('DASHSCOPE_API_KEY')
|
||||
modelName = os.getenv('EMBEDDING_MODEL')
|
||||
return DashScopeEmbedding(model_name=modelName,
|
||||
text_type=DashScopeTextEmbeddingType.TEXT_TYPE_QUERY,api_key = api_key)
|
||||
|
||||
def init_fastembed():
|
||||
"""
|
||||
Use Qdrant Fastembed as the local embedding provider.
|
||||
"""
|
||||
# from llama_index.embeddings.fastembed import FastEmbedEmbedding
|
||||
#
|
||||
# embed_model_map: Dict[str, str] = {
|
||||
# # Small and multilingual
|
||||
# "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
# # Large and multilingual
|
||||
# "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", # noqa: E501
|
||||
# }
|
||||
#
|
||||
# # This will download the model automatically if it is not already downloaded
|
||||
# Settings.embed_model = FastEmbedEmbedding(
|
||||
# model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
|
||||
# )
|
||||
pass
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
@register(ModelPlateCategory,'azure-openai')
|
||||
class AzureOpenaiPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
# from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
# from llama_index.llms.azure_openai import AzureOpenAI
|
||||
#
|
||||
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
|
||||
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
|
||||
# max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
|
||||
# dimensions = os.getenv("EMBEDDING_DIM")
|
||||
#
|
||||
# azure_config = {
|
||||
# "api_key": os.environ["AZURE_OPENAI_KEY"],
|
||||
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
|
||||
# or os.getenv("OPENAI_API_VERSION"),
|
||||
# }
|
||||
#
|
||||
# return AzureOpenAI(
|
||||
# model=os.getenv("MODEL"),
|
||||
# max_tokens=int(max_tokens) if max_tokens is not None else None,
|
||||
# temperature=float(temperature),
|
||||
# deployment_name=llm_deployment,
|
||||
# **azure_config,
|
||||
# )
|
||||
pass
|
||||
|
||||
def init_groq():
|
||||
# from llama_index.llms.groq import Groq
|
||||
#
|
||||
# model_map: Dict[str, str] = {
|
||||
# "llama3-8b": "llama3-8b-8192",
|
||||
# "llama3-70b": "llama3-70b-8192",
|
||||
# "mixtral-8x7b": "mixtral-8x7b-32768",
|
||||
# }
|
||||
#
|
||||
# Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
|
||||
# # Groq does not provide embeddings, so we use FastEmbed instead
|
||||
# init_fastembed()
|
||||
pass
|
||||
def embedding(self):
|
||||
# from llama_index.core.constants import DEFAULT_TEMPERATURE
|
||||
# from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
|
||||
#
|
||||
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
|
||||
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
|
||||
# max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
|
||||
# dimensions = os.getenv("EMBEDDING_DIM")
|
||||
#
|
||||
# azure_config = {
|
||||
# "api_key": os.environ["AZURE_OPENAI_KEY"],
|
||||
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
|
||||
# or os.getenv("OPENAI_API_VERSION"),
|
||||
# }
|
||||
# return AzureOpenAIEmbedding(
|
||||
# model=os.getenv("EMBEDDING_MODEL"),
|
||||
# dimensions=int(dimensions) if dimensions is not None else None,
|
||||
# deployment_name=embedding_deployment,
|
||||
# **azure_config,
|
||||
# )
|
||||
pass
|
||||
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
def init_anthropic():
|
||||
# from llama_index.llms.anthropic import Anthropic
|
||||
#
|
||||
# model_map: Dict[str, str] = {
|
||||
# "claude-3-opus": "claude-3-opus-20240229",
|
||||
# "claude-3-sonnet": "claude-3-sonnet-20240229",
|
||||
# "claude-3-haiku": "claude-3-haiku-20240307",
|
||||
# "claude-2.1": "claude-2.1",
|
||||
# "claude-instant-1.2": "claude-instant-1.2",
|
||||
# }
|
||||
#
|
||||
# Settings.llm = Anthropic(model=model_map[os.getenv("MODEL")])
|
||||
# # Anthropic does not provide embeddings, so we use FastEmbed instead
|
||||
# init_fastembed()
|
||||
pass
|
||||
@register(ModelPlateCategory,'fastembed')
|
||||
class FastembedPlatform(ModelPlatform):
|
||||
@abstractmethod
|
||||
def model(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def embedding(self):
|
||||
# from llama_index.embeddings.fastembed import FastEmbedEmbedding
|
||||
#
|
||||
# embed_model_map: Dict[str, str] = {
|
||||
# # Small and multilingual
|
||||
# "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
# # Large and multilingual
|
||||
# "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", # noqa: E501
|
||||
# }
|
||||
#
|
||||
# # This will download the model automatically if it is not already downloaded
|
||||
# Settings.embed_model = FastEmbedEmbedding(
|
||||
# model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
|
||||
# )
|
||||
pass
|
||||
|
||||
def init_gemini():
|
||||
# from llama_index.embeddings.gemini import GeminiEmbedding
|
||||
# from llama_index.llms.gemini import Gemini
|
||||
#
|
||||
# model_name = f"models/{os.getenv('MODEL')}"
|
||||
# embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"
|
||||
#
|
||||
# Settings.llm = Gemini(model=model_name)
|
||||
# Settings.embed_model = GeminiEmbedding(model_name=embed_model_name)
|
||||
pass
|
||||
@abstractmethod
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
def init_mistral():
|
||||
# from llama_index.embeddings.mistralai import MistralAIEmbedding
|
||||
# from llama_index.llms.mistralai import MistralAI
|
||||
#
|
||||
# Settings.llm = MistralAI(model=os.getenv("MODEL"))
|
||||
# Settings.embed_model = MistralAIEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
|
||||
pass
|
||||
@register(ModelPlateCategory,'groq')
|
||||
class GroqPlatform(ModelPlatform):
|
||||
@abstractmethod
|
||||
def model(self):
|
||||
# from llama_index.llms.groq import Groq
|
||||
#
|
||||
# model_map: Dict[str, str] = {
|
||||
# "llama3-8b": "llama3-8b-8192",
|
||||
# "llama3-70b": "llama3-70b-8192",
|
||||
# "mixtral-8x7b": "mixtral-8x7b-32768",
|
||||
# }
|
||||
#
|
||||
# Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
|
||||
# # Groq does not provide embeddings, so we use FastEmbed instead
|
||||
# init_fastembed()
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def embedding(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
@register(ModelPlateCategory,'anthropic')
|
||||
class AnthropicPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
# from llama_index.llms.anthropic import Anthropic
|
||||
#
|
||||
# model_map: Dict[str, str] = {
|
||||
# "claude-3-opus": "claude-3-opus-20240229",
|
||||
# "claude-3-sonnet": "claude-3-sonnet-20240229",
|
||||
# "claude-3-haiku": "claude-3-haiku-20240307",
|
||||
# "claude-2.1": "claude-2.1",
|
||||
# "claude-instant-1.2": "claude-instant-1.2",
|
||||
# }
|
||||
#
|
||||
# Settings.llm = Anthropic(model=model_map[os.getenv("MODEL")])
|
||||
# # Anthropic does not provide embeddings, so we use FastEmbed instead
|
||||
# init_fastembed()
|
||||
pass
|
||||
|
||||
def embedding(self):
|
||||
pass
|
||||
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
@register(ModelPlateCategory,'gemini')
|
||||
class GeminiPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
# from llama_index.llms.gemini import Gemini
|
||||
# model_name = f"models/{os.getenv('MODEL')}"
|
||||
# return Gemini(model=model_name)
|
||||
pass
|
||||
|
||||
def embedding(self):
|
||||
# from llama_index.embeddings.gemini import GeminiEmbedding
|
||||
# embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"
|
||||
# return GeminiEmbedding(model_name=embed_model_name)
|
||||
pass
|
||||
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
@register(ModelPlateCategory,'mistral')
|
||||
class MistralPlatform(ModelPlatform):
|
||||
def model(self):
|
||||
# from llama_index.llms.mistralai import MistralAI
|
||||
# return MistralAI(model=os.getenv("MODEL"))
|
||||
pass
|
||||
|
||||
def embedding(self):
|
||||
# from llama_index.embeddings.mistralai import MistralAIEmbedding
|
||||
# return MistralAIEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
|
||||
pass
|
||||
|
||||
def rerank(self):
|
||||
pass
|
||||
|
||||
def init_ProjectInfo():
|
||||
prjObj = ProjectInfo()
|
||||
@@ -239,3 +332,6 @@ def init_ProjectInfo():
|
||||
for prjInfo in prjInfos:
|
||||
prjObj.add(prjInfo['name'],prjInfo['flag'])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
from llama_index.llms.dashscope import DashScope
|
||||
from llama_index.core.base.llms.types import LLMMetadata
|
||||
|
||||
class DashScopeGenerationModels:
|
||||
"""DashScope Qwen serial models."""
|
||||
|
||||
QWEN_TURBO = "qwen-turbo"
|
||||
QWEN_PLUS = "qwen-plus"
|
||||
QWEN_MAX = "qwen-max"
|
||||
QWEN_MAX_1201 = "qwen-max-1201"
|
||||
QWEN_MAX_LONGCONTEXT = "qwen-max-longcontext"
|
||||
QWEN2_MATH_72B_INSTRUCT = 'qwen2-math-72b-instruct'
|
||||
|
||||
DASHSCOPE_MODEL_META = {
|
||||
DashScopeGenerationModels.QWEN_TURBO: {
|
||||
"context_window": 1024 * 8,
|
||||
"num_output": 1024 * 8,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
DashScopeGenerationModels.QWEN_PLUS: {
|
||||
"context_window": 1024 * 32,
|
||||
"num_output": 1024 * 32,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
DashScopeGenerationModels.QWEN_MAX: {
|
||||
"context_window": 1024 * 8,
|
||||
"num_output": 1024 * 8,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
DashScopeGenerationModels.QWEN_MAX_1201: {
|
||||
"context_window": 1024 * 8,
|
||||
"num_output": 1024 * 8,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
DashScopeGenerationModels.QWEN_MAX_LONGCONTEXT: {
|
||||
"context_window": 1024 * 30,
|
||||
"num_output": 1024 * 30,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
DashScopeGenerationModels.QWEN2_MATH_72B_INSTRUCT: {
|
||||
"context_window": 1024 * 8,
|
||||
"num_output": 1024 * 8,
|
||||
"is_chat_model": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CustomDashScope(DashScope):
|
||||
@property
|
||||
def metadata(self) -> LLMMetadata:
|
||||
DASHSCOPE_MODEL_META[self.model_name]["num_output"] = (
|
||||
self.max_tokens or DASHSCOPE_MODEL_META[self.model_name]["num_output"]
|
||||
)
|
||||
return LLMMetadata(
|
||||
model_name=self.model_name, **DASHSCOPE_MODEL_META[self.model_name]
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from typing import Dict, List
|
||||
|
||||
class ClsRegister:
|
||||
clsLst:Dict[str,Dict[str,str]] = {}
|
||||
|
||||
@classmethod
|
||||
def add(cls,catalog,name,obj) -> None:
|
||||
if catalog in cls.clsLst:
|
||||
registry = cls.clsLst[catalog]
|
||||
registry[name] = obj
|
||||
else:
|
||||
registry:Dict[str,str] = {}
|
||||
registry[name] = obj
|
||||
cls.clsLst[catalog] = registry
|
||||
|
||||
@classmethod
|
||||
def get(cls,catalog,name,fuzzy:bool=False) -> None:
|
||||
if catalog in cls.clsLst:
|
||||
registry = cls.clsLst[catalog]
|
||||
for key,value in registry.items():
|
||||
if fuzzy:
|
||||
if key in name:
|
||||
return value
|
||||
else:
|
||||
if key == name:
|
||||
return value
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def getClsList(cls,catalog) -> None:
|
||||
res_Lst = []
|
||||
if catalog in cls.clsLst:
|
||||
registry = cls.clsLst[catalog]
|
||||
for key,value in registry.items():
|
||||
res_Lst.append(value)
|
||||
return res_Lst
|
||||
|
||||
|
||||
def register(catalog,name):
|
||||
def decorator(className):
|
||||
ClsRegister.add(catalog,name,className)
|
||||
return className
|
||||
return decorator
|
||||
Reference in New Issue
Block a user