ly/zjdataai-app
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: ly/zjdataai-app:6f5548ee617b09941a081d3f35f31234c08297e6
Branches Tags
ly/zjdataai-app:main ly/zjdataai-app:dev-web ly/zjdataai-app:dev ly/zjdataai-app:dev-db
..
compare: ly/zjdataai-app:dev-db
Branches Tags
ly/zjdataai-app:dev-web ly/zjdataai-app:dev ly/zjdataai-app:main ly/zjdataai-app:dev-db

1 Commits
6f5548ee61 .. dev-db

Author SHA1 Message Date
wanyaokun 7e58a1a223 实现多工程数据存储支持 2024-08-13 13:11:17 +08:00
99 changed files with 1315 additions and 381411 deletions
Expand all files Collapse all files
.gitmodules
-3
View File
@@ -1,3 +0,0 @@
[submodule "webapp"]
path = webapp
url = https://git.97id.com/ly/webapp.git
backend/.env.example
-113
View File
@@ -1,113 +0,0 @@
# The Llama Cloud API key.
# LLAMA_CLOUD_API_KEY=
SQL_DATABASE_URL=mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1
#SQL_DATABASE_URL=mysql+pymysql://zjinfo2:GSKcziSdBixDXwcd@110.42.234.166:3306/zjinfo2
SQLITE_DATABASE_URL=sqlite:///./source.db
# The number of similar embeddings to return when retrieving documents.
TOP_K=10
#--------------------------
# 是否启用混合检索
HYBRID_ENABLED = true
# 混合检索阈值
HYBRID_ALPHA = 0.6
# 是否启用检索重排功能
RERANK_ENABLED=true
#---------- rerank- Xinference ----------------
#RERANK_PROVIDER=xinference
#RERANK_MODEL=bge-reranker-v2-m3
#RERANK_BASE_URL=http://10.1.16.39:9995
#RERANK_TOP_N=5
#RERANK_THRESHOLD=0.3
#---------- rerank- ollama ----------------
RERANK_PROVIDER=ollama
RERANK_MODEL= /models/bge-reranker-base
RERANK_TOP_N=5
RERANK_THRESHOLD=0.3
#---------- model - Xinference ----------------
#MODEL_PROVIDER=xinference
#OPENAI_API_KEY=xinference
#BASE_URL=http://172.20.0.145:9995
#MODEL=Qwen2-72B-Instruct-GPTQ-Int8
## Temperature for sampling from the model.
#LLM_TEMPERATURE=0.1
#---------- model - dashscope ----------------
MODEL_PROVIDER=dashscope
DASHSCOPE_API_KEY=sk-221d2d202e104618a56002ce2e7dc0d0
MODEL=qwen2-math-72b-instruct
#---------- embedding - Xinference ----------------
#EMBEDDING_PROVIDER=xinference
#EMBEDDING_MODEL=bge-m3
#EMBEDDING_BASE_URL=http://10.1.16.39:9995
#EMBEDDING_DIM=1024
---------- embedding - dashscope ----------------
EMBEDDING_PROVIDER=dashscope
EMBEDDING_MODEL=text-embedding-v1
# The questions to help users get started (multi-line).
CONVERSATION_STARTERS=本工程指什么？\n总算表有哪些费用？\n项目划分哪些内容构成？\n其他费用表有哪些内容？
# The time in milliseconds to wait for the stream to return a response.
STREAM_TIMEOUT=60000
# 向量存储数据库类型，目前可选：chroma、qdrant
VECTOR_STORE_TYPE=chroma
# The name of the collection in your vector database
VECTOR_STORE_COLLECTION=default
# The API endpoint for your vector database
# VECTOR_STORE_HOST=
# The port for your vector database
# VECTOR_STORE_PORT=
# The local path to the vector database.
# Specify this if you are using a local vector database.
# Otherwise, use VECTOR_STORE__HOST and VECTOR_STORE__PORT config above
VECTOR_STORE_PATH=./storage_vector
BM_RETRIEVER_PATH =./storage_bm
PHOENIX_API_KEY=123456
PHOENIX_URL=http://localhost:6006/v1/traces
PHOENIX_PROJECT_NAME=ly_zjapp
#OTEL_SERVICE_NAME=ly_zjapp
#OTEL_RESOURCE_ATTRIBUTES=openinference.project.name=ly_zjapp
# The address to start the backend app.
APP_HOST=0.0.0.0
# The port to start the backend app.
APP_PORT=8000
FILESERVER_URL_PREFIX=/api/files
# E2B_API_KEY key is required to run code interpreter tool. Get it here: https://e2b.dev/docs/getting-started/api-key
# E2B_API_KEY=
# The system prompt for the AI model.
SYSTEM_PROMPT="You are a weather forecast agent. You help users to get the weather forecast for a given location.
-You are a Python interpreter that can run any python code in a secure environment.
- The python code runs in a Jupyter notebook. Every time you call the 'interpreter' tool, the python code is executed in a separate cell.
- You are given tasks to complete and you run python code to solve them.
- It's okay to make multiple calls to interpreter tool. If you get an error or the result is not what you expected, you can call the tool again. Don't give up too soon!
- Plot visualizations using matplotlib or any other visualization library directly in the notebook.
- You can install any pip package (if it exists) by running a cell with pip install.
"
PRJTOJSON_URL = 'http://10.1.6.60:8092'
PROJECT_TITLE = "您好，我是博微工程理解小助手，您可以问我有关[线路工程]工程数据的相关问题！"
CHAT_UPLOAD_FILECACHE = "./output/uploaded"
JIEBA_DATA=./nltk_data
NLTK_DATA=./nltk_data
backend/.env.xinference
-123
View File
@@ -1,123 +0,0 @@
JIEBA_DATA=./nltk_data
NLTK_DATA=./nltk_data
SQLITE_DATABASE_URL=sqlite:///./source.db
DATA_SOURCE_CACHE=./restapi
# The Llama Cloud API key.
# LLAMA_CLOUD_API_KEY=
SQL_DATABASE_URL=mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1
#SQL_DATABASE_URL=mysql+pymysql://zjinfo2:GSKcziSdBixDXwcd@110.42.234.166:3306/zjinfo2
SQLITE_DATABASE_URL=sqlite:///./source.db
# The number of similar embeddings to return when retrieving documents.
TOP_K=10
#--------------------------
# 是否启用混合检索
HYBRID_ENABLED = true
# 混合检索阈值
HYBRID_ALPHA = 0.6
#--------------------------
# 是否启用检索重排功能
RERANK_ENABLED=true
#---------- rerank- Xinference ----------------
RERANK_PROVIDER=xinference
RERANK_MODEL=bge-reranker-v2-m3
RERANK_BASE_URL=http://10.1.16.39:9995
RERANK_TOP_N=5
RERANK_THRESHOLD=0.3
#---------- model - Xinference ----------------
MODEL_PROVIDER=xinference # The provider for the AI models to use.
OPENAI_API_KEY=xinference # The OpenAI API key to use.
BASE_URL=http://10.1.0.142:9995
MODEL=Qwen2-72B-Instruct-GPTQ-Int8
LLM_TEMPERATURE=0.1 # Temperature for sampling from the model.
#LLM_MAX_TOKENS= # Maximum number of tokens to generate.
#---------- embedding - Xinference ----------------
EMBEDDING_PROVIDER=xinference
EMBEDDING_MODEL=bge-m3
EMBEDDING_BASE_URL=http://10.1.16.39:9995
EMBEDDING_DIM=1024 # Dimension of the embedding model to use.
##---------- OpenAI ----------------
## The provider for the AI models to use.
#MODEL_PROVIDER=openai
## The OpenAI API key to use.
#OPENAI_API_KEY=xinference
#BASE_URL=http://10.1.0.142:9995/v1
#MODEL=Qwen2-72B-Instruct-GPTQ-Int4
## Temperature for sampling from the model.
#LLM_TEMPERATURE=0.1
## Maximum number of tokens to generate.
##LLM_MAX_TOKENS=
## Name of the embedding model to use.
#EMBEDDING_MODEL=text-embedding-v2
## Dimension of the embedding model to use.
#EMBEDDING_DIM=1024
#---------- DashScope ----------------
#DASHSCOPE_API_KEY=sk-02c8540e86d84b7ca0e6f4f51bac6e60
## The provider for the AI models to use.
#MODEL_PROVIDER=dashscope
## The name of LLM model to use.
#MODEL=qwen-max
## Name of the embedding model to use.
#EMBEDDING_MODEL=text-embedding-v2
# The questions to help users get started (multi-line).
CONVERSATION_STARTERS=本工程指什么？\n总算表有哪些费用？\n项目划分哪些内容构成？\n其他费用表有哪些内容？
# The time in milliseconds to wait for the stream to return a response.
STREAM_TIMEOUT=60000
# 向量存储数据库类型，目前可选：chroma、qdrant
VECTOR_STORE_TYPE=chroma
# The name of the collection in your vector database
VECTOR_STORE_COLLECTION=default
# The API endpoint for your vector database
# VECTOR_STORE_HOST=
# The port for your vector database
# VECTOR_STORE_PORT=
# The local path to the vector database.
# Specify this if you are using a local vector database.
# Otherwise, use VECTOR_STORE__HOST and VECTOR_STORE__PORT config above
VECTOR_STORE_PATH=./storage_vector
BM_RETRIEVER_PATH =./storage_bm
PHOENIX_API_KEY=123456
PHOENIX_URL=http://localhost:6006/v1/traces
PHOENIX_PROJECT_NAME=ly_zjapp
#OTEL_SERVICE_NAME=ly_zjapp
#OTEL_RESOURCE_ATTRIBUTES=openinference.project.name=ly_zjapp
# The address to start the backend app.
APP_HOST=0.0.0.0
# The port to start the backend app.
APP_PORT=8000
FILESERVER_URL_PREFIX=/api/files
# E2B_API_KEY key is required to run code interpreter tool. Get it here: https://e2b.dev/docs/getting-started/api-key
# E2B_API_KEY=
# The system prompt for the AI model.
SYSTEM_PROMPT="You are a weather forecast agent. You help users to get the weather forecast for a given location.
-You are a Python interpreter that can run any python code in a secure environment.
- The python code runs in a Jupyter notebook. Every time you call the 'interpreter' tool, the python code is executed in a separate cell.
- You are given tasks to complete and you run python code to solve them.
- It's okay to make multiple calls to interpreter tool. If you get an error or the result is not what you expected, you can call the tool again. Don't give up too soon!
- Plot visualizations using matplotlib or any other visualization library directly in the notebook.
- You can install any pip package (if it exists) by running a cell with pip install.
"
PRJTOJSON_URL = 'http://10.1.6.60:8092'
PROJECT_TITLE = "您好，我是博微工程理解小助手，您可以问我有关[线路工程]工程数据的相关问题！"
CHAT_UPLOAD_FILECACHE = "./output/uploaded"
backend/.gitignore
-3
View File
@@ -2,6 +2,3 @@ __pycache__
storage
.env
output
/storage_vector/
/.idea/
/.python-version
backend/app/api/routers/app.py
-616
View File
@@ -1,616 +0,0 @@
import asyncio
import json
import logging
import time
from typing import Dict, List, Any, Optional, AsyncGenerator
from aiostream import stream
from fastapi import APIRouter, Request,HTTPException
from fastapi.responses import StreamingResponse
from llama_index.core import BaseCallbackHandler
from llama_index.core.base.llms.types import ChatMessage
from llama_index.core.callbacks import CBEventType
from llama_index.core.chat_engine.types import StreamingAgentChatResponse
from llama_index.core.tools import ToolOutput
from llama_index.core.schema import NodeWithScore
from pydantic import BaseModel
from app.api.routers.request.base import userMng, conversations,message,ProjectInfo,feedback
from app.api.routers.request.baseConfig import *
from app.api.routers.request.models import ChatRequestData,ChatFileUploadRequest
from app.engine import get_chat_engine
import uuid
from app.api.routers.services.fileServices import PrjFileLoadService,ChatFileService
from app.api.routers.services.suggestion import NextQuestionSuggestion
import time
from llama_index.core.settings import Settings
logger = logging.getLogger("uvicorn")
v1_router = v = APIRouter()
gEvent_handler = None
CH_Event_map={
'CHUNKING':'文本切片',
'NODE_PARSING':'节点解析',
'EMBEDDING':'生成向量',
'LLM':'知识问答',
'QUERY':'查询',
'RETRIEVE':'检索',
'SYNTHESIZE':'答案合成',
'TREE':'总结',
'SUB_QUESTION':'问题分解',
'TEMPLATING':'生成提示词模板',
'FUNCTION_CALL':'函数调用',
'RERANKING':'节点重排',
'EXCEPTION':'执行异常',
'AGENT_STEP':'单步执行'
}
class ChatCallbackEvent(BaseModel):
event_type: ChatEventType
payload: Optional[Dict[str, Any]] = None
def get_common_param(self)-> dict:
return {
'event': self.event_type.value,
'conversation_id':self.payload.get("conversation_id"),
'message_id': self.payload.get("message_id"),
'created_at': int(time.time()),
'task_id': self.payload.get("task_id")
}
def get_WorkflowStart_param(self) -> dict:
params = self.get_common_param()
params.update({
'workflow_run_id':self.payload.get('workflow_run_id'),
'data':{
"id": self.payload.get('workflow_run_id'),
"workflow_id": self.payload.get('workflow_id'),
"sequence_number": 1709,
"inputs": {
"sys.query": f"开始查询 {self.payload.get('query')}",
"sys.files": [],
"sys.conversation_id": self.payload.get('conversation_id'),
"sys.user_id": self.payload.get('use_id')
},
"created_at": int(time.time())
}
})
return params
def get_WorkflowFinished_param(self) -> dict:
params = self.get_common_param()
params.update({
'workflow_run_id':self.payload.get('workflow_run_id'),
'data':{
"id": self.payload.get('workflow_run_id'),
"workflow_id": self.payload.get('workflow_id'),
"sequence_number": 1709,
"status": "succeeded",
"outputs": {
"answer": self.payload.get('response')
},
"error": '',
"elapsed_time": 36.03764106379822,
"total_tokens": 11707,
"total_steps": 10,
"created_by": {
"id": str(uuid.uuid4()),
"user": self.payload.get('use_id')
},
"created_at": int(time.time()),
"finished_at": int(time.time()),
"files": []
}
})
return params
def get_NodeStart_param(self) -> dict:
params = self.get_common_param()
params.update({
'workflow_run_id':self.payload.get('workflow_run_id'),
'data':{
"id": self.payload.get('nodeid'),
"node_id": self.payload.get('nodeid'),
"node_type": "http-request",
"title": CH_Event_map[self.payload.get('title')],
"index": self.payload.get('index'),
"predecessor_node_id": self.payload.get('predecessor_node_id'),
"inputs": '',
"created_at": 1724398751,
"extras": {}
}
})
return params
def get_NodeFinished_param(self) -> dict:
params = self.get_common_param()
params.update({
'workflow_run_id':self.payload.get('workflow_run_id'),
'data':{
"id": self.payload.get('nodeid'),
"node_id": self.payload.get('nodeid'),
"node_type": "http-request",
"title": CH_Event_map[self.payload.get('title')],
"index": self.payload.get('index'),
"predecessor_node_id": self.payload.get('predecessor_node_id'),
"inputs": '',
"process_data": '',
"outputs": '',
"status": "succeeded",
"error": '',
"elapsed_time": 0.10402441816404462,
"execution_metadata": '',
"created_at": 1724398751,
"finished_at": 1724398751,
"files": []
}
})
return params
def get_Message_param(self) -> dict:
params = self.get_common_param()
params.update({
'id':self.payload.get('message_id'),
'answer':self.payload.get('answer')
})
return params
def get_MessageEnd_param(self) -> dict:
params = self.get_common_param()
nodeInfos = []
source_nodes = self.payload.get('source_node')
if source_nodes is not None:
for i in range(len(source_nodes)):
source_node:NodeWithScore = source_nodes[i]
metadata:dict = source_node.node.metadata
nodeInfo = {
"position": i,
"dataset_id": metadata.get("pipeline_id"),
"dataset_name": metadata.get("file_name"),
"document_id": source_node.node_id,
"document_name": metadata.get("file_name"),
"data_source_type": "upload_file",
"segment_id": source_node.node_id,
"retriever_from": "workflow",
"score": source_node.score,
"hit_count": 1,
"word_count": 632,
"segment_position": i,
"index_node_hash": "",
"content": source_node.text
}
nodeInfos.append(nodeInfo)
params.update({
'id':self.payload.get('message_id'),
'metadata':{
"retriever_resources":nodeInfos,
"usage":{
"prompt_tokens": 4972,
"prompt_unit_price": "0.0",
"prompt_price_unit": "0.0",
"prompt_price": "0.0",
"completion_tokens": 332,
"completion_unit_price": "0.0",
"completion_price_unit": "0.0",
"completion_price": "0.0",
"total_tokens": 5304,
"total_price": "0.0",
"currency": "USD",
"latency": 4.897703120019287
}
}
})
return params
def to_response(self)-> dict|None:
try:
match self.event_type.value:
case "workflow_started":
return self.get_WorkflowStart_param()
case "workflow_finished":
return self.get_WorkflowFinished_param()
case "node_started":
return self.get_NodeStart_param()
case 'node_finished':
return self.get_NodeFinished_param()
case 'message':
return self.get_Message_param()
case 'message_end':
return self.get_MessageEnd_param()
case _:
return None
except Exception as e:
logger.error(f"转换回应时间时发生错误，原因: {e}")
return None
class ChatEventCallbackHandler(BaseCallbackHandler):
_aqueue: asyncio.Queue
is_done: bool = False
def __init__(self):
"""Initialize the base callback handler."""
ignored_events = [
# CBEventType.CHUNKING,
# CBEventType.NODE_PARSING,
# CBEventType.EMBEDDING,
# CBEventType.LLM,
# CBEventType.TEMPLATING,
]
super().__init__(ignored_events, ignored_events)
self._aqueue = asyncio.Queue()
self._response: StreamingAgentChatResponse = None
self._ids:Dict[str,Any] = {}
self._chatData:ChatRequestData = None
self._nodeStack:List[str] = []
self._firstEventID:str = None
def setInitParams(self,ids:dict,data:ChatRequestData):
self._ids = ids
self._chatData = data
self._firstEventID = None
def setResponse(self,response: StreamingAgentChatResponse):
self._response = response
def on_event_start(
self,
event_type: CBEventType,
payload: Optional[Dict[str, Any]] = None,
event_id: str = "",
**kwargs: Any,
) -> str:
if self._firstEventID is None:
self._firstEventID = event_id
self.start()
logger.info("event_start:{} type:{} payload:{}\n".format(event_id, event_type, payload))
self._nodeStack.append(event_id)
nindex = len(self._nodeStack) - 1
args:Dict[str,Any] = self._ids
args.update(
{
'nodeid':event_id,
'title':event_type.name,
'index':nindex + 1,
'predecessor_node_id': self._nodeStack[nindex - 1] if nindex > 0 else ''
}
)
nd_event = ChatCallbackEvent(event_type = ChatEventType.NODE_START,payload = args)
if nd_event.to_response() is not None:
self._aqueue.put_nowait(nd_event)
def on_event_end(
self,
event_type: CBEventType,
payload: Optional[Dict[str, Any]] = None,
event_id: str = "",
**kwargs: Any,
) -> None:
logger.info("event_end:{} type:{} payload:{}\n".format(event_id, event_type, payload))
#self.response = payload.get("response","")
args:Dict[str,Any] = self._ids
nodeID = self._nodeStack[-1]
if nodeID == event_id:
nindex = len(self._nodeStack) - 1
args.update(
{
'nodeid':event_id,
'title':event_type.name,
'index':nindex + 1,
'predecessor_node_id':self._nodeStack[nindex - 1] if nindex > 0 else ''
}
)
nd_event = ChatCallbackEvent(event_type = ChatEventType.NODE_FINISHED,payload = args)
if nd_event.to_response() is not None:
self._aqueue.put_nowait(nd_event)
self._nodeStack.pop()
if self._firstEventID is not None and self._firstEventID == event_id:
self.finished()
def start_trace(self, trace_id: Optional[str] = None) -> None:
"""No-op."""
logger.info("trace_start:{}\n".format(trace_id))
def end_trace(
self,
trace_id: Optional[str] = None,
trace_map: Optional[Dict[str, List[str]]] = None,
) -> None:
"""No-op."""
logger.info("trace_end:{} trace_map:{}\n".format(trace_id, trace_map))
async def async_event_gen(self) -> AsyncGenerator[ChatCallbackEvent, None]:
while not self._aqueue.empty() or not self.is_done:
try:
yield await asyncio.wait_for(self._aqueue.get(), timeout=0.1)
except asyncio.TimeoutError:
pass
def makeWorkflow_startEvent(self)->ChatCallbackEvent:
args:Dict[str,Any] = self._ids
args.update(
{
'use_id': self._chatData.user,
'query': self._chatData.query,
'conversation_id': self._chatData.conversation_id
}
)
return ChatCallbackEvent(event_type = ChatEventType.WORKFLOW_START,payload = args)
def makeWorkflow_finishedEvent(self)->ChatCallbackEvent:
args:Dict[str,Any] = self._ids
args.update(
{
'response': '',
'conversation_id': self._chatData.conversation_id
}
)
return ChatCallbackEvent(event_type = ChatEventType.WORKFLOW_FINISHED,payload = args)
def makeMessage_EndEvent(self)->ChatCallbackEvent:
args:Dict[str,Any] = self._ids
if self._response is not None:
args.update({
'source_node': self._response.source_nodes
})
msgEnt_event = ChatCallbackEvent(event_type = ChatEventType.MESSAGE_END,payload = args)
return msgEnt_event
def start(self):
#添加工作流开始事件
wf_event = self.makeWorkflow_startEvent()
if wf_event.to_response() is not None:
self._aqueue.put_nowait(wf_event)
def finished(self):
wf_event = self.makeWorkflow_finishedEvent()
if wf_event.to_response() is not None:
self._aqueue.put_nowait(wf_event)
msgEnt_event = self.makeMessage_EndEvent()
if msgEnt_event.to_response() is not None:
self._aqueue.put_nowait(msgEnt_event)
class IDManager:
def createID(self):
return {
"message_id" : str(uuid.uuid4()),
'task_id':str(uuid.uuid4()),
'workflow_run_id': str(uuid.uuid4()),
"workflow_id": str(uuid.uuid4())
}
class ChatStreamResponse(StreamingResponse):
TEXT_PREFIX = "data: "
DATA_PREFIX = "data: "
ids:Dict[str,Any] = {}
data:ChatRequestData = None
@classmethod
def convert_Message(cls, token: str):
params = cls.ids
params.update({
'answer':token,
'conversation_id':cls.data.conversation_id
})
event = ChatCallbackEvent(event_type = ChatEventType.MESSAGE,payload = params)
data_str = json.dumps(event.to_response())
return f"{cls.DATA_PREFIX}{data_str}\n\n"
@classmethod
def convert_Event(cls, data: dict):
data_str = json.dumps(data)
return f"{cls.DATA_PREFIX}{data_str}\n\n"
def __init__(
self,
request: Request,
event_handler: ChatEventCallbackHandler,
response: StreamingAgentChatResponse,
data: ChatRequestData,
ids:Dict[str,Any]
):
ChatStreamResponse.ids = ids
ChatStreamResponse.data = data
content = ChatStreamResponse.content_generator(
request, event_handler, response, data
)
super().__init__(content=content)
@classmethod
async def content_generator(
cls,
request: Request,
event_handler: ChatEventCallbackHandler,
response: StreamingAgentChatResponse,
data: ChatRequestData
):
# Yield the text response
async def _chat_response_generator():
final_response = ""
async for token in response.async_response_gen():
final_response += token
yield ChatStreamResponse.convert_Message(token)
# 存储消息历史
message().add(user_id=data.user,conversation_id=data.conversation_id,query=data.query,answer=final_response)
# the text_generator is the leading stream, once it's finished, also finish the event stream
event_handler.is_done = True
event_handler.setResponse(response)
# Yield the events from the event handler
async def _event_generator():
async for event in event_handler.async_event_gen():
event_response = event.to_response()
if event_response is not None:
yield ChatStreamResponse.convert_Event(event_response)
combine = stream.merge(_chat_response_generator(), _event_generator())
is_stream_started = False
async with combine.stream() as streamer:
async for output in streamer:
if not is_stream_started:
is_stream_started = True
yield output
if await request.is_disconnected():
break
@v.post("/chat-messages")
async def post_chatmessages(request: Request, data: ChatRequestData):
global gEvent_handler
userMng.findNoExistCreate(data.user)
data.conversation_id = data.conversation_id if data.conversation_id else str(uuid.uuid4())
conversaObj = conversations()
conversationinfo = conversaObj.get(data.conversation_id)
if conversationinfo is None:
conversationinfo = conversaObj.add(data.conversation_id, data.user, "新建会话",inputs= data.inputs)
# 生成聊天参数
last_message_content = ChatMessage.from_str(data.query)
filters = None
params = data.inputs or {}
# 启动聊天事件监听
ids = IDManager().createID()
if gEvent_handler is None:
gEvent_handler = ChatEventCallbackHandler()
Settings.llm.callback_manager.handlers.append(gEvent_handler)
if gEvent_handler is not None:
gEvent_handler.setInitParams(ids = ids,data = data)
# 获取聊天引擎对象
chat_engine = get_chat_engine(filters=filters, params=params)
# 执行异步聊天
response = await chat_engine.astream_chat(data.query)
# 返回异步消息回应
return ChatStreamResponse(request, gEvent_handler, response, data,ids)
@v.get("/messages")
async def query_messages(user:str, conversation_id:str):
#conversation_id = default_conversation_id if conversation_id is None else conversation_id
datas = []
records = message().gets(user,conversation_id)
if records is None:
return {
"limit": 20,
"has_more": False,
"data": []
}
for record in records:
res = record.dict()
feeds = feedback().query(res['id'])
res["message_files"] = []
res["feedback"] = {'rating':feeds['rating'] } if feeds != None else ''
res["retriever_resources"] = []
res["created_at"] = 1723444905
res["agent_thoughts"] = []
res["status"] = "normal"
res["error"] = ''
datas.append(res)
return {
"limit": 20,
"has_more": False,
"data": datas
}
@v.post("/conversations/{itemid}/name")
async def post_conversations(request: Request,itemid:str,params:Dict[str,Any]):
consaObj = conversations()
consaObj.rename(itemid,'知识问答')
cond = {
'id':itemid,
'user_id':params['user']
}
results = consaObj.query(**cond)
if len(results) > 0:
res = results[0]
return {
"id": res['id'],
"name": res['name'],
"inputs": res['inputs'],
"status": res['status'],
"introduction": res['introduction'],
"created_at": res['created_at'],
#"工程位置"
}
return 'null'
@v.get("/conversations")
async def query_conversations(user:str, first_id:str = None, limit:str = None, pinned:str = None):
user_id = '' if user is None else user
userMng.findNoExistCreate(user_id)
return {
"limit": 20,
"has_more": False,
"data": conversations().gets(user_id)
}
@v.get("/parameters")
async def query_parameters(user:str):
prjObj = ProjectInfo()
return BaseConfig().ParamterCfg(projectInfo = prjObj.projectNames())
@v.post("/messages/{message_id}/feedbacks")
async def post_feedbacks(request: Request,message_id:str,params:Dict[str,Any]):
if params['rating'] is None:
feedback().delete(message_id)
else:
results = message().query(message_id)
if len(results) > 0:
result = results[0]
feedback().add(message_id=message_id,query=result['query'],
answer=result['answer'],rating=params['rating'])
@v.post("/files/upload")
def upload_file(request: ChatFileUploadRequest):
try:
logger.info("Processing file")
resluts = ChatFileService.process_file(request.base64)
return {
'id':resluts.get('id'),
'name': resluts.get('name'),
'size': resluts.get('size'),
'extension':resluts.get('extension'),
'mime_type':resluts.get('mime_type'),
'created_by':str(uuid.uuid4()),
'created_at':int(time.time())
}
except Exception as e:
logger.error(f"Error processing file: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Error processing file")
@v.post("/project")
def upload_file(request: ChatFileUploadRequest):
try:
logger.info("Processing file")
return PrjFileLoadService.process_file(request.base64)
except Exception as e:
logger.error(f"Error processing file: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Error processing file")
@v.post("/messages/{message_id}/suggested")
async def post_suggested(request: Request,message_id:str,user:str):
questions = await NextQuestionSuggestion.suggest_next_questions(message_id)
return {
"result": "success",
"data":questions
}
backend/app/api/routers/chat.py
+1 -3
View File
@@ -48,8 +48,6 @@ async def chat(
):
try:
last_message_content = data.get_last_message_content()
# 由于基于历史消息的提示词没有调整好，所以暂时屏蔽历史消息
data.messages.clear()
messages = data.get_history_messages()
doc_ids = data.get_chat_document_ids()
@@ -126,7 +124,7 @@ async def chat_config() -> ChatConfig:
starter_questions = None
conversation_starters = os.getenv("CONVERSATION_STARTERS")
if conversation_starters and conversation_starters.strip():
starter_questions = conversation_starters.strip().split("\\n")
starter_questions = conversation_starters.strip().split("\n")
return ChatConfig(starter_questions=starter_questions)
backend/app/api/routers/models.py
+3 -4
View File
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Literal, Optional, Set
from llama_index.core.llms import ChatMessage, MessageRole
from llama_index.core.schema import NodeWithScore
from pydantic import BaseModel, Field, validator, field_validator
from pydantic import BaseModel, Field, validator
from pydantic.alias_generators import to_camel
logger = logging.getLogger("uvicorn")
@@ -89,7 +89,7 @@ class ChatData(BaseModel):
}
}
@field_validator("messages")
@validator("messages")
def messages_must_not_be_empty(cls, v):
if len(v) == 0:
raise ValueError("Messages must not be empty")
@@ -173,8 +173,7 @@ class SourceNodes(BaseModel):
def from_source_node(cls, source_node: NodeWithScore):
metadata = source_node.node.metadata
url = cls.get_url_from_metadata(metadata)
#text = 'filename' in metadata and metadata['filename'] or source_node.node.node_id
text = source_node.node.text
text = 'filename' in metadata and metadata['filename'] or source_node.node.node_id
return cls(
id=source_node.node.node_id,
metadata=metadata,
backend/app/api/routers/request/base.py
-189
View File
@@ -1,189 +0,0 @@
from datetime import datetime
import uuid
from app.api.routers.request.baseConfig import BaseConfig
from app.api.routers.request.dbOrm import DBManager
from typing import List
dbManage = DBManager()
class conversations:
def __init__(self) -> None:
self._tableName = 'conversations'
dbManage.createTable(self._tableName)
def gets(self,user_id:str):
records = dbManage.query(self._tableName,user_id = user_id)
datas = []
for record in records:
datas.append(record)
return datas
def get(self, id:str):
records = dbManage.query(self._tableName, id=id)
if len(records) >0:
return records[0]
return None
def add(self,id:str, user_id:str, name:str,inputs:dict):
template = BaseConfig().ConversationCfg()
template['id'] = id
template['user_id'] = user_id
template['name'] = name
template['created_at'] = 1724399038
template['inputs'] = inputs
dbManage.addRecord(self._tableName,template)
def delete(self,id:str):
dbManage.delete(self._tableName,id=id)
def rename(self,id:str,name:str):
data = {'name':name}
dbManage.update(self._tableName,data,id=id)
def query(self,**condition):
results = []
records = dbManage.query(self._tableName,**condition)
for record in records:
results.append(record.dict())
return results
class user:
def __init__(self) -> None:
self._tableName = 'user'
dbManage.createTable(self._tableName)
def gets(self):
return dbManage.query(self._tableName)
def get(self,id:str):
return dbManage.query(self._tableName,id = id)
def add(self,id:str):
info = {
'id':id,
'createtime': datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
dbManage.addRecord(self._tableName,info)
def delete(self,id:str):
dbManage.delete(self._tableName,id = id)
class userMng:
userObj = user()
@classmethod
def findNoExistCreate(cls,user_id:str):
userInfo = cls.userObj.get(user_id)
if len(userInfo) == 0:
cls.userObj.add(user_id)
def remove(cls,user_id:str):
cls.userObj.delete(user_id)
class parameter:
def __init__(self) -> None:
self._tableName = 'parameters'
dbManage.createTable(self._tableName)
def get(self,user_id:str):
records = dbManage.query(self._tableName,user_id = user_id)
data = {}
for record in records:
key = record['name']
value = record['value']
data[key] = value
return data
def set(self,user_id:str):
dbManage.addRecord(self._tableName,{})
def delete(self,user_id:str):
dbManage.delete(self._tableName,user_id = user_id)
class message:
def __init__(self) -> None:
self._tableName = 'messages'
dbManage.createTable(self._tableName)
def gets(self,user_id:str,conversation_id:str):
records = dbManage.query(self._tableName,user_id = user_id,conversation_id = conversation_id)
datas = []
for record in records:
datas.append(record)
return datas
def add(self,user_id:str,conversation_id:str,query:str,answer:str):
template = BaseConfig.MessageCfg()
template['id'] = str(uuid.uuid4())
template['user_id'] = user_id
template['conversation_id'] = conversation_id
template['query'] = query
template['answer'] = answer
dbManage.addRecord(self._tableName,template)
def delete(self,user_id:str):
dbManage.delete(self._tableName,user_id = user_id)
def query(self,id:str):
results = []
condition = {'id':id}
records = dbManage.query(self._tableName,**condition)
for record in records:
results.append(record.dict())
return results
class feedback:
def __init__(self) -> None:
self._tableName = 'feedbacks'
dbManage.createTable(self._tableName)
def add(self,message_id:str,query:str,answer:str,rating:str):
record = {
'message_id': message_id,
'query': query,
'answer': answer,
'rating': rating,
}
dbManage.addRecord(self._tableName,record)
def delete(self,message_id:str):
cond = {'message_id':message_id}
dbManage.delete(self._tableName,**cond)
def query(self,message_id:str):
cond = {'message_id':message_id}
records = dbManage.query(self._tableName,**cond)
if len(records) > 0:
return records[0].dict()
return None
class ProjectInfo:
def __init__(self) -> None:
self._tableName = 'projectInfos'
dbManage.createTable(self._tableName)
def add(self,name:str,flag:str):
info = dbManage.query(self._tableName,prjFlag = flag)
if len(info) == 0:
record = {
'prjectName': name,
'prjFlag': flag
}
dbManage.addRecord(self._tableName,record)
def projectNames(self)->List[str]:
records = dbManage.query(self._tableName)
names = []
for record in records:
data:dict = record.dict()
name = data.get('prjectName')
if name !='':
names.append(name)
return names
def prjFalg(self,name:str):
records = dbManage.query(self._tableName)
for record in records:
data:dict = record.dict()
if data.get('prjectName') == name:
return data['prjFlag']
return ''
backend/app/api/routers/request/baseConfig.py
-103
View File
@@ -1,103 +0,0 @@
from pydantic import BaseModel
import os
from enum import Enum
class BaseConfig(BaseModel):
projectInfo:str = os.getenv("PROJECT_TITLE","会话提示消息")
def ParamterCfg(self,**args):
prjItems = args.get('projectInfo')
questions = os.getenv("CONVERSATION_STARTERS", "dev")
return{
"opening_statement": self.projectInfo,
"suggested_questions": questions.split('\n'),
"suggested_questions_after_answer": {
"enabled": False
},
"speech_to_text": {
"enabled": False
},
"text_to_speech": {
"enabled": False,
"language": "",
"voice": ""
},
"retriever_resource": {
"enabled": True
},
"annotation_reply": {
"enabled": False
},
"more_like_this": {
"enabled": False
},
"user_input_form": [
{
"select": {
"variable": "projectname",
"label": "\u5de5\u7a0b\u540d\u79f0",
"type": "select",
"max_length": 48,
"required": True,
"options": prjItems
}
}
],
"sensitive_word_avoidance": {
"enabled": False
},
"file_upload": {
"image": {
"enabled": False,
"number_limits": 3,
"transfer_methods": [
"remote_url"
]
}
},
"system_parameters": {
"image_file_size_limit": "10",
"language": "",
"voice": "",
},
"retriever_resource": {
"enabled": True
},
"annotation_reply": {
"enabled": False
},
"more_like_this": {
"enabled": False
},
}
def ConversationCfg(self):
return{
"id": "",
'user_id':'',
"name": "",
"inputs": {},
"status": "normal",
"introduction": self.projectInfo,
"created_at":''
}
@classmethod
def MessageCfg(cls):
return {
"id": "",
'user_id':'',
"conversation_id": "",
"inputs": {},
"query": "",
"answer": ""
}
class ChatEventType(str, Enum):
WORKFLOW_START = "workflow_started"
WORKFLOW_FINISHED = "workflow_finished"
NODE_START = "node_started"
NODE_FINISHED = "node_finished"
MESSAGE = "message"
MESSAGE_END = "message_end"
backend/app/api/routers/request/dbOrm.py
-239
View File
@@ -1,239 +0,0 @@
import os
from typing import Dict, List, Any
from pydantic import BaseModel
from sqlalchemy import create_engine, Column, String, Integer, JSON,Float
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.orm import sessionmaker, declarative_base
Base = declarative_base()
#orm类
class ConversationOrm(Base):
__tablename__ = "conversations"
id = Column(String, primary_key=True)
user_id = Column(String)
name = Column(String)
inputs = Column(JSON)
status = Column(String)
introduction = Column(String)
created_at = Column(Integer)
def update(self,data:Dict[str,Any]):
if 'name' in data:
self.name = data['name']
class UserOrm(Base):
__tablename__ = "user"
id = Column(String, primary_key=True)
createtime = Column(String)
class ParametersOrm(Base):
__tablename__ = "parameters"
user_id = Column(String,primary_key=True)
name = Column(String)
value = Column(JSON)
class MessagesOrm(Base):
__tablename__ = "messages"
id = Column(String,primary_key=True)
user_id = Column(String)
conversation_id = Column(String)
inputs = Column(JSON)
query = Column(String)
answer = Column(String)
class FeedBackOrm(Base):
__tablename__ = "feedbacks"
message_id = Column(String,primary_key=True)
query = Column(String)
answer = Column(String)
rating = Column(String)
class ProjectInfoOrm(Base):
__tablename__ = "projectInfos"
prjFlag = Column(String,primary_key=True)
prjectName = Column(String)
#数据结构
class ConversationModel(BaseModel):
id: str
name: str
inputs: Dict[str, Any]
status: str
introduction: str
created_at: int
class Config:
from_attributes=True
@classmethod
def orm(cls):
return ConversationOrm
class UserModel(BaseModel):
id: str
createtime: str
class Config:
from_attributes=True
@classmethod
def orm(cls):
return UserOrm
class ParametersModel(BaseModel):
user_id : str
name : str
value : Dict[str, Any]
class Config:
from_attributes=True
@classmethod
def orm(cls):
return ParametersOrm
class MessagesModel(BaseModel):
id :str
conversation_id :str
inputs : Dict[str, Any]
query : str
answer : str
class Config:
from_attributes=True
@classmethod
def orm(cls):
return MessagesOrm
class FeedBackModel(BaseModel):
message_id :str
query :str
answer :str
rating :str
class Config:
from_attributes=True
@classmethod
def orm(cls):
return FeedBackOrm
class ProjectInfoModel(BaseModel):
prjectName:str
prjFlag:str
class Config:
from_attributes=True
@classmethod
def orm(cls):
return ProjectInfoOrm
class DBManager:
def __init__(self) -> None:
DATABASE_URL = os.getenv("SQLITE_DATABASE_URL")
self._engine = create_engine(DATABASE_URL)
self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self._engine)
def createTable(self,tableName:str):
if self._engine is None:
return
if not self.exist(tableName):
Base.metadata.tables[tableName].create(self._engine)
def addRecord(self,tableName:str,record:Dict[str,Any]):
ormCls = self._get_orm(tableName)
if ormCls is None:
return
session = self.SessionLocal()
data = ormCls(**record)
session.add(data)
session.commit()
def addRecords(self,tableName:str,records:List[Dict[str,Any]]):
ormCls = self._get_orm(tableName)
if ormCls is None:
return
datas = []
session = self.SessionLocal()
for record in records:
datas.append(ormCls(**record))
session.add(datas)
session.commit()
def delete(self,tableName:str,**filter):
session = self.SessionLocal()
ormCls = self._get_orm(tableName)
if ormCls is None:
return
records = session.query(ormCls).filter_by(**filter).all()
if records is not None:
for record in records:
session.delete(record)
session.commit()
def update(self,tableName:str,data:Dict[str,Any],**filter):
if not self.exist(tableName):
return
session = self.SessionLocal()
ormCls = self._get_orm(tableName)
if ormCls is None:
return
if len(filter) > 0:
records = session.query(ormCls).filter_by(**filter).all()
else:
records = session.query(ormCls).all()
for record in records:
if record is not None:
record.update(data)
session.commit()
def query(self,tableName:str,**filter):
session = self.SessionLocal()
ormCls = self._get_orm(tableName)
if ormCls is None:
return
modelCls = self._get_model(ormCls)
if modelCls is None:
return
if filter is not None:
records = session.query(ormCls).filter_by(**filter).all()
else:
records = session.query(ormCls).all()
datas = []
for record in records:
datas.append(modelCls.from_orm(record))
return datas
def exist(self,tableName:str)->bool:
if self._engine is None:
return
inspector = Inspector.from_engine(self._engine)
return inspector.has_table(tableName)
def _get_orm(self,tableName:str):
subClss = Base.__subclasses__()
for sunCls in subClss:
if sunCls.__tablename__ == tableName:
return sunCls
return None
def _get_model(self,orm:Any):
subClss = BaseModel.__subclasses__()
for sunCls in subClss:
if 'orm' in sunCls.__dict__ and sunCls.orm() == orm:
return sunCls
return None
backend/app/api/routers/request/models.py
-17
View File
@@ -1,17 +0,0 @@
from typing import Dict, Any
from pydantic import BaseModel
from typing import Optional
class ChatRequestData(BaseModel):
inputs: Dict[str,Any]
query: str
user: str
response_mode: str
files: Any
conversation_id: str = None
class ChatFileUploadRequest(BaseModel):
base64: str
backend/app/api/routers/services/fileServices.py
-134
View File
@@ -1,134 +0,0 @@
import base64,os,mimetypes,requests,tempfile
from typing import List,Dict,Any
from uuid import uuid4
from app.settings import init_settings
from app.engine.loaders import get_document_Types, get_documents,getFileCacahePath
from app.engine.vectordb import get_vector_store
from app.engine.generate import get_doc_store,run_pipeline,persist_storage
from llama_index.core.schema import Document
from pathlib import Path
from llama_index.core.readers.file.base import (
_try_loading_included_file_formats as get_file_loaders_map,
)
from llama_index.readers.file import FlatReader
from llama_index.core.ingestion import IngestionPipeline
from llama_index.core import VectorStoreIndex
from app.engine.index import get_index
STORAGE_DIR = os.getenv("STORAGE_DIR", "storage")
class PrjFileLoadService:
@staticmethod
def store_and_parse_file(file_data):
prjtoJson_url = os.getenv('PRJTOJSON_URL')
convert_url = prjtoJson_url +'/prj_convert_clt2json'
files ={'file':file_data}
response1 = requests.post(
url = convert_url,
files=files
)
if response1.text is None or response1.text=='':
return None
load_url = prjtoJson_url +'/file_download'
response2 = requests.post(
url = load_url,
data=response1.text
)
if response2.text is None or response2.content=='':
return None
try:
tempFilePath:str = tempfile.gettempdir() + f"\\{uuid4().hex}.zip"
with open(tempFilePath,'wb') as file:
file.write(response2.content)
prjID = str(uuid4())
filePath = getFileCacahePath() + f'/Projects/{prjID}'
os.makedirs(filePath)
import zipfile
with zipfile.ZipFile(tempFilePath,'r') as zip_File:
for zip_info in zip_File.infolist():
zip_info.filename = zip_info.filename.encode('cp437').decode('gbk')
zip_File.extract(zip_info,filePath)
os.remove(tempFilePath)
return f'Projects_{prjID}'
except Exception as e:
return None
@staticmethod
def process_file(base64_content: str) -> str:
prjFlag = PrjFileLoadService.store_and_parse_file(base64_content)
if prjFlag is None:
return None
#生成向量并持久化至本地
documents = get_documents(prjFlag)
for doc in documents:
doc.metadata["private"] = "false"
docstore = get_doc_store(prjFlag)
vector_store = get_vector_store(prjFlag)
_ = run_pipeline(docstore, vector_store, documents)
persist_storage(docstore, vector_store)
return prjFlag
class ChatFileService:
PRIVATE_STORE_PATH = os.getenv('CHAT_UPLOAD_FILECACHE','output/uploaded')
resluts:Dict[str,Any] = {}
@staticmethod
def process_file(base64_content: str) -> dict:
file_data, extension = ChatFileService.preprocess_base64_file(base64_content)
documents = ChatFileService.store_and_parse_file(file_data, extension)
pipeline = IngestionPipeline()
nodes = pipeline.run(documents=documents)
current_index = get_index()
pipeline = IngestionPipeline()
nodes = pipeline.run(documents=documents)
if current_index is None:
current_index = VectorStoreIndex(nodes=nodes)
else:
current_index.insert_nodes(nodes=nodes)
current_index.storage_context.persist(
persist_dir=os.environ.get("STORAGE_DIR", "storage")
)
return ChatFileService.resluts
@staticmethod
def preprocess_base64_file(base64_content: str) -> tuple:
header, data = base64_content.split(",", 1)
mime_type = header.split(";")[0].split(":", 1)[1]
extension = mimetypes.guess_extension(mime_type)
ChatFileService.resluts['mime_type'] = mime_type
ChatFileService.resluts['extension'] = extension
return base64.b64decode(data), extension
@staticmethod
def store_and_parse_file(file_data, extension) -> List[Document]:
os.makedirs(ChatFileService.PRIVATE_STORE_PATH, exist_ok=True)
fileID = uuid4().hex
file_name = f"{fileID}{extension}"
file_path = Path(os.path.join(ChatFileService.PRIVATE_STORE_PATH, file_name))
ChatFileService.resluts['id'] = fileID
ChatFileService.resluts['file_name'] = file_name
with open(file_path, "wb") as f:
f.write(file_data)
ChatFileService.resluts['size'] = os.path.getsize(file_path)
reader_cls = ChatFileService.default_file_loaders_map().get(extension)
if reader_cls is None:
raise ValueError(f"File extension {extension} is not supported")
reader = reader_cls()
documents = reader.load_data(file_path)
for doc in documents:
doc.metadata["file_name"] = file_name
doc.metadata["private"] = "true"
return documents
@staticmethod
def default_file_loaders_map():
default_loaders = get_file_loaders_map()
default_loaders[".txt"] = FlatReader
return default_loaders
backend/app/api/routers/services/suggestion.py
-43
View File
@@ -1,43 +0,0 @@
from typing import List
from app.api.routers.request.base import message
from llama_index.core.prompts import PromptTemplate
from llama_index.core.settings import Settings
from pydantic import BaseModel
NEXT_QUESTIONS_SUGGESTION_PROMPT = PromptTemplate(
"你是一个乐于助人的助手！你的任务是对用户可能会问的下一个问题给出建议。 "
"\n这是对话历史记录"
"\n---------------------\n{conversation}\n---------------------"
"考虑到对话历史记录，仅限于现在知识库已有内容, 请给我 $number_of_questions 个你接下来可能会问题的问题！"
)
N_QUESTION_TO_GENERATE = 3
class NextQuestions(BaseModel):
"""A list of questions that user might ask next"""
questions: List[str]
class NextQuestionSuggestion:
@staticmethod
async def suggest_next_questions(
message_id: str,
number_of_questions: int = N_QUESTION_TO_GENERATE,
) -> List[str]:
last_user_message = None
last_assistant_message = None
results = message().query(message_id)
if len(results) > 0:
last_user_message = results[0]['query']
last_assistant_message = results[0]['answer']
conversation: str = f"{last_user_message}\n{last_assistant_message}"
output: NextQuestions = await Settings.llm.astructured_predict(
NextQuestions,
prompt=NEXT_QUESTIONS_SUGGESTION_PROMPT,
conversation=conversation,
nun_questions=number_of_questions,
)
return output.questions
return []
backend/app/api/services/file.py
+3 -1
View File
@@ -87,7 +87,9 @@ class PrivateFileService:
nodes = pipeline.run(documents=documents)
# Add the nodes to the index and persist it
current_index = get_index()
indexs = get_index()
if len(indexs) > 0:
current_index = list(indexs.values())[0]
# Insert the documents into the index
if isinstance(current_index, LlamaCloudIndex):
backend/app/api/services/suggestion.py
+3 -3
View File
@@ -6,10 +6,10 @@ from llama_index.core.settings import Settings
from pydantic import BaseModel
NEXT_QUESTIONS_SUGGESTION_PROMPT = PromptTemplate(
"你是一个乐于助人的助手！你的任务是对用户可能会问的下一个问题给出建议。 "
"\n这是对话历史记录"
"You're a helpful assistant! Your task is to suggest the next question that user might ask. "
"\nHere is the conversation history"
"\n---------------------\n{conversation}\n---------------------"
"考虑到对话历史记录，仅限于现在知识库已有内容, 请给我 $number_of_questions 个你接下来可能会问题的问题！"
"Given the conversation history, please give me $number_of_questions questions that you might ask next!"
)
N_QUESTION_TO_GENERATE = 3
backend/app/engine/__init__.py
+47 -41
View File
@@ -1,73 +1,79 @@
import os
from llama_index.core.agent import AgentRunner, ReActChatFormatter
from llama_index.core import SQLDatabase, SummaryIndex, VectorStoreIndex
from llama_index.core.indices.struct_store import SQLTableRetrieverQueryEngine
from llama_index.core.objects import SQLTableNodeMapping, ObjectIndex
from llama_index.core.settings import Settings
from llama_index.core.agent import AgentRunner, StructuredPlannerAgent, FunctionCallingAgentWorker
from llama_index.core.tools.query_engine import QueryEngineTool
from sqlalchemy import create_engine, Engine
from app.engine.engine import create_query_engine, create_summary_query_engine
from app.engine.index import get_index
from app.engine.prompt import ReActChatFormatter_messages, tree_summary_query_engine_tool_messages, \
query_engine_tool_messages, summary_query_tool_messages
#from app.engine.loaders.db import makeDescriptionByEngine
from app.engine.loaders.db import makeDescriptionByEngine
from app.engine.tools import ToolFactory
from app.api.routers.request.base import ProjectInfo
from llama_index.core.response_synthesizers import ResponseMode
from app.engine.index import get_index
def getPrjFalg(params:dict=None)->str:
prjFlag = ''
if params is not None:
prjFlag = ProjectInfo().prjFalg(params.get('projectname'))
return prjFlag
sql_database = None
sql_obj_index = None
def get_chat_engine(filters=None, params:dict=None):
def get_chat_engine(filters=None, params=None):
system_prompt = os.getenv("SYSTEM_PROMPT")
top_k = int(os.getenv("TOP_K", "3"))
use_reranker = os.getenv("RERANK_ENABLED")
tools = []
global sql_obj_index
global sql_database
if sql_obj_index is None:
sqlengine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
sql_database = SQLDatabase(sqlengine)
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
sql_obj_index = ObjectIndex.from_objects(
table_schema_objs,
table_node_mapping,
index_cls=VectorStoreIndex,
)
# 创建SQL查询工具
# sql_query_engine = create_summary_query_engine(index)
# sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
# name="zjdata_query_tool",
# description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具"
# )
#tools.append(sql_query_tool)
sql_query_engine = SQLTableRetrieverQueryEngine(sql_database,
sql_obj_index.as_retriever(similarity_top_k=top_k),
verbose=True,)
sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
name="zjdata_query_tool",
description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具")
# Add query tool if index exists
index = get_index(getPrjFalg(params))
indexs = get_index()
if len(indexs) > 0:
index = list(indexs.values())[0]
if index is not None:
summary_query_engine = create_summary_query_engine(index,top_k,use_reranker,filters)
summary_index = SummaryIndex(index.vector_store.get_nodes(node_ids=None))
summary_query_engine = summary_index.as_query_engine()
summary_query_tool = QueryEngineTool.from_defaults( query_engine=summary_query_engine, name="summary_query_tool",
description=summary_query_tool_messages,
description="适用于任何需要进行全面总结、概括的要求。",
#description="适用于任何需要对所有内容进行全面总结的请求。有关电力造价领域更具体部分的问题，请使用zj_query_engine_tool",
)
query_engine = create_query_engine(index,top_k,use_reranker,filters,response_mode = ResponseMode.TREE_SUMMARIZE)
# 创建向量检索查询工具
query_engine = index.as_query_engine(
similarity_top_k=top_k, filters=filters
)
query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine, name="zj_query_tool",
description=query_engine_tool_messages)
query_engine = create_query_engine(index,top_k,use_reranker,filters,response_mode = ResponseMode.TREE_SUMMARIZE)
query_engine_tool_1 = QueryEngineTool.from_defaults(query_engine=query_engine, name="zj_query_tool_1",
description=tree_summary_query_engine_tool_messages)
tools.append(query_engine_tool)
tools.append(query_engine_tool_1)
description="由博微公司编制的关于电力造价知识、电力造价编制软件知识和造价工程文件结构的知识库。适用于查询电力领域、电力造价领域、博微、博微电力、博微造价等业务等内容。如果本知识库没有直接答案但有解决思路的可以返回解决办法后建议使用“zjdata_query_tool”工具。",
)
tools.append(summary_query_tool)
tools.append(query_engine_tool)
#tools.append(sql_query_tool)
# Add additional tools
tools += ToolFactory.from_env()
react_chat_formatter = ReActChatFormatter.from_defaults(ReActChatFormatter_messages)
agentrunner = AgentRunner.from_llm(
return AgentRunner.from_llm(
llm=Settings.llm,
tools=tools,
react_chat_formatter=react_chat_formatter,
system_prompt=system_prompt,
verbose=True,
)
return agentrunner
# create the function calling worker for reasoning
# worker = FunctionCallingAgentWorker.from_tools(
# tools, verbose=True
backend/app/engine/engine.py
-135
View File
@@ -1,135 +0,0 @@
import os
from llama_index.core import SummaryIndex, SQLDatabase, VectorStoreIndex
from llama_index.core.indices.struct_store import SQLTableRetrieverQueryEngine
from llama_index.core.objects import SQLTableNodeMapping, ObjectIndex, SQLTableSchema
from llama_index.core.query_engine import RetrieverQueryEngine
from llama_index.core.response_synthesizers import ResponseMode
from llama_index.readers.database import DatabaseReader
from sqlalchemy import create_engine
from util.register import *
from app.engine.prompt import text_qa_template, refine_template, summary_template, simple_template
from app.engine.retriever.HybridRetriever import HybridRetriever
from app.engine.response.treeSummResponse import CustomTreeResponse
from llama_index.core.settings import Settings
ModelPlateCategory = '模型平台'
def get_node_postprocessors():
rerank_enabled = os.getenv("RERANK_ENABLED").title()
if rerank_enabled is None or rerank_enabled == 'False':
return []
Rerank_provider = os.getenv("RERANK_PROVIDER")
modelPaltCls = ClsRegister.get(ModelPlateCategory,Rerank_provider)
postprocess = None
if modelPaltCls is not None:
modelPalt = modelPaltCls()
postprocess = modelPalt.rerank()
else:
raise ValueError(f"Invalid rerank provider: {Rerank_provider}")
return postprocess
def makeDescriptionByEngine(sql_database:SQLDatabase):
reader = DatabaseReader(sql_database)
table_names = sql_database.get_usable_table_names()
table_schema_objs = []
for table_name in table_names:
columns = sql_database.get_table_columns(table_name)
if len(columns) > 150:
continue
stats_txt = ""
if table_name == 'gongchengshuxing':
stats_txt = '该表中有以下属性:'
documents = reader.load_data(query='select name from gongchengshuxing')
for index in range(len(documents) if len(documents) < 30 else 30):
if index == 0:
continue
elif index > 1:
stats_txt += ','
stats_txt += documents[index].text.split(':')[1]
tbSchema = (SQLTableSchema(table_name=table_name, context_str=stats_txt))
table_schema_objs.append(tbSchema)
return table_schema_objs
def get_Retriever(index,**kwargs):
strEnableHybrid = os.getenv("HYBRID_ENABLED",'False')
bEnableHybrid = True if strEnableHybrid is not None and strEnableHybrid.title() == 'True' else False
if bEnableHybrid:
alpha = float(os.getenv("HYBRID_ALPHA", "0.5"))
retriever = HybridRetriever(index,alpha = alpha,**kwargs)
else:
retriever = index.as_retriever(**kwargs)
return retriever
def get_synthesizer():
return CustomTreeResponse(
llm=Settings.llm,
summary_template=summary_template,
use_async=True,
streaming=False,
)
sql_database = None
sql_obj_index = None
# Create a summary query engine
def create_summary_query_engine(top_k=3, use_reranker=False, filters=None):
global sql_obj_index
global sql_database
if sql_obj_index is None or sql_database is None:
sqlengine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
sql_database = SQLDatabase(sqlengine)
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
sql_obj_index = ObjectIndex.from_objects(
table_schema_objs,
table_node_mapping,
index_cls=VectorStoreIndex,
)
# 创建SQL查询工具
sql_query_engine = SQLTableRetrieverQueryEngine(sql_database,
sql_obj_index.as_retriever(similarity_top_k=top_k),
verbose=True,
)
return sql_query_engine
# Create a summary query engine
def create_summary_query_engine(index, top_k=3, use_reranker=False, filters=None):
summary_index = SummaryIndex(index.vector_store.get_nodes(node_ids=None))
summary_query_engine = summary_index.as_query_engine(
response_mode=ResponseMode.TREE_SUMMARIZE,
use_async=True,
streaming=False,
)
return summary_query_engine
# Create a query engine
def create_query_engine(index, top_k=3, use_reranker=False, filters=None, response_mode=None):
# 创建向量检索查询工具
postprocess = None
if use_reranker:
postprocess = get_node_postprocessors()
query_engine = RetrieverQueryEngine.from_args(
get_Retriever(index,
similarity_top_k=top_k,
filters=filters),
text_qa_template=text_qa_template,
refine_template=refine_template,
summary_template = summary_template,
simple_template = simple_template,
node_postprocessors=postprocess,
use_async=True,
streaming=False,
response_mode = response_mode
)
return query_engine
backend/app/engine/generate.py
+5 -21
View File
@@ -8,9 +8,8 @@ import os
from app.engine.loaders import get_document_Types, get_documents
from app.engine.vectordb import get_vector_store
from app.settings import init_settings
from app.engine.retriever.CHBM25Retriever import CHBM25Retriever
from llama_index.core.ingestion import IngestionPipeline
from llama_index.core.node_parser import SentenceSplitter,MarkdownNodeParser
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core.settings import Settings
from llama_index.core.storage import StorageContext
from llama_index.core.storage.docstore import SimpleDocumentStore
@@ -20,7 +19,6 @@ logger = logging.getLogger()
STORAGE_DIR = os.getenv("STORAGE_DIR", "storage")
def get_doc_store(docType:str):
# If the storage directory is there, load the document store from it.
@@ -31,15 +29,13 @@ def get_doc_store(docType:str):
else:
return SimpleDocumentStore()
def run_pipeline(docstore, vector_store, documents):
pipeline = IngestionPipeline(
transformations=[
#SentenceSplitter(
#chunk_size=Settings.chunk_size,
#chunk_overlap=Settings.chunk_overlap,
#),
#MarkdownNodeParser(),
SentenceSplitter(
chunk_size=Settings.chunk_size,
chunk_overlap=Settings.chunk_overlap,
),
Settings.embed_model,
],
docstore=docstore,
@@ -52,7 +48,6 @@ def run_pipeline(docstore, vector_store, documents):
return nodes
def persist_storage(docstore, vector_store):
storage_context = StorageContext.from_defaults(
docstore=docstore,
@@ -60,15 +55,6 @@ def persist_storage(docstore, vector_store):
)
storage_context.persist(STORAGE_DIR)
def persist_BMRetriever(vector_store):
STORAGE_DIR = os.getenv("BM_RETRIEVER_PATH", "storage_bm")
nodes = vector_store.get_nodes([])
top_k = min(int(os.getenv("TOP_K", "3")),len(nodes))
bmRetriver = CHBM25Retriever.from_defaults(similarity_top_k=top_k,nodes = nodes)
bmRetriver.persist(STORAGE_DIR)
def generate_datasource():
init_settings()
logger.info("Generate index for the provided data")
@@ -88,11 +74,9 @@ def generate_datasource():
# Build the index and persist storage
persist_storage(docstore, vector_store)
persist_BMRetriever(vector_store)
logger.info("Finished generating the index")
if __name__ == "__main__":
from phoenix.trace import using_project
with using_project(os.getenv("PHOENIX_PROJECT_NAME") + "_generate") as obj:
backend/app/engine/index.py
+18 -10
View File
@@ -1,15 +1,23 @@
import logging
from llama_index.core.indices import VectorStoreIndex
from app.engine.vectordb import get_vector_store
from app.engine.loaders import get_document_Types
from typing import Dict,Any
from app.engine.generate import get_document_Types
logger = logging.getLogger("uvicorn")
def get_index(prjFlag:str):
if prjFlag is None or prjFlag == '':
raise ValueError('无效的工程标识')
logger.info("Connecting vector store...")
store = get_vector_store(prjFlag)
index = VectorStoreIndex.from_vector_store(store)
logger.info("Finished load index from vector store.")
return index
indexs = {}
def get_index(params=None):
global indexs
if len(index) <= 0:
logger.info("Connecting vector store...")
docTypes = get_document_Types()
for docType in docTypes:
store = get_vector_store(docType)
# Load the index from the vector store
# If you are using a vector store that doesn't store text,
# you must load the index from both the vector store and the document store
index = VectorStoreIndex.from_vector_store(store)
logger.info("Finished load index from vector store.")
indexs[docType] = index
return indexs
backend/app/engine/loaders/__init__.py
+20 -50
View File
@@ -1,16 +1,15 @@
import logging
import yaml
from app.engine.loaders.db import DBLoaderConfig, get_db_documents
from app.engine.loaders.file import FileLoaderConfig, get_file_documents
from app.engine.loaders.web import WebLoaderConfig, get_web_documents
from app.engine.loaders.file import getProjectName
import os
logger = logging.getLogger(__name__)
def load_configs():
with open("config/loaders.yaml",encoding='utf-8') as f:
with open("config/loaders.yaml") as f:
configs = yaml.safe_load(f)
return configs
@@ -33,7 +32,8 @@ def path_difference(path1:str, path2:str):
pathKey+=path2_parts[j] + '_'
return pathKey[0:-1]
def getFileCacahePath():
def get_document_Types():
import os
rootPath = 'data'
configs = load_configs()
if configs is not None and len(configs.items()) > 0:
@@ -41,10 +41,7 @@ def getFileCacahePath():
if loader_type == "file":
rootPath = FileLoaderConfig(**loader_config).data_dir
break
return rootPath
def get_document_Types():
rootPath = getFileCacahePath()
types = []
dirStack = [rootPath]
while len(dirStack) > 0:
@@ -57,54 +54,27 @@ def get_document_Types():
types.append(path_difference(rootPath,curDir))
return types
def getProjectInfos():
config = load_configs()
if config is None or len(config.items()) == 0:
return None
prjDir = None
for loader_type, loader_config in config.items():
if loader_config.get('enable', True):
loader_config = loader_config or []
config = FileLoaderConfig(**loader_config)
prjDir = config.data_dir
break
if prjDir is None:
return None
prjInfos = []
prjFlags = get_document_Types()
for prjFlag in prjFlags:
fileDir = os.path.join(config.data_dir,prjFlag.replace('_','\\'))
prjInfo = {}
prjInfo['flag'] = prjFlag
prjInfo['name'] = getProjectName(fileDir)
prjInfos.append(prjInfo)
return prjInfos
def get_documents(docType:str):
documents = []
config = load_configs()
if config is None or len(config.items()) == 0:
return documents
return documents
for loader_type, loader_config in config.items():
if loader_config.get('enable', True): # 检查 enable 字段
logger.info(
f"Loading documents from loader: {loader_type}, config: {loader_config}"
)
logger.info(
f"Loading documents from loader: {loader_type}, config: {loader_config}"
)
loader_config = loader_config or []
match loader_type:
case "file":
document = get_file_documents(FileLoaderConfig(**loader_config),docType)
case "web":
document = get_web_documents(WebLoaderConfig(**loader_config))
case "db":
document = get_db_documents(configs=[DBLoaderConfig(**cfg) for cfg in loader_config])
case _:
raise ValueError(f"Invalid loader type: {loader_type}")
documents.extend(document)
loader_config = loader_config or []
match loader_type:
case "file":
document = get_file_documents(FileLoaderConfig(**loader_config),docType)
case "web":
document = get_web_documents(WebLoaderConfig(**loader_config))
case "db":
document = get_db_documents(configs=[DBLoaderConfig(**cfg) for cfg in loader_config])
case _:
raise ValueError(f"Invalid loader type: {loader_type}")
documents.extend(document)
return documents
backend/app/engine/loaders/db.py
+67 -24
View File
@@ -1,15 +1,24 @@
import os
import logging
from typing import List
from typing import Any, List, Optional
from llama_index.core import SQLDatabase, Document
from llama_index.readers.database import DatabaseReader
from pydantic import BaseModel
from sqlalchemy import create_engine, text
from llama_index.core.readers.base import BaseReader
from llama_index.core.schema import Document
from llama_index.core.utilities.sql_wrapper import SQLDatabase
from sqlalchemy import text
from sqlalchemy.engine import Engine
from llama_index.core import SQLDatabase, Document
from llama_index.core.objects import SQLTableSchema, SQLTableNodeMapping
from llama_index.core.readers.base import BaseReader
from llama_index.readers.database import DatabaseReader
from pydantic import BaseModel, validator
from llama_index.core.indices.vector_store import VectorStoreIndex
from sqlalchemy import create_engine
logger = logging.getLogger(__name__)
class CustomDatabaseReader(DatabaseReader):
class CustomDatabaseReader(BaseReader):
"""Simple Database reader.
Concatenates each row into Document used by LlamaIndex.
@@ -82,20 +91,19 @@ class CustomDatabaseReader(DatabaseReader):
Returns:
List[Document]: A list of Document objects.
"""
dco_str = ""
dco_str = ""
with self.sql_database.engine.connect() as connection:
if query is None:
raise ValueError("A query parameter is necessary to filter the data")
else:
result = connection.execute(text(query))
dco_str += ", ".join(
dco_str = ", ".join(
[f"{entry}" for entry in result.keys()]
) + "\n"
)
for item in result.fetchall():
# Fetch each item
# fetch each item
record_str = ", ".join(
[f"{entry}" for col, entry in zip(result.keys(), item)]
)
@@ -109,36 +117,71 @@ class CustomDatabaseReader(DatabaseReader):
class DBLoaderConfig(BaseModel):
uri: str
queries: List[dict]
queries: List[str]
def get_db_documents(configs: List[DBLoaderConfig]) -> List[Document]:
def makeDescriptionByEngine(sql_database:SQLDatabase):
reader = DatabaseReader(sql_database)
table_names = sql_database.get_usable_table_names()
table_schema_objs = []
for table_name in table_names:
columns = sql_database.get_table_columns(table_name)
if len(columns) > 150:
continue
stats_txt = ""
if table_name == 'gongchengshuxing':
stats_txt = '该表中有以下属性:'
documents = reader.load_data(query='select name from gongchengshuxing')
for index in range(len(documents) if len(documents) < 30 else 30):
if index == 0:
continue
elif index > 1:
stats_txt += ','
stats_txt += documents[index].text.split(':')[1]
tbSchema = (SQLTableSchema(table_name=table_name, context_str=stats_txt))
table_schema_objs.append(tbSchema)
return table_schema_objs
def get_db_documents(configs: list[DBLoaderConfig]):
docs = []
if not configs or not configs[0].uri:
if len(configs) == 0 or configs[0].uri == "":
logger.warning(
f"Failed to load database, error message: uri is empty. Return as empty document list."
)
return docs
metadata = {
'file_type': 'application/booway.document.zj',
#'file_name':'',
'file_type':'application/booway.document.zj',
#'file_path':'',
#'file_size':'',
#'creation_date':'',
#'last_modified_date':'',
}
#from llama_index.readers.database import DatabaseReader
for entry in configs:
engine = create_engine(entry.uri)
sql_database = SQLDatabase(engine)
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
nodes = table_node_mapping.to_nodes(table_schema_objs)
for node in nodes:
node.metadata.update(metadata)
docs.extend(nodes)
queries = entry.queries or []
loader = CustomDatabaseReader(sql_database)
for query_dict in entry.queries:
query = query_dict.get("sql", "")
explanation = query_dict.get("explanation", "")
for query in queries:
logger.info(f"Loading data from database with query: {query}")
documents = loader.load_data(query=query)
# 添加解释到元数据中
for doc in documents:
doc.metadata["explanation"] = explanation
doc.metadata.update(metadata) # 更新或添加额外的元数据
docs.append(doc)
return docs
docs.extend(documents)
return docs
backend/app/engine/loaders/file.py
+2 -38
View File
@@ -6,9 +6,6 @@ from llama_index.core.readers.base import BaseReader
from llama_index.core.readers.json import JSONReader
from llama_parse import LlamaParse
from pydantic import BaseModel, validator
from app.engine.loaders.markdownReader import ChunkMarkdownReader
from app.engine.loaders.projectJson import ProjectJson
logger = logging.getLogger(__name__)
@@ -44,13 +41,9 @@ def llama_parse_extractor() -> Dict[str, LlamaParse]:
return {file_type: parser for file_type in SUPPORTED_FILE_TYPES}
def llama_local_extractor() -> Dict[str, BaseReader]:
parser = {
".json" : JSONReader(clean_json=False,levels_back=0),
".md" : ChunkMarkdownReader(),
}
return parser
return {"json" : JSONReader}
def get_file_documents(config: FileLoaderConfig,childPath: str):
def get_file_documents(config: FileLoaderConfig, childPath: str):
from llama_index.core.readers import SimpleDirectoryReader
try:
@@ -90,32 +83,3 @@ def get_file_documents(config: FileLoaderConfig,childPath: str):
else:
# Raise the error if it is not the case of empty data dir
raise e
def prjFileSuffix(dir:str):
entries = os.listdir(dir)
file_names = [entry for entry in entries if os.path.isfile(os.path.join(dir, entry))]
if len(file_names) > 0:
return os.path.splitext(file_names[0])[1]
return ''
def getProjectName(dir:str):
suffix = prjFileSuffix(dir)
if suffix== '.json':
prjJson = ProjectJson(dir)
prjJson.parse()
tb = prjJson.table('工程属性')
records = tb.records()
for record in records:
name = record.value('名称')
if name == '工程名称':
return record.value('值')
elif suffix == '.md':
md_files = [f for f in os.listdir(dir) if f.endswith('.md')]
for md_file in md_files:
prjPath = os.path.join(dir, md_file)
basename = os.path.splitext(md_file)[0]
if basename =='工程属性':
rd = ChunkMarkdownReader()
rd.load_data(prjPath)
return rd.findValue("名称=='工程名称'",'值')
return ''
backend/app/engine/loaders/markDown.py
-64
View File
@@ -1,64 +0,0 @@
from app.engine.loaders.projectJson import *
class MarkDown:
def __init__(self,table:JsonTable,path:str) -> None:
self._table = table
self._path = path
def build(self):
flds:Dict[str,Field] = self._table.fields()
records:List[Record] = self._table.records()
columns:list = []
colComments:list = []
ignores:List[str] = []
for name,fld in flds.items():
if name =='_id' or name =='nodeType' or name =='relTbId':
ignores.append(name)
continue
columns.append(fld.value('name'))
colComments.append(fld.value('alias'))
rowdatas = []
for record in records:
datas = []
for col in columns:
if col in ignores:
continue
txt:str = record.value(col)
datas.append(txt.replace('\n'," "))
rowdatas.append(datas)
content = self.convert(self._table.name(),self._table.comment(),columns,colComments,rowdatas)
with open(self._path, 'w',encoding='utf-8') as file:
file.write(content)
def convert(self,tableName:str,tableComment:str,columns:list,colComments:list,rowdatas:list):
strTitle = "# " + tableName + '\n'
if tableName!='':
strTitle+= f"备注：{tableComment}" + '\n'
for i in range(len(columns)):
strTitle+= f"- 字段名称：{columns[i]}" + '\n'
comment = colComments[i]
if comment!='':
strTitle+= f" - 备注：{comment}" + '\n'
markdown_table = "|"
# 添加列标题
markdown_table += "|".join(columns) + "|\n"
# 添加分隔行
markdown_table += "|" + "|".join(['---' for _ in columns]) + "|\n"
# 遍历每个数据行
for row in rowdatas:
# 添加数据行
markdown_table += "|" + "|".join(row) + "|\n"
return strTitle + "\n" + markdown_table
prjSon = ProjectJson('')
prjSon.parse()
tables = prjSon.tables()
for name,table in tables.items():
mdObj = MarkDown(table,f'')
mdObj.build()
backend/app/engine/loaders/markdownReader.py
-89
View File
@@ -1,89 +0,0 @@
from llama_index.readers.file.markdown import MarkdownReader
from typing import Any, Dict, List, Optional, Tuple
import re
from llama_index.core.utils import get_tokenizer
class ChunkMarkdownReader(MarkdownReader):
def __init__(
self,
*args: Any,
chunkSize:int = 2048,
**kwargs: Any,
) -> None:
self._chunkSize = chunkSize
self._tokenizer = get_tokenizer()
self._colheader = ''
self._rows = []
super().__init__(*args,**kwargs)
def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
markdown_tups: List[Tuple[Optional[str], str]] = []
lines = markdown_text.split("\n")
strTitle = ''
tokensNum:int = 0
current_lines = []
strheader:str = ''
headerSize:int = 0
for line in lines:
tokensNum += self._token_size(line)
if tokensNum > self._chunkSize and len(current_lines) > 0:
if len(markdown_tups) == 0:
markdown_tups.append((strTitle + strheader , "\n".join(current_lines)))
else:
markdown_tups.append((strheader , "\n".join(current_lines)))
tokensNum = headerSize
current_lines.clear()
current_lines.append(line)
if strTitle!='' and strheader!='':
self._rows.append(line)
if line == '\n' or line == '\r':
if tokensNum > self._chunkSize:
raise ValueError('标题Token数大于chunkSize大小')
strTitle = "\n".join(current_lines)
#headerSize = headerSize + self._token_size(strTitle)
current_lines.clear()
if line.startswith("|---"):
self._colheader = current_lines[0]
strheader = "\n".join(current_lines)
headerSize= headerSize + self._token_size(strheader)
current_lines.clear()
if len(current_lines) > 0:
if len(markdown_tups) == 0:
markdown_tups.append((strTitle + strheader , "\n".join(current_lines)))
else:
markdown_tups.append((strheader , "\n".join(current_lines)))
return [
(
key if key is None else re.sub(r"#", "", key).strip(),
re.sub(r"<.*?>", "", value),
)
for key, value in markdown_tups
]
def _token_size(self, text: str) -> int:
return len(self._tokenizer(text))
def findValue(self,expression:str,Field:str):
cols = self._colheader.split('|')
cols = [item for item in cols if item]
for row in self._rows:
rowtrs = row.split('|')
rowdatas = [item for item in rowtrs if item and (item!='\r' or item!='\n')]
if len(rowdatas) == 0:
continue
gData = {}
for cName,rValue in zip(cols,rowdatas):
gData[cName] = rValue
if eval(expression,gData):
return gData[Field]
return ''
backend/app/engine/loaders/projectJson.py
-94
View File
@@ -1,94 +0,0 @@
from typing import Dict,List,Any
import json,os
class Record:
def __init__(self,datas:Dict[str,Any]) -> None:
self._datas:Dict[str,Any] = datas
def value(self,key:str):
if key in self._datas:
return self._datas.get(key)
return ''
class Field:
def __init__(self,datas:Dict[str,Any]) -> None:
self._datas:Dict[str,Any] = datas
def value(self,key:str):
if key in self._datas:
return self._datas.get(key)
return ''
class JsonTable:
def __init__(self,filePth:str) -> None:
self._filePth = filePth
self._fields:Dict[str,Field] = {}
self._records:List[Record] = []
self._fileName = os.path.splitext(os.path.basename(filePth))[0]
self._name = ''
self._comment = ''
def parse(self):
with open(self._filePth, 'r',encoding='utf-8') as file:
jsObj = json.load(file)
data:dict = jsObj.get('table')
self._name = data.get('name')
self._comment = data.get('comment')
Jsfields = data.get('fields')
for jsfiled in Jsfields:
field = Field(jsfiled)
self._fields[field.value('name')] =field
JsRecords = data.get('records')
for jsRecord in JsRecords:
self._records.append(Record(jsRecord))
def records(self):
return self._records
def fields(self):
return self._fields
def name(self):
return self._fileName
def comment(self):
return self._comment
class ProjectJson:
def __init__(self,dir:str) -> None:
self._dir = dir
self._tables:Dict[str,JsonTable] = {}
def parse(self):
json_files = [f for f in os.listdir(self._dir) if f.endswith('.json')]
for json_file in json_files:
prjPath = os.path.join(self._dir, json_file)
tb = JsonTable(prjPath)
tb.parse()
basename = os.path.splitext(json_file)[0]
self._tables[basename] = tb
def table(self,tableName:str):
return self._tables[tableName]
def tables(self):
return self._tables
def getProjectName(dir:str):
result = dir.split('\\')
if len(result) > 0:
return result[-1]
return "未知工程名称"
prjJson = ProjectJson(dir)
prjJson.parse()
tb:JsonTable = prjJson.table('工程属性')
records = tb.records()
for record in records:
name = record.value('名称')
if name == '工程名称':
return record.value('值')
return ''
backend/app/engine/prompt.py
-119
View File
@@ -1,119 +0,0 @@
from llama_index.core import PromptTemplate
text_qa_template_str = (
"# 角色\n"
"你是一名擅长博微造价工程数据问答的专家，可以根据电力工程文件中的内容回答用户问题。\n"
"\n"
"# 任务描述：\n"
"请仔细阅读所给的文档片段，并根据其内容回答问题。\n"
"您需要判断文档的内容是否可以回答问题，不要强行回答。如果可以回答，答案必须严格遵循文档内容，即使与事实不符。\n"
"如果答案与事实不符，直接给出答案，不要做解释。\n"
"\n"
"# 回答规则：\n"
"- 请使用与文档材料相同的语言回答问题。\n"
"- 评估文档是否含有足够信息回答问题。无关时不要回答。\n"
"- 如果问题能被回答，你的回答必须严格遵循文档内容，即使与事实不符。一定不要做多余解释。\n"
"- 如果问题能被回答，直接引用文档的相关信息保证答案准确、完整，并追求简洁。\n"
"- 当文档中只有少量信息与问题相关时，重点关注这部分信息，这种情况下一定回答。\n"
"- 当文档中信息与问题无关时，请不要额外发散回答，只需要回答“我不知道这个问题的答案”。\n"
"\n"
"来自多个来源的文档片段如下,请充分理解以下参考资料内容，组织出满足用户提问的条理清晰的回复。\n"
"---------------------\n"
"{context_str}\n"
"---------------------\n"
"鉴于来自多个来源的文档片段而非先验知识，回答查询。\n"
"如果是表结构或者是数据库的相关内容，只用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
"Query: {query_str}\n"
"Answer: "
)
text_qa_template = PromptTemplate(text_qa_template_str)
refine_template_str = (
"这是原本的问题： {query_str}\n"
"我们已经提供了回答: {existing_answer}\n"
"现在我们有机会改进这个回答 "
"使用以下更多上下文（仅当有助于改进回答时使用）\n"
"你需要仔细的判断新的上下文的信息与原本问题必须一个字都不差，如果有一点差别，那就不能改变我现有的回答。\n"
"在判断回答是否正确的时候，你应该仔细对比新的上下文中包含的信息是否与原本的问题一字不差，如果一字不差，才能当作新的正确回答。\n"
"如果新的上下文对回答没有影响，或者原来的回答已经正确，不要在上次回答的后边再加上多余的补充信息，直接返回原本的回答。\n"
"判断一下如果原回答正确，且在新的上下文仍然包含正确的回答，请将新的回答与原回答一起返回。\n"
"------------\n"
"{context_msg}\n"
"------------\n"
"如果回答中已经包含有正确答案，不要返回多余的解释等信息，只返回正确答案\n"
"如果是表结构或者是数据库的相关内容，仅用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
"改进的回答: "
)
refine_template = PromptTemplate(refine_template_str)
summary_template_str = (
"# 角色\n"
"你是一名擅长博微造价工程数据问答的专家，可以根据电力工程文件中的内容回答用户问题。\n"
"\n"
"# 任务描述：\n"
"请仔细阅读所给的文档片段，并根据其内容回答问题。\n"
"您需要判断文档的内容是否可以回答问题，不要强行回答。如果可以回答，答案必须严格遵循文档内容，即使与事实不符。\n"
"如果答案与事实不符，直接给出答案，不要做解释。\n"
"\n"
"# 回答规则：\n"
"- 请使用与文档材料相同的语言回答问题。\n"
"- 评估文档是否含有足够信息回答问题。无关时不要回答。\n"
"- 如果问题能被回答，你的回答必须严格遵循文档内容，即使与事实不符。一定不要做多余解释。\n"
"- 如果问题能被回答，直接引用文档的相关信息保证答案准确、完整，并追求简洁。\n"
"- 当文档中只有少量信息与问题相关时，重点关注这部分信息，这种情况下一定回答。\n"
"- 当文档中信息与问题无关时，请不要额外发散回答，只需要回答“我不知道这个问题的答案”。\n"
"\n"
"来自多个来源的文档片段如下,请充分理解以下参考资料内容，组织出满足用户提问的条理清晰的回复。\n"
"---------------------\n"
"{context_str}\n"
"---------------------\n"
"鉴于来自多个来源的文档片段而非先验知识，回答查询。\n"
"如果是表结构或者是数据库的相关内容，只用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
"Query: {query_str}\n"
"Answer: "
)
summary_template = PromptTemplate(summary_template_str)
simple_template_str = (
"{query_str}"
)
simple_template = PromptTemplate(simple_template_str)
ReActChatFormatter_messages = (
"您的设计旨在帮助完成各种任务，从回答问题到提供其他类型分析的摘要。\n\n"
"##工具\n\n"
"你可以访问各种工具。你有责任按照你认为合适的顺序使用这些工具来完成当前的任务。\n"
"这可能需要将任务分解为子任务，并使用不同的工具来完成每个子任务。\n\n"
"你可以访问以下工具：\n"
"{tool_desc}\n\n\n"
"##输出格式\n\n"
"请用与问题相同的语言回答，并使用以下格式：\n\n"
"'''\n"
"Thought: 用户当前的语言是：(user's language)。我需要使用工具来帮助我回答问题。\n"
"Action: 如果使用工具,则为工具名称(one of {tool_names})。\n"
"Action Input: 输入给工具的内容，使用JSON格式表示kwargs（例如{{\"input\": \"hello world\", \"num_beams\": 5}}）\n"
"'''\n\n"
"请始终以Thought开始。\n\n"
"切勿用Markdown代码标记包围你的响应。如果需要，可以在响应中使用代码标记。\n\n"
"请为Action Input使用有效的JSON格式。不要这样做{{\'input\': \'hello world\', \'num_beams\': 5}}。\n\n"
"如果使用此格式，用户将以下面的格式进行回应：\n\n"
"'''\n"
"Observation: 工具响应\n"
"'''\n\n"
"你应该继续重复上述格式，直到你有足够的信息来回答问题而无需使用更多工具。此时，你必须使用以下两种格式之一进行回答：\n\n"
"'''\nThought: 我可以不用任何工具来回答。我将使用用户的语言来回答。\n"
"Answer: [你的答案（与用户问题相同的语言）]\n"
"'''\n\n"
"'''\n"
"Thought: 我无法使用提供的工具回答问题。\n"
"Answer: [你的答案（与用户问题相同的语言）]\n"
"'''\n\n##如果从工具中得到的回应是Empty Response，那么只需要回答“我不知道”，不需要额外回答别的内容。## 当前对话\n\n"
"以下是当前对话，由人类和助手的消息交替组成。\n"
)
summary_query_tool_messages = "适用于任何需要进行全面总结、概括的要求。"
query_engine_tool_messages = "适用于回答任何问题。"
tree_summary_query_engine_tool_messages = "在询问工程中单位的具体数值，例如用量，费率，合计，金额等的时候建议使用本工具。"
backend/app/engine/rerank/ollamRerank.py
-70
View File
@@ -1,70 +0,0 @@
from typing import Any, List, Optional
from llama_index.core.postprocessor import SentenceTransformerRerank
from llama_index.core.schema import MetadataMode, NodeWithScore, QueryBundle
from llama_index.core.callbacks import CBEventType, EventPayload
from llama_index.core.bridge.pydantic import PrivateAttr
class OllamaRerank(SentenceTransformerRerank):
_score_threshold: float = PrivateAttr()
def __init__(
self,
top_n: int = 2,
model: str = "cross-encoder/stsb-distilroberta-base",
device: Optional[str] = None,
keep_retrieval_score: Optional[bool] = False,
score_threshold:float = 0.3
):
self._score_threshold = score_threshold
super().__init__(top_n,model,device,keep_retrieval_score)
@classmethod
def class_name(cls) -> str:
return "OllamaRerank"
def _postprocess_nodes(
self,
nodes: List[NodeWithScore],
query_bundle: Optional[QueryBundle] = None,
) -> List[NodeWithScore]:
if query_bundle is None:
raise ValueError("Missing query bundle in extra info.")
if len(nodes) == 0:
return []
query_and_nodes = [
(
query_bundle.query_str,
node.node.get_content(metadata_mode=MetadataMode.EMBED),
)
for node in nodes
]
with self.callback_manager.event(
CBEventType.RERANKING,
payload={
EventPayload.NODES: nodes,
EventPayload.MODEL_NAME: self.model,
EventPayload.QUERY_STR: query_bundle.query_str,
EventPayload.TOP_K: self.top_n,
},
) as event:
scores = self._model.predict(query_and_nodes)
assert len(scores) == len(nodes)
for node, score in zip(nodes, scores):
if self.keep_retrieval_score:
node.node.metadata["retrieval_score"] = node.score
node.score = score
for i in range(len(nodes)-1,-1,-1):
node = nodes[i]
if node.score < self._score_threshold:
nodes.remove(node)
new_nodes = sorted(nodes, key=lambda x: -x.score if x.score else 0)[
: self.top_n
]
event.on_end(payload={EventPayload.NODES: new_nodes})
return new_nodes
backend/app/engine/response/treeSummResponse.py
-234
View File
@@ -1,234 +0,0 @@
from llama_index.core.response_synthesizers.tree_summarize import TreeSummarize
from typing import Any, Optional, Sequence,List
import asyncio
from llama_index.core.callbacks.base import CallbackManager
from llama_index.core.indices.prompt_helper import PromptHelper
from llama_index.core.prompts import BasePromptTemplate
from llama_index.core.service_context import ServiceContext
from llama_index.core.service_context_elements.llm_predictor import LLMPredictorType
from llama_index.core.types import BaseModel,RESPONSE_TEXT_TYPE
from llama_index.core.async_utils import run_async_tasks
from llama_index.core.utils import get_tokenizer
from llama_index.core.prompts.prompt_utils import get_empty_prompt_txt
class CustomTreeResponse(TreeSummarize):
def __init__(
self,
llm: Optional[LLMPredictorType] = None,
callback_manager: Optional[CallbackManager] = None,
prompt_helper: Optional[PromptHelper] = None,
summary_template: Optional[BasePromptTemplate] = None,
output_cls: Optional[BaseModel] = None,
streaming: bool = False,
use_async: bool = False,
verbose: bool = False,
service_context: Optional[ServiceContext] = None,
) -> None:
self._tokenizer = get_tokenizer()
super().__init__(llm,callback_manager,prompt_helper,summary_template,output_cls
,streaming,use_async,verbose,service_context)
async def aget_response(
self,
query_str: str,
text_chunks: Sequence[str],
**response_kwargs: Any,
) -> RESPONSE_TEXT_TYPE:
"""Get tree summarize response."""
summary_template = self._summary_template.partial_format(query_str=query_str)
text_chunks = self.repack(text_chunks=text_chunks)
if self._verbose:
print(f"{len(text_chunks)} text chunks after repacking")
# give final response if there is only one chunk
if len(text_chunks) == 1:
response: RESPONSE_TEXT_TYPE
if self._streaming:
response = await self._llm.astream(
summary_template, context_str=text_chunks[0], **response_kwargs
)
else:
if self._output_cls is None:
response = await self._llm.apredict(
summary_template,
context_str=text_chunks[0],
**response_kwargs,
)
else:
response = await self._llm.astructured_predict(
self._output_cls,
summary_template,
context_str=text_chunks[0],
**response_kwargs,
)
# return pydantic object if output_cls is specified
return response
else:
# summarize each chunk
if self._output_cls is None:
tasks = [
self._llm.apredict(
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
else:
tasks = [
self._llm.astructured_predict(
self._output_cls,
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
summary_responses = await asyncio.gather(*tasks)
if self._output_cls is not None:
summaries = [summary.json() for summary in summary_responses]
else:
summaries = summary_responses
# recursively summarize the summaries
return await self.aget_response(
query_str=query_str,
text_chunks=summaries,
**response_kwargs,
)
def get_response(
self,
query_str: str,
text_chunks: Sequence[str],
**response_kwargs: Any,
) -> RESPONSE_TEXT_TYPE:
"""Get tree summarize response."""
summary_template = self._summary_template.partial_format(query_str=query_str)
text_chunks = self.repack(text_chunks=text_chunks)
if self._verbose:
print(f"{len(text_chunks)} text chunks after repacking")
# give final response if there is only one chunk
if len(text_chunks) == 1:
response: RESPONSE_TEXT_TYPE
if self._streaming:
response = self._llm.stream(
summary_template, context_str=text_chunks[0], **response_kwargs
)
else:
if self._output_cls is None:
response = self._llm.predict(
summary_template,
context_str=text_chunks[0],
**response_kwargs,
)
else:
response = self._llm.structured_predict(
self._output_cls,
summary_template,
context_str=text_chunks[0],
**response_kwargs,
)
return response
else:
# summarize each chunk
if self._use_async:
if self._output_cls is None:
tasks = [
self._llm.apredict(
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
else:
tasks = [
self._llm.astructured_predict(
self._output_cls,
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
summary_responses = run_async_tasks(tasks)
if self._output_cls is not None:
summaries = [summary.json() for summary in summary_responses]
else:
summaries = summary_responses
else:
if self._output_cls is None:
summaries = [
self._llm.predict(
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
else:
summaries = [
self._llm.structured_predict(
self._output_cls,
summary_template,
context_str=text_chunk,
**response_kwargs,
)
for text_chunk in text_chunks
]
summaries = [summary.json() for summary in summaries]
# recursively summarize the summaries
return self.get_response(
query_str=query_str, text_chunks=summaries, **response_kwargs
)
def repack( self,text_chunks: Sequence[str],) ->List[str]:
prompt_str = get_empty_prompt_txt(self._summary_template)
num_prompt_tokens = self._token_size(prompt_str)
avaliableSize = self._get_available_context_size(num_prompt_tokens)
ava_chunks = []
sumSize = 0
results = []
for text_chunk in text_chunks:
one_chunk_size = self._token_size(text_chunk)
if one_chunk_size > avaliableSize:
raise ValueError("文本块大小大于可用上下文大小")
sumSize = sumSize + one_chunk_size
if sumSize > avaliableSize:
results.append(self._merge_chunks(ava_chunks))
ava_chunks.clear()
sumSize = 0
ava_chunks.append(text_chunk)
if len(ava_chunks) > 0:
results.append(self._merge_chunks(ava_chunks))
return results
def _get_available_context_size(self, num_prompt_tokens: int) -> int:
llm_metadata = self._llm.metadata
context_size_tokens = llm_metadata.context_window - num_prompt_tokens - llm_metadata.num_output
if context_size_tokens < 0:
raise ValueError(
f"Calculated available context size {context_size_tokens} was"
" not non-negative."
)
return context_size_tokens
def _token_size(self, text: str) -> int:
return len(self._tokenizer(text))
def _merge_chunks(self,ava_chunks:list):
return "\n\n".join([c.strip() for c in ava_chunks if c.strip()])
backend/app/engine/retriever/CHBM25Retriever.py
-133
View File
@@ -1,133 +0,0 @@
import json
import logging
import os
from typing import Any, Callable, Dict, List, Optional, cast
from llama_index.core.base.base_retriever import BaseRetriever
from llama_index.core.callbacks.base import CallbackManager
from llama_index.core.constants import DEFAULT_SIMILARITY_TOP_K
from llama_index.core.indices.vector_store.base import VectorStoreIndex
from llama_index.core.schema import BaseNode, IndexNode, NodeWithScore, QueryBundle
from llama_index.core.storage.docstore.types import BaseDocumentStore
from llama_index.core.vector_stores.utils import (
node_to_metadata_dict,
metadata_dict_to_node,
)
import bm25s
from app.engine.retriever.CHTokener import chTokenize
CHDEFAULT_PERSIST_ARGS = {"similarity_top_k": "similarity_top_k", "_verbose": "verbose"}
CHDEFAULT_PERSIST_FILENAME = "retriever.json"
class CHBM25Retriever(BaseRetriever):
def __init__(
self,
nodes: Optional[List[BaseNode]] = None,
existing_bm25: Optional[bm25s.BM25] = None,
similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K,
callback_manager: Optional[CallbackManager] = None,
objects: Optional[List[IndexNode]] = None,
object_map: Optional[dict] = None,
verbose: bool = False,
) -> None:
self.similarity_top_k = similarity_top_k
if existing_bm25 is not None:
self.bm25 = existing_bm25
self.corpus = existing_bm25.corpus
else:
from nltk.corpus import stopwords
if nodes is None:
raise ValueError("Please pass nodes or an existing BM25 object.")
self.corpus = [node_to_metadata_dict(node) for node in nodes]
corpus_tokens = chTokenize(
[node.get_content() for node in nodes],
show_progress=verbose,
)
self.bm25 = bm25s.BM25()
self.bm25.index(corpus_tokens, show_progress=verbose)
super().__init__(
callback_manager=callback_manager,
object_map=object_map,
objects=objects,
verbose=verbose,
)
@classmethod
def from_defaults(
cls,
index: Optional[VectorStoreIndex] = None,
nodes: Optional[List[BaseNode]] = None,
docstore: Optional[BaseDocumentStore] = None,
similarity_top_k: int = DEFAULT_SIMILARITY_TOP_K,
verbose: bool = False,
) -> "CHBM25Retriever":
if sum(bool(val) for val in [index, nodes, docstore]) != 1:
raise ValueError("Please pass exactly one of index, nodes, or docstore.")
if index is not None:
docstore = index.docstore
if docstore is not None:
nodes = cast(List[BaseNode], list(docstore.docs.values()))
assert (
nodes is not None
), "Please pass exactly one of index, nodes, or docstore."
return cls(
nodes=nodes,
similarity_top_k=similarity_top_k,
verbose=verbose,
)
def get_persist_args(self) -> Dict[str, Any]:
"""Get Persist Args Dict to Save."""
return {
CHDEFAULT_PERSIST_ARGS[key]: getattr(self, key)
for key in CHDEFAULT_PERSIST_ARGS
if hasattr(self, key)
}
def persist(self, path: str, **kwargs: Any) -> None:
"""Persist the retriever to a directory."""
self.bm25.save(path, corpus=self.corpus, **kwargs)
with open(os.path.join(path, CHDEFAULT_PERSIST_FILENAME), "w") as f:
json.dump(self.get_persist_args(), f, indent=2)
@classmethod
def from_persist_dir(cls, path: str, **kwargs: Any) -> "CHBM25Retriever":
"""Load the retriever from a directory."""
bm25 = bm25s.BM25.load(path, load_corpus=True, **kwargs)
with open(os.path.join(path, CHDEFAULT_PERSIST_FILENAME)) as f:
retriever_data = json.load(f)
return cls(existing_bm25=bm25, **retriever_data)
def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:
query = query_bundle.query_str
tokenized_query = chTokenize(
query,show_progress=self._verbose
)
indexes, scores = self.bm25.retrieve(
tokenized_query, k=self.similarity_top_k, show_progress=self._verbose
)
# batched, but only one query
indexes = indexes[0]
scores = scores[0]
nodes: List[NodeWithScore] = []
for idx, score in zip(indexes, scores):
# idx can be an int or a dict of the node
if isinstance(idx, dict):
node = metadata_dict_to_node(idx)
else:
node_dict = self.corpus[int(idx)]
node = metadata_dict_to_node(node_dict)
nodes.append(NodeWithScore(node=node, score=float(score)))
return nodes
backend/app/engine/retriever/CHTokener.py
-50
View File
@@ -1,50 +0,0 @@
import os
from typing import Any, Dict, List, Union, Callable, NamedTuple
from bm25s.tokenization import *
try:
from tqdm.auto import tqdm
except ImportError:
def tqdm(iterable, *args, **kwargs):
return iterable
import jieba
jiebapath = os.environ.get("JIEBA_DATA", "")
jieba.set_dictionary(os.path.join(jiebapath, 'dict.txt')) #设置字典
jieba.initialize() #初始化jeiba
def chinese_tokenizer(text: str) -> List[str]:
from nltk.corpus import stopwords
tokens = jieba.lcut(text)
return [token for token in tokens if token not in stopwords.words('chinese')]
def chTokenize(
texts,
show_progress: bool = True,
leave: bool = False,
) -> Union[List[List[str]], Tokenized]:
if isinstance(texts, str):
texts = [texts]
corpus_ids = []
token_to_index = {}
for text in tqdm(
texts, desc="Split strings", leave=leave, disable=not show_progress
):
splitted = chinese_tokenizer(text)
doc_ids = []
for token in splitted:
if token not in token_to_index:
token_to_index[token] = len(token_to_index)
token_id = token_to_index[token]
doc_ids.append(token_id)
corpus_ids.append(doc_ids)
return Tokenized(ids=corpus_ids, vocab=token_to_index)
backend/app/engine/retriever/HybridRetriever.py
-89
View File
@@ -1,89 +0,0 @@
import os
from typing import Optional, Any, Dict, List
from llama_index.core.base.base_retriever import BaseRetriever
from llama_index.core.schema import NodeWithScore, QueryBundle
from app.engine.retriever.CHBM25Retriever import CHBM25Retriever
class HybridRetriever(BaseRetriever):
def __init__(
self,
vector_index,
similarity_top_k: int = 2,
out_top_k: Optional[int] = None,
alpha: float = 0.5,
filters = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self._vector_index = vector_index
self._embed_model = vector_index._embed_model
self._out_top_k = out_top_k or similarity_top_k
self._vecRetriever = vector_index.as_retriever(
similarity_top_k=similarity_top_k,filters = filters
)
self._bm25Retriever = None
STORAGE_DIR = os.getenv("BM_RETRIEVER_PATH", "storage_bm")
if os.path.exists(STORAGE_DIR) and len(os.listdir(STORAGE_DIR)) > 0:
self._bm25Retriever = CHBM25Retriever.from_persist_dir(STORAGE_DIR)
else:
nodes = self._vector_index.vector_store.get_nodes(None)
similarity_top_k = min(len(nodes),similarity_top_k)
self._bm25Retriever = CHBM25Retriever.from_defaults(similarity_top_k=similarity_top_k,nodes=nodes)
self._bm25Retriever.persist(STORAGE_DIR)
self._alpha = alpha
def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:
vecNodes:List[NodeWithScore] = self._vecRetriever.retrieve(query_bundle.query_str)
bmNodes:List[NodeWithScore] = self._bm25Retriever.retrieve(query_bundle.query_str)
bmDic:Dict[str,NodeWithScore] = {}
for node in bmNodes:
bmDic[node.node_id] = node
vecScores = [node_with_score.score for node_with_score in vecNodes]
bmSores = [node_with_score.score for node_with_score in bmNodes]
vec_min_score = min(vecScores) if len(vecScores) > 0 else 0
vec_max_score = max(vecScores) if len(vecScores) > 0 else 0
bm_min_score = min(bmSores) if len(bmSores) > 0 else 0
bm_max_score = max(bmSores) if len(bmSores) > 0 else 0
result_tups = []
for i in range(len(vecNodes)):
node = vecNodes[i]
bmScore = 0.0
if node.node_id in bmDic:
bmScore = bmDic[node.node_id].score
bmDic.pop(node.node_id)
else:
bmScore = 0.0
bmScore = self.normal_score(bmScore,bm_min_score,bm_max_score)
vecScore = self.normal_score(node.score,vec_min_score,vec_max_score)
full_similarity = (self._alpha * vecScore) + (
(1 - self._alpha) * bmScore
)
result_tups.append((full_similarity, node))
for _,node in bmDic.items():
full_similarity = (1 - self._alpha) * node.score
result_tups.append((full_similarity, node))
result_tups = sorted(result_tups, key=lambda x: x[0], reverse=True)
for full_score, node in result_tups:
node.score = full_score
return [n for _, n in result_tups][:self._out_top_k]
def normal_score(self,score,min,max):
if min == max:
return 1.0 if score > 0 else 0.0
else:
return (score - min) / (max - min)
backend/app/engine/tools/__init__.py
+5 -4
View File
@@ -1,9 +1,10 @@
import importlib
import os
import yaml
from llama_index.core.tools.function_tool import FunctionTool
import json
import importlib
from cachetools import cached, LRUCache
from llama_index.core.tools.tool_spec.base import BaseToolSpec
from llama_index.core.tools.function_tool import FunctionTool
class ToolType:
@@ -45,7 +46,7 @@ class ToolFactory:
def from_env() -> list[FunctionTool]:
tools = []
if os.path.exists("config/tools.yaml"):
with open("config/tools.yaml", "r", encoding='UTF-8') as f:
with open("config/tools.yaml", "r") as f:
tool_configs = yaml.safe_load(f)
if tool_configs != None and len(tool_configs.items()) != 0:
for tool_type, config_entries in tool_configs.items():
backend/app/engine/vectordb.py
+5 -2
View File
@@ -7,10 +7,11 @@ qclient = None
def get_qdrant_vector_store(docType:str):
collection_name = docType
#collection_name = os.getenv("VECTOR_STORE_COLLECTION", "default")
vector_store_path = os.getenv("VECTOR_STORE_PATH")
host=os.getenv("VECTOR_STORE_HOST", "127.0.0.1"),
port=int(os.getenv("VECTOR_STORE_PORT", "6333")),
vector_store_path =os.path.join(vector_store_path,docType)
if not vector_store_path or not host:
raise ValueError(
@@ -34,8 +35,10 @@ def get_qdrant_vector_store(docType:str):
return vector_store
def get_chroma_vector_store(docType:str):
#collection_name = os.getenv("VECTOR_STORE_COLLECTION", "default")
collection_name = docType
vector_store_path =os.path.join(os.getenv("VECTOR_STORE_PATH"),docType)
vector_store_path = os.getenv("VECTOR_STORE_PATH")
vector_store_path =os.path.join(vector_store_path,docType)
# if VECTOR_STORE_PATH is set, use a local ChromaVectorStore from the path
# otherwise, use a remote ChromaVectorStore (ChromaDB Cloud is not supported yet)
if vector_store_path:
backend/app/settings.py
+178 -312
View File
@@ -1,335 +1,201 @@
import os
from typing import Dict
from abc import abstractmethod
from llama_index.core.constants import DEFAULT_TEMPERATURE
from llama_index.core.settings import Settings
from llama_index.embeddings.xinference import XinferenceEmbedding
from llama_index.llms.xinference import Xinference
#from llama_index.embeddings.xinference import XinferenceEmbedding
from llama_index.llms.xinference.base import DEFAULT_XINFERENCE_TEMP
from llama_index.postprocessor.xinference_rerank import XinferenceRerank
from app.engine.loaders import getProjectInfos
from app.api.routers.request.base import ProjectInfo
from modelProvide.customDashScope import CustomDashScope
from util.register import *
from llama_index.core.callbacks import CallbackManager
ModelPlateCategory = '模型平台'
def init_settings():
model_provider = os.getenv("MODEL_PROVIDER")
modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,model_provider)
if modelPaltCls is not None:
modelPalt:ModelPlatform = modelPaltCls()
Settings.llm = modelPalt.model()
else:
raise ValueError(f"Invalid model provider: {model_provider}")
embedding_provider = os.getenv("EMBEDDING_PROVIDER")
modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,embedding_provider)
if modelPalt is not None:
modelPalt:ModelPlatform = modelPaltCls()
Settings.embed_model = modelPalt.embedding()
else:
raise ValueError(f"Invalid embedding provider: {embedding_provider}")
match model_provider:
case "openai":
init_openai()
case "dashscope":
init_dashscope()
case "groq":
init_groq()
case "ollama":
init_ollama()
case "anthropic":
init_anthropic()
case "gemini":
init_gemini()
case "mistral":
init_mistral()
case "azure-openai":
init_azure_openai()
case "t-systems":
from .llmhub import init_llmhub
init_llmhub()
case _:
raise ValueError(f"Invalid model provider: {model_provider}")
Settings.llm.callback_manager = CallbackManager()
Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))
class ModelPlatform:
@abstractmethod
def model(self):
pass
@abstractmethod
def embedding(self):
pass
@abstractmethod
def rerank(self):
pass
@register(ModelPlateCategory,'ollama')
class OllamaPlatform(ModelPlatform):
def model(self):
from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
request_timeout = float(
os.getenv("OLLAMA_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
)
Settings.llm = Ollama(
base_url=base_url, model=os.getenv("MODEL"), request_timeout=request_timeout
)
pass
def embedding(self):
#from llama_index.embeddings.ollama import OllamaEmbedding
# base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
# Settings.embed_model = OllamaEmbedding(
# base_url=base_url,
# model_name=os.getenv("EMBEDDING_MODEL"),
# )
pass
def rerank(self):
from app.engine.rerank.ollamRerank import OllamaRerank
modelpath = os.getcwd() + os.getenv('RERANK_MODEL')
top_n = os.getenv('RERANK_TOP_N',5)
threshold = float(os.getenv('RERANK_THRESHOLD',0.3))
rerank = OllamaRerank(
model=modelpath,
top_n=top_n,
device="cpu",
score_threshold= threshold
)
return [rerank]
def init_ollama():
# from llama_index.embeddings.ollama import OllamaEmbedding
# from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
#
# base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
# request_timeout = float(
# os.getenv("OLLAMA_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
# )
# Settings.embed_model = OllamaEmbedding(
# base_url=base_url,
# model_name=os.getenv("EMBEDDING_MODEL"),
# )
# Settings.llm = Ollama(
# base_url=base_url, model=os.getenv("MODEL"), request_timeout=request_timeout
# )
pass
@register(ModelPlateCategory,'xinference')
class XinferencePlatform(ModelPlatform):
def model(self):
base_url = os.getenv("BASE_URL")
model = os.getenv("MODEL")
max_tokens = int(os.getenv("LLM_MAX_TOKENS")) if os.getenv("LLM_MAX_TOKENS") is not None else None
temperature = float(os.getenv("LLM_TEMPERATURE", DEFAULT_XINFERENCE_TEMP))
return Xinference(model, base_url, temperature, max_tokens)
def embedding(self):
base_url = os.getenv("BASE_URL")
embedding_base_url = os.getenv("EMBEDDING_BASE_URL")
embedding_base_url = embedding_base_url if embedding_base_url != None and embedding_base_url != "" else base_url
def init_openai():
from llama_index.core.constants import DEFAULT_TEMPERATURE
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.openai import OpenAI
embed_model_name = os.getenv("EMBEDDING_MODEL")
dimensions = os.getenv("EMBEDDING_DIM")
dimensions = int(dimensions) if dimensions is not None else None
return XinferenceEmbedding(embed_model_name, embedding_base_url)
def rerank(self):
rerank_model = os.getenv("RERANK_MODEL")
rerank_url = os.getenv("RERANK_BASE_URL")
rerank_top_n = os.getenv("RERANK_TOP_N")
rerank_threshold = os.getenv("RERANK_THRESHOLD")
postprocess = None
if rerank_model is not None:
postprocess = [XinferenceRerank(rerank_model, rerank_url, top_n=rerank_top_n, threshold=rerank_threshold)]
return postprocess
max_tokens = os.getenv("LLM_MAX_TOKENS")
config = {
"model": os.getenv("MODEL"),
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
"max_tokens": int(max_tokens) if max_tokens is not None else None,
}
Settings.llm = OpenAI(**config)
@register(ModelPlateCategory,'openai')
class OpenAIPlatform(ModelPlatform):
def model(self):
from llama_index.core.constants import DEFAULT_TEMPERATURE
from llama_index.llms.openai import OpenAI
dimensions = os.getenv("EMBEDDING_DIM")
config = {
"model": os.getenv("EMBEDDING_MODEL"),
"dimensions": int(dimensions) if dimensions is not None else None,
}
Settings.embed_model = OpenAIEmbedding(**config)
max_tokens = os.getenv("LLM_MAX_TOKENS")
config = {
"model": os.getenv("MODEL"),
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
"max_tokens": int(max_tokens) if max_tokens is not None else None,
}
return OpenAI(**config)
def embedding(self):
from llama_index.embeddings.openai import OpenAIEmbedding
dimensions = os.getenv("EMBEDDING_DIM")
config = {
"model": os.getenv("EMBEDDING_MODEL"),
"dimensions": int(dimensions) if dimensions is not None else None,
}
return OpenAIEmbedding(**config)
def rerank(self):
pass
def init_dashscope():
from llama_index.llms.dashscope import DashScope,DashScopeGenerationModels
from llama_index.embeddings.dashscope import DashScopeEmbedding,DashScopeBatchTextEmbeddingModels,DashScopeTextEmbeddingType,DashScopeTextEmbeddingModels
@register(ModelPlateCategory,'dashscope')
class DashscopePlatform(ModelPlatform):
def model(self):
apikey = os.getenv('DASHSCOPE_API_KEY')
modelName = os.getenv('MODEL')
return CustomDashScope(model_name=modelName,api_key = apikey)
max_tokens = os.getenv("LLM_MAX_TOKENS")
config = {
"model": os.getenv("MODEL"),
"temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
"max_tokens": int(max_tokens) if max_tokens is not None else None,
}
Settings.llm = llm = DashScope(model_name=DashScopeGenerationModels.QWEN_MAX)
def embedding(self):
from llama_index.embeddings.dashscope import DashScopeEmbedding,DashScopeTextEmbeddingType,DashScopeTextEmbeddingModels
api_key = os.getenv('DASHSCOPE_API_KEY')
modelName = os.getenv('EMBEDDING_MODEL')
return DashScopeEmbedding(model_name=modelName,
text_type=DashScopeTextEmbeddingType.TEXT_TYPE_QUERY,api_key = api_key)
def rerank(self):
pass
@register(ModelPlateCategory,'azure-openai')
class AzureOpenaiPlatform(ModelPlatform):
def model(self):
# from llama_index.core.constants import DEFAULT_TEMPERATURE
# from llama_index.llms.azure_openai import AzureOpenAI
#
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
# max_tokens = os.getenv("LLM_MAX_TOKENS")
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
# dimensions = os.getenv("EMBEDDING_DIM")
#
# azure_config = {
# "api_key": os.environ["AZURE_OPENAI_KEY"],
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
# or os.getenv("OPENAI_API_VERSION"),
# }
#
# return AzureOpenAI(
# model=os.getenv("MODEL"),
# max_tokens=int(max_tokens) if max_tokens is not None else None,
# temperature=float(temperature),
# deployment_name=llm_deployment,
# **azure_config,
# )
pass
def embedding(self):
# from llama_index.core.constants import DEFAULT_TEMPERATURE
# from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
#
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
# max_tokens = os.getenv("LLM_MAX_TOKENS")
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
# dimensions = os.getenv("EMBEDDING_DIM")
#
# azure_config = {
# "api_key": os.environ["AZURE_OPENAI_KEY"],
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
# or os.getenv("OPENAI_API_VERSION"),
# }
# return AzureOpenAIEmbedding(
# model=os.getenv("EMBEDDING_MODEL"),
# dimensions=int(dimensions) if dimensions is not None else None,
# deployment_name=embedding_deployment,
# **azure_config,
# )
pass
def rerank(self):
pass
@register(ModelPlateCategory,'fastembed')
class FastembedPlatform(ModelPlatform):
@abstractmethod
def model(self):
pass
@abstractmethod
def embedding(self):
# from llama_index.embeddings.fastembed import FastEmbedEmbedding
#
# embed_model_map: Dict[str, str] = {
# # Small and multilingual
# "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
# # Large and multilingual
# "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", # noqa: E501
# }
#
# # This will download the model automatically if it is not already downloaded
# Settings.embed_model = FastEmbedEmbedding(
# model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
# )
pass
@abstractmethod
def rerank(self):
pass
@register(ModelPlateCategory,'groq')
class GroqPlatform(ModelPlatform):
@abstractmethod
def model(self):
# from llama_index.llms.groq import Groq
#
# model_map: Dict[str, str] = {
# "llama3-8b": "llama3-8b-8192",
# "llama3-70b": "llama3-70b-8192",
# "mixtral-8x7b": "mixtral-8x7b-32768",
# }
#
# Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
# # Groq does not provide embeddings, so we use FastEmbed instead
# init_fastembed()
pass
@abstractmethod
def embedding(self):
pass
@abstractmethod
def rerank(self):
pass
@register(ModelPlateCategory,'anthropic')
class AnthropicPlatform(ModelPlatform):
def model(self):
# from llama_index.llms.anthropic import Anthropic
#
# model_map: Dict[str, str] = {
# "claude-3-opus": "claude-3-opus-20240229",
# "claude-3-sonnet": "claude-3-sonnet-20240229",
# "claude-3-haiku": "claude-3-haiku-20240307",
# "claude-2.1": "claude-2.1",
# "claude-instant-1.2": "claude-instant-1.2",
# }
#
# Settings.llm = Anthropic(model=model_map[os.getenv("MODEL")])
# # Anthropic does not provide embeddings, so we use FastEmbed instead
# init_fastembed()
pass
def embedding(self):
pass
def rerank(self):
pass
@register(ModelPlateCategory,'gemini')
class GeminiPlatform(ModelPlatform):
def model(self):
# from llama_index.llms.gemini import Gemini
# model_name = f"models/{os.getenv('MODEL')}"
# return Gemini(model=model_name)
pass
def embedding(self):
# from llama_index.embeddings.gemini import GeminiEmbedding
# embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"
# return GeminiEmbedding(model_name=embed_model_name)
pass
def rerank(self):
pass
@register(ModelPlateCategory,'mistral')
class MistralPlatform(ModelPlatform):
def model(self):
# from llama_index.llms.mistralai import MistralAI
# return MistralAI(model=os.getenv("MODEL"))
pass
def embedding(self):
# from llama_index.embeddings.mistralai import MistralAIEmbedding
# return MistralAIEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
pass
def rerank(self):
pass
def init_ProjectInfo():
prjObj = ProjectInfo()
prjInfos:list[tuple] = getProjectInfos()
for prjInfo in prjInfos:
prjObj.add(prjInfo['name'],prjInfo['flag'])
dimensions = os.getenv("EMBEDDING_DIM")
config = {
"model": os.getenv("EMBEDDING_MODEL"),
"dimensions": int(dimensions) if dimensions is not None else None,
}
Settings.embed_model = DashScopeEmbedding(model_name=DashScopeTextEmbeddingModels.TEXT_EMBEDDING_V2,
text_type=DashScopeTextEmbeddingType.TEXT_TYPE_QUERY)
def init_azure_openai():
# from llama_index.core.constants import DEFAULT_TEMPERATURE
# from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
# from llama_index.llms.azure_openai import AzureOpenAI
#
# llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
# embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
# max_tokens = os.getenv("LLM_MAX_TOKENS")
# temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
# dimensions = os.getenv("EMBEDDING_DIM")
#
# azure_config = {
# "api_key": os.environ["AZURE_OPENAI_KEY"],
# "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
# "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
# or os.getenv("OPENAI_API_VERSION"),
# }
#
# Settings.llm = AzureOpenAI(
# model=os.getenv("MODEL"),
# max_tokens=int(max_tokens) if max_tokens is not None else None,
# temperature=float(temperature),
# deployment_name=llm_deployment,
# **azure_config,
# )
#
# Settings.embed_model = AzureOpenAIEmbedding(
# model=os.getenv("EMBEDDING_MODEL"),
# dimensions=int(dimensions) if dimensions is not None else None,
# deployment_name=embedding_deployment,
# **azure_config,
# )
pass
def init_fastembed():
"""
Use Qdrant Fastembed as the local embedding provider.
"""
# from llama_index.embeddings.fastembed import FastEmbedEmbedding
#
# embed_model_map: Dict[str, str] = {
# # Small and multilingual
# "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
# # Large and multilingual
# "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", # noqa: E501
# }
#
# # This will download the model automatically if it is not already downloaded
# Settings.embed_model = FastEmbedEmbedding(
# model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
# )
pass
def init_groq():
# from llama_index.llms.groq import Groq
#
# model_map: Dict[str, str] = {
# "llama3-8b": "llama3-8b-8192",
# "llama3-70b": "llama3-70b-8192",
# "mixtral-8x7b": "mixtral-8x7b-32768",
# }
#
# Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
# # Groq does not provide embeddings, so we use FastEmbed instead
# init_fastembed()
pass
def init_anthropic():
# from llama_index.llms.anthropic import Anthropic
#
# model_map: Dict[str, str] = {
# "claude-3-opus": "claude-3-opus-20240229",
# "claude-3-sonnet": "claude-3-sonnet-20240229",
# "claude-3-haiku": "claude-3-haiku-20240307",
# "claude-2.1": "claude-2.1",
# "claude-instant-1.2": "claude-instant-1.2",
# }
#
# Settings.llm = Anthropic(model=model_map[os.getenv("MODEL")])
# # Anthropic does not provide embeddings, so we use FastEmbed instead
# init_fastembed()
pass
def init_gemini():
# from llama_index.embeddings.gemini import GeminiEmbedding
# from llama_index.llms.gemini import Gemini
#
# model_name = f"models/{os.getenv('MODEL')}"
# embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"
#
# Settings.llm = Gemini(model=model_name)
# Settings.embed_model = GeminiEmbedding(model_name=embed_model_name)
pass
def init_mistral():
# from llama_index.embeddings.mistralai import MistralAIEmbedding
# from llama_index.llms.mistralai import MistralAI
#
# Settings.llm = MistralAI(model=os.getenv("MODEL"))
# Settings.embed_model = MistralAIEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
pass
backend/config/loaders.yaml
+9 -37
View File
@@ -1,48 +1,20 @@
file:
enable: true # 添加 enable 字段
# use_llama_parse: Use LlamaParse if `true`. Needs a `LLAMA_CLOUD_API_KEY` from https://cloud.llamaindex.ai set as environment variable
use_llama_parse: false
#db:
db:
# The configuration for the database loader, only supports MySQL and PostgreSQL databases for now.
# uri: The URI for the database. E.g.: mysql+pymysql://user:password@localhost:3306/db or postgresql+psycopg2://user:password@localhost:5432/db
# query: The query to fetch data from the database. E.g.: SELECT * FROM table
#- uri: mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1
#enable: false # 添加 enable 字段
#queries:
#- sql: select * from ProjectProperties;
#explanation: "工程属性表数据，层级关系包含在博微电力造价工程文件格式_ProjectProperties.json文件中。"
- uri: mysql+pymysql://zjinfo1:Dy2Bcr53Hm5xRkba@110.42.234.166:3306/zjinfo1
#- uri: mysql+pymysql://zjinfo:Y6EAjEEdSYmskA8B@110.42.234.166:3306/zjinfo
# - uri: mysql+pymysql://zjinfo2:GSKcziSdBixDXwcd@110.42.234.166:3306/zjinfo2
queries:
- select * from ProjectProperties limit 30;
- select Name, Code, Amount, Amount_Total from TotalCalculateTable
- select SerialNumber, Name, Quantity, Rate, Sum_Price from ProjectDivision where Level = 1 limit 30;
- select Name, Code, Rate, Amount from OtherFee
#- sql: select Id, ParentId, Level, Name, Code, Amount, Amount_Total from TotalCalculateTable;
#explanation: "总算表数据，层级关系包含在博微电力造价工程文件格式_TotalCalculateTable.json文件中。"
#- sql: select Id, ParentId, Level, SerialNumber, Name, Quantity, Rate, Sum_Price from ProjectDivision where ProfessionalType = '线路';
#explanation: "专业类型为线路的项目划分表数据，层级关系包含在博微电力造价工程文件格式_ProjectDivision.json文件中。"
#- sql: select Id, ParentId, Level, SerialNumber, Name, Quantity, Rate, Sum_Price from ProjectDivision where ProfessionalType = '余物清理';
#explanation: "专业类型为余物清理的项目划分表数据，层级关系包含在博微电力造价工程文件格式_ProjectDivision.json文件中。"
#- sql: select Id, ParentId, Level, SerialNumber, Name, Quantity, Rate, Sum_Price from ProjectDivision where ProfessionalType = '拆除线路';
#explanation: "专业类型为拆除线路的项目划分表数据，层级关系包含在博微电力造价工程文件格式_ProjectDivision.json文件中。"
#- sql: select Id, ParentId, Level, Name, Code, Rate, Amount from OtherFee;
#explanation: "其他费用表数据，层级关系包含在博微电力造价工程文件格式_OtherFee.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '线路取费表'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '线路取费表(调试工程)aa'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '大型土石方取费表'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '线路取费表（余物清理）'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '线路取费表（余物清理）(1)'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from FeeCollectionTable where FeeCollection_Table_Name = '线路取费表（拆除）'
#explanation: "取费表名称为线路取费表的取费表数据，层级关系包含在博微电力造价工程文件格式_FeeCollectionTable.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from ProjectQuantities where Professional_Type = '线路'
#explanation: "专业类型为线路的工程量表数据，层级关系包含在博微电力造价工程文件格式_ProjectQuantities.json文件中"
#- sql: select Name, Code, Calculation_Formula, Rate, from ProjectQuantities where Professional_Type = '余物清理'
#explanation: "专业类型为余物清理的工程量表数据，层级关系包含在博微电力造价工程文件格式_ProjectQuantities.json文件中"
#web:
# driver_arguments:
# # The arguments to pass to the webdriver. E.g.: add --headless to run in headless mode
backend/data/projectstruct/博微电力造价工程文件格式_FeeCollectionTable.json
+61
View File
@@ -0,0 +1,61 @@
{
"Table": [
{
"name": "FeeCollectionTable",
"alias": "",
"comment": "取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。",
"fileds": [
{
"name": "FeeCollectionTableName",
"alias": "取费名，取费名称，取费表名称",
"comment": "取费表名称"
},
{
"name": "Name",
"alias": "费用名，名称，项目名",
"comment": "费用名称，项目名称"
},
{
"name": "SerialNumber",
"alias": "费用序号，序号，序列号",
"comment": "费用表序号"
},
{
"name": "Code",
"alias": "代码，代号，编号",
"comment": "费用代码"
},
{
"name": "CalculationFormula",
"alias": "表达式，公式，计算式",
"comment": "取费基数"
},
{
"name": "Rate",
"alias": "费用利率，费率",
"comment": "取费费率"
},
{
"name": "Remarks",
"alias": "备注，说明",
"comment": "费用项备注说明"
},
{
"name": "Major",
"alias": "专业",
"comment": "取费表专业"
},
{
"name": "Type",
"alias": "类型，取费类型",
"comment": "取费表类型"
},
{
"name": "Path",
"alias": "费用全路径，路径",
"comment": "费用项层级全路径"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_OtherFee.json
+76
View File
@@ -0,0 +1,76 @@
{
"Table": [
{
"name": "OtherFee",
"alias": "",
"comment": "其他费用表被称为“工程费用中其他费用明细”。其他费用是指为完成工程项目建设所必需的，但不属于建筑工程费、安装工程费、设备购置费、基本预备费的其他相关费用。包括建设场地征用及清理费、项目建设管理费、项目建设技术服务费、生产准备费、大件运输措施费、专业爆破服务费等。查询示例: SELECT Rate FROM OtherFee WHERE Name = 'findname'。",
"fileds": [
{
"name": "Id",
"alias": "id，项目id，费用id",
"comment": "费用项目id"
},
{
"name": "ParentId",
"alias": "父id，父级id",
"comment": "费用项目父级id"
},
{
"name": "Level",
"alias": "层级，层编号，层号",
"comment": "层级编号，从1开始"
},
{
"name": "Name",
"alias": "费用名，名称，项目名",
"comment": "费用名称，项目名称"
},
{
"name": "SerialNumber",
"alias": "序号，序列号",
"comment": "费用表序号"
},
{
"name": "Code",
"alias": "代码，代号，编号",
"comment": "费用代码"
},
{
"name": "CalculationFormula",
"alias": "表达式，公式，计算式",
"comment": "取费基数"
},
{
"name": "Rate",
"alias": "费用利率，费率",
"comment": "取费费率"
},
{
"name": "Amount",
"alias": "金额，价格",
"comment": "金额、合计、费用，\n单位为元"
},
{
"name": "Remarks",
"alias": "备注，说明",
"comment": "费用项备注说明"
},
{
"name": "Compilation_Basis",
"alias": "编制依据，编制来源",
"comment": "费用项编制依据"
},
{
"name": "WBS_Code",
"alias": "WBS编号，WBS编码",
"comment": "费用项WBS编码"
},
{
"name": "Path",
"alias": "费用全路径，路径",
"comment": "费用项层级全路径"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_ProjectDivision.json
+126
View File
@@ -0,0 +1,126 @@
{
"Table": [
{
"name": "ProjectDivision",
"alias": "",
"comment": "项目划分表是用于存储工程项目划分树状数据。内部包含安装工程项目划分，建筑工程项目划分，线路项目划分，工程分部分项。查询示例: SELECT Sum_Price FROM ProjectDivision WHERE Name = 'findname'。",
"fileds": [
{
"name": "Id",
"alias": "id，项目id，费用id",
"comment": "项目划分id"
},
{
"name": "ParentId",
"alias": "父id，父级id",
"comment": "项目划分父级id"
},
{
"name": "Level",
"alias": "层级，层编号，层号",
"comment": "层级编号，从1开始"
},
{
"name": "Quantity",
"alias": "个数，数量，数目",
"comment": "项目划分数量"
},
{
"name": "SerialNumber",
"alias": "项目序号，序号，序列号",
"comment": "项目划分序号"
},
{
"name": "Name",
"alias": "项目名，名称",
"comment": "项目名称"
},
{
"name": "Encoding",
"alias": "译码，编码",
"comment": "项目划分编码"
},
{
"name": "Sum_Price",
"alias": "合计，合价",
"comment": "项目划分合价，分部分项费用"
},
{
"name": "FeeCollectionTableName",
"alias": "取费表",
"comment": "项目划分的取费表，此项目划分选用的取费表"
},
{
"name": "Remarks",
"alias": "备注，说明",
"comment": "备注"
},
{
"name": "WBS_Code",
"alias": "WBS编号，WBS编码",
"comment": "WBS编码"
},
{
"name": "Manual_Adjustment_Coefficient",
"alias": "人工调差系数",
"comment": "此项目划分下人工调差系数"
},
{
"name": "Material_Adjustment_Coefficient",
"alias": "材料调差系数",
"comment": "此项目划分下材料调差系数"
},
{
"name": "Mechanical_Adjustment_Coefficient",
"alias": "机械调差系数",
"comment": "此项目划分下机械调差系数"
},
{
"name": "Demolition_Manual_Adjustment_Coefficient",
"alias": "拆除人工调差系数",
"comment": "此项目划分下拆除人工调差系数"
},
{
"name": "Demolition_Material_Adjustment_Coefficient",
"alias": "拆除材料调差系数",
"comment": "此项目划分下拆除材料调差系数"
},
{
"name": "Demolition_Mechanical_Adjustment_Coefficient",
"alias": "拆除机械调差系数",
"comment": "此项目划分下拆除机械调差系数"
},
{
"name": "ProfessionalType",
"alias": "专业类型",
"comment": "专业类型，字段值有变电安装、变电建筑、线路等。变电安装等于安装工程，变电建筑等于建筑工程，线路等于安装工程。"
},
{
"name": "Unit",
"alias": "单位",
"comment": "项目划分单位"
},
{
"name": "CalculationFormula",
"alias": "表达式，公式，计算式",
"comment": "项目划分计算式"
},
{
"name": "Rate",
"alias": "费用利率，费率",
"comment": "项目划分费率"
},
{
"name": "Code",
"alias": "代码，代号，编号",
"comment": "项目划分代码"
},
{
"name": "Path",
"alias": "路径，项目全路径",
"comment": "项目划分层级全路径"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_ProjectDivisions_CostPreview.json
+201
View File
@@ -0,0 +1,201 @@
{
"Table": [
{
"name": "ProjectDivisions_CostPreview",
"alias": "",
"comment": "项目划分_费用预览表也被称为“项目划分费用预览”、“项目划分取费费用”。其中包含项目划分合价、直接费、间接费、利润、税金、主材费等。查询示例: SELECT Total FROM ProjectDivisions_CostPreview WHERE Id = '15'。",
"fileds": [
{
"name": "Id",
"alias": "id，项目id",
"comment": "项目划分id"
},
{
"name": "ParentId",
"alias": "父id，父级id",
"comment": "项目划分父级id"
},
{
"name": "Level",
"alias": "层级，层编号，层号",
"comment": "层级编号，从1开始"
},
{
"name": "ProfessionalType",
"alias": "专业类型",
"comment": "专业类型，字段值有变电安装、变电建筑、线路等。变电安装等于安装工程，变电建筑等于建筑工程，线路等于安装工程。"
},
{
"name": "FeeCollectionTableName",
"alias": "取费表",
"comment": "项目划分的取费表，此项目划分选用的取费表"
},
{
"name": "Direct_Cost",
"alias": "直接费",
"comment": "直接费是指施工过程中直接耗用于建筑、安装工程产品的各项费用的总和。包括直接工程费和措施费。"
},
{
"name": "Direct_Project_Cost",
"alias": "直接工程费",
"comment": "直接工程费是指按照正常的施工条件，在施工过程中耗费的构成工程实体的各项费用。包括人工费、材料费和施工机械使用费。"
},
{
"name": "Quota_Direct_Cost",
"alias": "定额直接费",
"comment": "定额直接费，包含人工费、材料费中已进入定额基价的消耗性材料费和施工机械使用费。"
},
{
"name": "Labor_Cost",
"alias": "人工费",
"comment": "人工费是指支付给直接从事建筑安装工程施工作业的生产人员的各项费用。包括基本工资、工资性补贴、辅助工资、职工福利费、生产人员劳动保护费。"
},
{
"name": "Material_Cost",
"alias": "材料费",
"comment": "材料费是指施工过程中一次性消耗材料及摊销材料的费用。指已进入定额基价的消耗性材料费。"
},
{
"name": "Construction_Machinery_Cost",
"alias": "施工机械使用费",
"comment": "施工机械使用费是指施工机械作业所发生的机械使用费以及机械的现场安拆费和场外运费。包括折旧费、检修费、维护费、安装及拆卸费、场外运费、操作人员人工费、燃料动力费、其他费等。"
},
{
"name": "Installation_Material_Cost",
"alias": "装置性材料费",
"comment": "装置性材料费是指建设工程中构成工艺系统实体的工艺性材料，也称主要材料费。装置性材料通常在概算或预算定额中未计价，也称未计价材料，也称主材。"
},
{
"name": "A_Supply_Installation_Material_Cost",
"alias": "甲供装置性材料费",
"comment": "供货方为甲供的装置性材料费。"
},
{
"name": "B_Supply_Installation_Material_Cost",
"alias": "乙供装置性材料费",
"comment": "供货方为乙供的装置性材料费。"
},
{
"name": "Measure_Cost",
"alias": "措施费",
"comment": "措施费是指为完成工程项目施工而进行施工准备、克服自然条件的不利影响和辅助施工所发生的不构成工程实体的各项费用。包括冬雨季施工增加费、夜间施工增加费、施工工具用具使用费、特殊地区施工增加费、临时设施费、施工机构迁移费、安全文明施工费。"
},
{
"name": "WinterRainySeasons_Additional_Construction_Cost",
"alias": "冬雨季施工增加费",
"comment": "冬雨季施工增加费是指按照合理的工期要求，建筑、安装工程必须在冬季、雨季期间连续施工而需要增加的费用。"
},
{
"name": "Night_Additional_Construction_Cost",
"alias": "夜间施工增加费",
"comment": "夜间施工增加费是指按照规程要求，工程必须在夜间连续施工所发生的夜班补助、夜间施工降效、夜间施工照明设备摊销及照明用电等费用。"
},
{
"name": "Construction_Tool_Usage_Cost",
"alias": "施工工具用具使用费",
"comment": "施工工具用具使用费是指施工企业的生产、检验、试验部门使用的不属于固定资产的工具用具和仪器仪表的购置、摊销和维护费用。"
},
{
"name": "Special_Areas_Additional_Construction_Cost",
"alias": "特殊地区施工增加费",
"comment": "特殊地区施工增加费是指在高海拔、酷热、严寒等地区施工:因特殊自然条件影响而需额外增加的施工费用。"
},
{
"name": "Temporary_Facility_Cost",
"alias": "临时设施费",
"comment": "临时设施费是指施工企业为满足现场正常生产、生活需要在现场必须搭设的生产、生活用临时建筑物、构筑物和其他临时设施所发生的费用，以及维修、拆除、折旧及摊销费，或临时设施的租赁费等。"
},
{
"name": "Construction_Organization_Relocation_Cost",
"alias": "施工机构迁移费",
"comment": "施工机构迁移费是指施工企业派遣施工队伍到所承建工程现场所发生的搬迁费用。包括职工调遣差旅费和调遣期间的工资，以及办公设备、工器具、家具、材料用品和施工机械等的搬迁费用。"
},
{
"name": "Safe_Civilized_Construction_Cost",
"alias": "安全文明施工费",
"comment": "安全文明施工费，包括安全生产费、文明施工费、环境保护费。"
},
{
"name": "Indirect_Cost",
"alias": "间接费",
"comment": "间接费是指建筑安装工程的施工过程中，为全工程项目服务而不直接消耗在特定产品对象上的费用。包括规费、企业管理费和施工企业配合调试费。"
},
{
"name": "Regulatory_Cost",
"alias": "规费",
"comment": "规费是指按照国家行政主管部门或省级政府和省级有关权力部门规定必须缴纳并计入建筑安装工程造价的费用。包括社会保险费和住房公积金。"
},
{
"name": "Social_Insurance_Premiums",
"alias": "社会保险费",
"comment": "社会保险费包括养老保险费、失业保险费、医疗保险费、生育保险费和工伤保险费。"
},
{
"name": "Housing_Provident_Fund",
"alias": "住房公积金",
"comment": "住房公积金是指企业按照规定标准为职工缴纳的住房公积金。"
},
{
"name": "Enterprise_Management_Cost",
"alias": "企业管理费",
"comment": "企业管理费是指建筑安装施工企业为组织施工生产和经营管理所发生的费用。"
},
{
"name": "Construction_Enterprise_Cooperation_Debugging_Cost",
"alias": "施工企业配合调试费",
"comment": "施工企业配合调试费是指在工程整套启动试运阶段，施工企业安装专业配合调试所发生的费用。"
},
{
"name": "Profit",
"alias": "利润",
"comment": "利润是指施工企业完成所承包工程获得的盈利。"
},
{
"name": "Taxes",
"alias": "税金",
"comment": "税金是指按照国家税法规定应计入建筑安装工程造价内的销项税额。"
},
{
"name": "Equipment_Cost",
"alias": "设备费",
"comment": "设备购置费是指为项目建设而购置或自制各种设备，并将设备运至施工现场指定位置所支出的费用。包括设备费和设备运杂费。"
},
{
"name": "B_Supply_Equipment_Excluding_Tax_Price",
"alias": "乙供设备不含税价",
"comment": "设备费中，供货方为乙供设备，不含税价"
},
{
"name": "A_Supply_Equipment_Tax_Price",
"alias": "甲供设备含税价",
"comment": "设备费中，供货方为甲供设备，含税价"
},
{
"name": "Installation_Cost",
"alias": "安装费",
"comment": "安装费包含定额直接费、措施费、间接费、利润、税金和一笔性费用。"
},
{
"name": "Main_Material_Cost",
"alias": "主材费",
"comment": "主材费指装置性材料费"
},
{
"name": "Total",
"alias": "总体费用，总计，总价，总的费用",
"comment": "总计包含安装费、主材费、设备费。"
},
{
"name": "Sum",
"alias": "合计，合价",
"comment": "项目划分合价，分部分项费用，项目划分费用。合计包含安装费和主材费。"
},
{
"name": "Path",
"alias": "路径，项目划分全路径",
"comment": "项目划分层级全路径"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_ProjectProperties.json
+31
View File
@@ -0,0 +1,31 @@
{
"Table": [
{
"name": "ProjectProperties",
"alias": "",
"comment": "工程属性表是用于存储整个工程的重要属性，访问该表都是为了通过属性名查找属性值。通常属性值有工程信息、工程属性、技经参数，表中包含工程总投资、工程总费用，工程主要费用，工程技经参数等。查询示例: SELECT Value FROM ProjectProperties WHERE Name = 'findname'。",
"fileds": [
{
"name": "Name\n",
"alias": "属性名，属性名称，属性",
"comment": "属性的唯一标识"
},
{
"name": "Value",
"alias": "属性值",
"comment": "属性对应的实际值"
},
{
"name": "Type",
"alias": "类型，属性类型",
"comment": "属性变量的类型"
},
{
"name": "Unit",
"alias": "单位",
"comment": "单位"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_ProjectQuantities.json
+281
View File
@@ -0,0 +1,281 @@
{
"Table": [
{
"name": "ProjectQuantities",
"alias": "",
"comment": "工程量表是项目划分下工程量，包含定额、主材、设备、一笔性费用。查询示例: SELECT BudgetPrice FROM ProjectQuantities WHERE Name = 'findname'。",
"fileds": [
{
"name": "Id",
"alias": "id",
"comment": "消耗量id，工程量id"
},
{
"name": "ParentId",
"alias": "父id，父级id",
"comment": "父级id"
},
{
"name": "ProjectDivisionId",
"alias": "项目划分id，项目id",
"comment": "父级项目划分id"
},
{
"name": "Quantity",
"alias": "个数，数量，数目",
"comment": "数量，消耗量数量，工程量数量，主材数量，定额数量，设备数量，项目划分单位"
},
{
"name": "FeatureSegment",
"alias": "特征段",
"comment": "线路特征段"
},
{
"name": "ParentQuantity",
"alias": "父级个数，父级数量",
"comment": "父级id的数量"
},
{
"name": "Name",
"alias": "名称",
"comment": "项目名称，工程量名称，消耗量名称，主材名称，定额名称，设备名称，材料名称"
},
{
"name": "Encoding",
"alias": "译码，编码",
"comment": "编码，定额编码，主材编码，设备编码"
},
{
"name": "SpecificationModel",
"alias": "规格型号",
"comment": "规格型号，主材规格型号，设备规格型号"
},
{
"name": "Unit",
"alias": "单位",
"comment": "单位，主材单位，定额单位，设备单位，项目划分单位"
},
{
"name": "BasePrice",
"alias": "基价",
"comment": "定额基价"
},
{
"name": "LaborCost",
"alias": "人工费",
"comment": "定额人工费"
},
{
"name": "MaterialCost",
"alias": "材料费",
"comment": "定额材料费"
},
{
"name": "MachineryCost",
"alias": "机械费",
"comment": "定额机械费"
},
{
"name": "QuotaCoefficient",
"alias": "定额系数",
"comment": "定额系数"
},
{
"name": "LaborCoefficient",
"alias": "人工系数",
"comment": "定额人工系数"
},
{
"name": "MaterialCoefficient",
"alias": "材料系数",
"comment": "定额材料系数"
},
{
"name": "MechanicalCoefficient",
"alias": "机械系数",
"comment": "定额机械系数"
},
{
"name": "ExpenseType",
"alias": "费用类型",
"comment": "费用类型，取值为取费、不取费"
},
{
"name": "BudgetPrice",
"alias": "预算价",
"comment": "预算价"
},
{
"name": "MarketPrice",
"alias": "市场价",
"comment": "间接费是指建筑安装工程的施工过程中，为全工程项目服务而不直接消耗在特定产品对象上的费用。包括规费、企业管理费和施工企业配合调试费。"
},
{
"name": "Supplier",
"alias": "供货方",
"comment": "供货方，设备供货方，主材供货方，取值为甲供、乙供"
},
{
"name": "Type",
"alias": "类型",
"comment": "工程量类型，取值定额、主材、设备、一笔性费用"
},
{
"name": "QuotaRange",
"alias": "定额范围",
"comment": "定额范围，取值概算、预算"
},
{
"name": "A_Supply_Material_Cost_Excluding_Tax",
"alias": "甲供材料费不含税",
"comment": "甲供材料费不含税"
},
{
"name": "A_Supply_Material_Cost_Including_Tax",
"alias": "甲供材料费含税",
"comment": "甲供材料费含税"
},
{
"name": "B_Supply_Material_Cost_Excluding_Tax",
"alias": "乙供材料费不含税",
"comment": "乙供材料费不含税"
},
{
"name": "B_Supply_Material_Cost_Including_Tax",
"alias": "乙供材料费含税",
"comment": "乙供材料费含税"
},
{
"name": "ScaffoldCalculation",
"alias": "脚手架计取",
"comment": "脚手架计取，取值计取、不计取"
},
{
"name": "Remarks",
"alias": "备注，说明",
"comment": "备注，说明"
},
{
"name": "FeeCollectionTableName",
"alias": "取费表",
"comment": "项目划分的取费表，工程量的取费表"
},
{
"name": "Quota_Section_Name",
"alias": "定额章节名称",
"comment": "定额章节名称"
},
{
"name": "ProfessionalType",
"alias": "专业类型",
"comment": "专业类型，字段值有变电安装、变电建筑、线路等。变电安装等于安装工程，变电建筑等于建筑工程，线路等于安装工程。"
},
{
"name": "split",
"alias": "拆分",
"comment": "是否为拆分材料，取值1为拆分，取值0为不拆分"
},
{
"name": "Loss",
"alias": "损耗",
"comment": "损耗率，主材损耗率"
},
{
"name": "SingleWeight",
"alias": "单重",
"comment": "单重，主材单重"
},
{
"name": "LineWeight",
"alias": "线重",
"comment": "线重，主材线重"
},
{
"name": "SupervisedMaterials",
"alias": "监造物料",
"comment": "监造物料，取值1为监造物料，取值0为非监造物料"
},
{
"name": "EquipmentMaterials",
"alias": "设备性材料",
"comment": "设备性材料，取值1为设备性材料，取值0为主材"
},
{
"name": "GrossWeight",
"alias": "毛重",
"comment": "毛重，主材毛重"
},
{
"name": "TransportationType",
"alias": "运输类型",
"comment": "运输类型，主材运输类型"
},
{
"name": "TransportationMiscellaneous",
"alias": "运杂费率",
"comment": "运杂费率，设备运杂费率"
},
{
"name": "EquipmentType",
"alias": "设备类型",
"comment": "设备类型，取值为主要设备、普通设备"
},
{
"name": "UnitPrice",
"alias": "单价",
"comment": "单价"
},
{
"name": "Market_Price_Excluding_Tax",
"alias": "市场价不含税",
"comment": "市场价不含税"
},
{
"name": "Market_Price_Including_Tax",
"alias": "市场价含税",
"comment": "市场价含税，设备含税价"
},
{
"name": "Budget_Price_Excluding_Tax",
"alias": "预算价不含税",
"comment": "预算价不含税"
},
{
"name": "Budget_Price_Including_Tax",
"alias": "预算价含税",
"comment": "预算价含税"
},
{
"name": "Unit_Price_Excluding_Tax",
"alias": "单价不含税",
"comment": "单价不含税，设备不含税价"
},
{
"name": "GroupPrice",
"alias": "分组合价",
"comment": "分组合价"
},
{
"name": "Pump_Truck_Pouring",
"alias": "泵车浇制",
"comment": "泵车浇制，取值1为泵车浇制，取值0为非泵车浇制"
},
{
"name": "On_Site_Preparation",
"alias": "现场制备",
"comment": "现场制备，取值1为现场制备，取值0为非现场制备"
},
{
"name": "Clear_Water_Concrete",
"alias": "清水混凝土",
"comment": "清水混凝土，取值1为清水混凝土，取值0为非清水混凝土"
},
{
"name": "Debugging_Fee_Calculation",
"alias": "调试费计取",
"comment": "调试费计取，取值计取、不计取"
}
]
}
]
}
backend/data/projectstruct/博微电力造价工程文件格式_TotalCalculateTable.json
+86
View File
@@ -0,0 +1,86 @@
{
"Table": [
{
"name": "TotalCalculateTable",
"alias": "总算表",
"comment": "总算表也被称为“工程总费用”、“工程费用”。其中包含本地工程、辅助设施工程、编制基准期价差、设备购置费、其他费用、基本预备费、特殊费用、工程静态投资、动态费用、价差预备费、建设期贷款利息、工程动态投资、可抵扣增值税额。查询示例: SELECT Amount FROM TotalCalculateTable WHERE Name = 'findname'。",
"fileds": [
{
"name": "Id",
"alias": "id，项目id，费用id",
"comment": "费用项目id"
},
{
"name": "ParentId",
"alias": "父id，父级id",
"comment": "费用项目父级id"
},
{
"name": "Level",
"alias": "层级，层编号，层号",
"comment": "层级编号，从1开始"
},
{
"name": "Name",
"alias": "费用名，名称，项目名",
"comment": "费用名称，项目名称"
},
{
"name": "SerialNumber",
"alias": "序号",
"comment": "工程费用序号"
},
{
"name": "Code",
"alias": "代码，代号，编号",
"comment": "费用代码"
},
{
"name": "Rate",
"alias": "费用利率，费率",
"comment": "费率"
},
{
"name": "Amount",
"alias": "金额，价格",
"comment": "合计费"
},
{
"name": "WBS_Code",
"alias": "WBS编号，WBS编码",
"comment": "费用编码"
},
{
"name": "Path",
"alias": "费用全路径，路径",
"comment": "费用名称全路径"
},
{
"name": "Amount_InstallationCost",
"alias": "安装价格，安装金额，金额_安装费",
"comment": "安装费金额"
},
{
"name": "Amount_EquipmentCost",
"alias": "设备价格，金额_设备费，设备金额",
"comment": "设备费金额"
},
{
"name": "Amount_OtherCost",
"alias": "其他费用价格，其他费用金额，金额_其他费",
"comment": "其他费金额"
},
{
"name": "Amount_Total",
"alias": "总的金额，金额_占总计，总体金额",
"comment": "合计费占总计"
},
{
"name": "Amount_UnitInvestment",
"alias": "合计投资金额，金额_单位投资",
"comment": "合计费单位投资"
}
]
}
]
}
backend/data/博微电力造价工程业务数据说明.docx
BIN
View File
Binary file not shown.
backend/data/工程造价基础知识.doc
BIN
View File
Binary file not shown.
backend/data/电力造价知识.docx
BIN
View File
Binary file not shown.
backend/main.py
+45 -50
View File
@@ -1,5 +1,5 @@
from dotenv import load_dotenv
load_dotenv()
import logging
@@ -10,63 +10,58 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import RedirectResponse
from app.api.routers.chat import chat_router
from app.api.routers.upload import file_upload_router
from app.api.routers.app import v1_router
from app.settings import init_settings,init_ProjectInfo
from app.settings import init_settings
from app.observability import init_observability
from fastapi.staticfiles import StaticFiles
from phoenix.trace import using_project
logger = logging.getLogger("uvicorn")
app = None
def init_webserver():
global app
app = FastAPI()
environment = os.getenv("ENVIRONMENT", "dev") # Default to 'development' if not set
if environment == "dev":
logger.warning("Running in development mode - allowing CORS for all origins")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
usPrj = using_project(os.getenv("PHOENIX_PROJECT_NAME"))
usPrj.__enter__()
def mount_static_files(directory, path):
if os.path.exists(directory):
for dir, _, _ in os.walk(directory):
relative_path = os.path.relpath(dir, directory)
mount_path = path if relative_path == "." else f"{path}/{relative_path}"
logger.info(f"Mounting static files '{dir}' at {mount_path}")
app.mount(mount_path, StaticFiles(directory=dir), name=f"{dir}-static")
import nest_asyncio
nest_asyncio.apply()
# Mount the data files to serve the file viewer
mount_static_files("data", "/api/files/data")
# Mount the output files from tools
mount_static_files("data_output", "/api/files/output")
app.include_router(chat_router, prefix="/api/chat")
app.include_router(file_upload_router, prefix="/api/chat/upload")
init_settings()
init_observability()
init_ProjectInfo()
app = FastAPI()
environment = os.getenv("ENVIRONMENT", "dev") # Default to 'development' if not set
if environment == "dev":
logger.warning("Running in development mode - allowing CORS for all origins")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
def mount_static_files(directory, path):
if os.path.exists(directory):
for dir, _, _ in os.walk(directory):
relative_path = os.path.relpath(dir, directory)
mount_path = path if relative_path == "." else f"{path}/{relative_path}"
logger.info(f"Mounting static files '{dir}' at {mount_path}")
app.mount(mount_path, StaticFiles(directory=dir), name=f"{dir}-static")
# Mount the data files to serve the file viewer
mount_static_files("data", "/api/files/data")
# Mount the output files from tools
mount_static_files("data_output", "/api/files/output")
app.include_router(chat_router, prefix="/api/chat")
app.include_router(file_upload_router, prefix="/api/chat/upload")
app.include_router(v1_router, prefix="/v1")
@app.get("/")
async def redirect_to_docs():
return RedirectResponse(url="/docs")
# Redirect to documentation page when accessing base URL
@app.get("/")
async def redirect_to_docs():
return RedirectResponse(url="/docs")
if __name__ == "__main__":
app_host = os.getenv("APP_HOST", "0.0.0.0")
app_port = int(os.getenv("APP_PORT", "8000"))
reload = True if environment == "dev" else False
reload = False
uvicorn.run(app="main:app", host=app_host, port=app_port, reload=reload)
from phoenix.trace import using_project
with using_project(os.getenv("PHOENIX_PROJECT_NAME")) as obj:
init_settings()
init_observability()
init_webserver()
app_host = os.getenv("APP_HOST", "0.0.0.0")
app_port = int(os.getenv("APP_PORT", "8000"))
#reload = True if environment == "dev" else False
reload = False
uvicorn.run(app=app, host=app_host, port=app_port, reload=reload)
backend/modelProvide/customDashScope.py
-64
View File
@@ -1,64 +0,0 @@
from llama_index.llms.dashscope import DashScope
from llama_index.core.base.llms.types import LLMMetadata
class DashScopeGenerationModels:
"""DashScope Qwen serial models."""
QWEN_TURBO = "qwen-turbo"
QWEN_PLUS = "qwen-plus"
QWEN_MAX = "qwen-max"
QWEN_MAX_1201 = "qwen-max-1201"
QWEN_MAX_LONGCONTEXT = "qwen-max-longcontext"
QWEN2_MATH_72B_INSTRUCT = 'qwen2-math-72b-instruct',
QWEN2_72B = 'qwen2-72b-instruct'
DASHSCOPE_MODEL_META = {
DashScopeGenerationModels.QWEN_TURBO: {
"context_window": 1024 * 8,
"num_output": 1024 * 8,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN_PLUS: {
"context_window": 1024 * 32,
"num_output": 1024 * 32,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN_MAX: {
"context_window": 1024 * 8,
"num_output": 1024 * 8,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN_MAX_1201: {
"context_window": 1024 * 8,
"num_output": 1024 * 8,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN_MAX_LONGCONTEXT: {
"context_window": 1024 * 30,
"num_output": 1024 * 30,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN2_MATH_72B_INSTRUCT: {
"context_window": 1024 * 2,
"num_output": 1024 * 8,
"is_chat_model": True,
},
DashScopeGenerationModels.QWEN2_72B: {
"context_window": 1024 * 2,
"num_output": 1024 * 8,
"is_chat_model": True,
},
}
class CustomDashScope(DashScope):
@property
def metadata(self) -> LLMMetadata:
DASHSCOPE_MODEL_META[self.model_name]["num_output"] = (
self.max_tokens or DASHSCOPE_MODEL_META[self.model_name]["num_output"]
)
return LLMMetadata(
model_name=self.model_name, **DASHSCOPE_MODEL_META[self.model_name]
)
backend/nltk_data/corpora/cmudict.zip
BIN
View File
Binary file not shown.
backend/nltk_data/corpora/stopwords.zip
BIN
View File
Binary file not shown.
backend/nltk_data/dict.txt
-349046
View File
File diff suppressed because it is too large Load Diff
backend/nltk_data/taggers/averaged_perceptron_tagger.zip
BIN
View File
Binary file not shown.
backend/nltk_data/tokenizers/punkt.zip
BIN
View File
Binary file not shown.
backend/poetry.lock
Generated
-8339
View File
File diff suppressed because it is too large Load Diff
backend/pyproject.toml
+21 -60
View File
@@ -10,54 +10,36 @@ readme = "README.md"
generate = "app.engine.generate:generate_datasource"
[tool.poetry.dependencies]
python = "^3.11,<3.13"
fastapi = "0.110.3"
python-dotenv = "^1.0.1"
python = "^3.11,<3.12"
fastapi = "^0.112.0"
python-dotenv = "^1.0.0"
aiostream = "^0.6.2"
cachetools = "^5.5.0"
llama-index = "0.10.63"
cachetools = "^5.3.3"
protobuf = "4.25.4"
nltk = "^3.9.1"
jieba = "^0.42.1"
transformers = "^4.43.0"
#arize-phoenix = "^4.12.0"
openinference-instrumentation-llama-index="^3.0.2"
llama-index = "^0.11.7"
llama-index-core = "^0.11.7"
llama-index-callbacks-arize-phoenix = "^0.2.1"
llama-index-llms-dashscope = "^0.2.0"
llama-index-embeddings-dashscope = "^0.2.1"
#llama-index-postprocessor-dashscope-rerank = "^0.2.0"
openinference-instrumentation-llama-index="2.2.3"
llama-index-callbacks-arize-phoenix = "^0.1.4"
llama-index-llms-dashscope = "^0.1.2"
llama-index-embeddings-dashscope = "^0.1.4"
llama-index-postprocessor-dashscope-rerank-custom = "0.1.0"
qdrant-client="^1.10.1"
llama-index-vector-stores-qdrant = "^0.2.14"
chroma="^0.5.5"
llama-index-vector-stores-chroma = "^0.1.10"
llama-index-readers-json = "^0.1.5"
llama-index-llms-ollama = "^0.3.1"
llama-index-embeddings-ollama = "^0.3.0"
xinference = "^0.15.0"
xinference-client = "^0.15.0"
llama-index-llms-xinference = "^0.2.1"
llama-index-embeddings-xinference = "^0.1.0"
llama-index-postprocessor-xinference-rerank = "^0.1.0"
qdrant-client="^1.11.0"
llama-index-vector-stores-qdrant = "^0.3.0"
chroma="^0.2.0"
llama-index-vector-stores-chroma = "^0.2.0"
llama-index-readers-json = "^0.2.0"
llama-index-retrievers-bm25 = "^0.3.0"
llama-index-experimental = "^0.3.0"
duckduckgo_search = "^6.2.10"
duckduckgo_search = "^6.2.6"
[tool.poetry.dependencies.uvicorn]
extras = [ "standard" ]
version = "^0.30.6"
version = "^0.23.2"
[tool.poetry.dependencies.llama-index-readers-database]
version = "^0.2.0"
version = "^0.1.3"
[tool.poetry.dependencies.pymysql]
version = "^1.1.1"
version = "^1.1.0"
extras = [ "rsa" ]
#[tool.poetry.dependencies.psycopg2]
@@ -70,30 +52,9 @@ extras = [ "rsa" ]
version = "^0.8"
[tool.poetry.dependencies.e2b_code_interpreter]
version = "^0.0.7"
[[tool.poetry.source]]
name = "ali"
url = "https://mirrors.aliyun.com/pypi/simple/"
priority = "primary"
[[tool.poetry.source]]
name = "tencent"
url = "https://mirrors.cloud.tencent.com/pypi/simple/"
priority = "primary"
[[tool.poetry.source]]
name = "tsinghua"
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple/"
priority = "primary"
version = "0.0.7"
[build-system]
requires = [ "poetry-core" ]
build-backend = "poetry.core.masonry.api"
tiktoken
build-backend = "poetry.core.masonry.api"
backend/run-test.bat
+1 -1
View File
@@ -1,4 +1,4 @@
rmdir /S /Q storage_vector
rmdir /S /Q storage
python tests/query.py
C:\Users\liuyue\AppData\Local\pypoetry\Cache\virtualenvs\app-laEO4lY0-py3.11\Scripts\python tests/query.py
backend/run.bat
+1 -1
View File
@@ -1 +1 @@
python main.py
C:\Users\liuyue\AppData\Local\pypoetry\Cache\virtualenvs\app-laEO4lY0-py3.11\Scripts\python main.py
backend/test1/FeeCollectionTable.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "人工费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是6.3500000000"
},
{
"question": "乙供装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "甲供装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "冬雨季施工增加费的费率是多少?",
"answer": "费率是3.5700000000"
},
{
"question": "材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "机械价差的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "规费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是35.7600000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "合计的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "社会保险费的费率是多少?",
"answer": "费率是15.0000000000"
},
{
"question": "间接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "合计的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "利润的费率是多少?",
"answer": "费率是5.2400000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "社会保险费的费率是多少?",
"answer": "费率是15.0000000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "乙供设备不含税价的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是17.1300000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是35.7600000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "甲供设备含税价的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "施工机械使用费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "定额直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "主材费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "施工企业配合调试费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "施工机械使用费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是6.3500000000"
},
{
"question": "施工工具用具使用费的费率是多少?",
"answer": "费率是3.8200000000"
},
{
"question": "措施费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "材料价差的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "措施费的费率是多少?",
"answer": "费率是100.0000000000"
}
]
backend/test1/OtherFee.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "前期工作管理费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程监理费的金额是多少?",
"answer": "金额是131009.9200000000"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "生产准备费的金额是多少?",
"answer": "金额是472373669.4635599852"
},
{
"question": "电力工程技术经济标准编制费的金额是多少?",
"answer": "金额是84352440.9756360054"
},
{
"question": "项目建设技术服务费的金额是多少?",
"answer": "金额是16855957065.4302005768"
},
{
"question": "工程保险费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其他的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "施工图文件评审费的金额是多少?",
"answer": "金额是24940.0000000000"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "项目前期工作费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其他的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "项目法人管理费的金额是多少?",
"answer": "金额是986923559.4149370193"
},
{
"question": "专业爆破服务费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "用地预审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设备材料监造费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设备材料监造费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察费的金额是多少?",
"answer": "金额是12122154260.0000000000"
},
{
"question": "项目法人管理费的金额是多少?",
"answer": "金额是986923559.4149370193"
},
{
"question": "社会稳定风险评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察费的金额是多少?",
"answer": "金额是12122154260.0000000000"
},
{
"question": "环境影响评价费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "使用林地可行性研究费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设计费的金额是多少?",
"answer": "金额是4042055949.4299998283"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "施工图文件评审费的金额是多少?",
"answer": "金额是24940.0000000000"
},
{
"question": "项目后评价费的金额是多少?",
"answer": "金额是421762204.8781780005"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察设计费的金额是多少?",
"answer": "金额是16164210209.4300003052"
},
{
"question": "前期工作管理费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "初步设计文件评审费的金额是多少?",
"answer": "金额是18560.0000000000"
},
{
"question": "特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "初步设计文件评审费的金额是多少?",
"answer": "金额是18560.0000000000"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "矿产压覆评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设计费的金额是多少?",
"answer": "金额是4042055949.4299998283"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "电力工程技术经济标准编制费的金额是多少?",
"answer": "金额是84352440.9756360054"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "矿产压覆评估费用的金额是多少?",
"answer": "金额是0E-10"
}
]
backend/test1/ParamTuner.py
-138
View File
@@ -1,138 +0,0 @@
import nest_asyncio
nest_asyncio.apply()
from llama_index.core import SimpleDirectoryReader
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core import VectorStoreIndex
from llama_index.core.evaluation import (
FaithfulnessEvaluator,
DatasetGenerator,
CorrectnessEvaluator,
SemanticSimilarityEvaluator,
)
from llama_index.experimental.param_tuner import ParamTuner
from llama_index.experimental.param_tuner.base import RunResult
from llama_index.llms.openai import OpenAI
import asyncio
# 初始化环境
from app.observability import init_observability
from app.settings import init_settings
from dotenv import load_dotenv
load_dotenv()
init_settings()
init_observability()
# 读取文档
documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data()
# 参数字典
param_dict = {
"chunk_size": [512, 1024],
"top_k": [1, 5],
"temperature": [0.1, 1.0]
}
# 辅助函数
def _build_index(chunk_size, documents):
# 构建索引
splitter = SentenceSplitter(chunk_size=chunk_size)
vector_index = VectorStoreIndex.from_documents(
documents, transformations=[splitter],
)
return vector_index
# 评估函数
def evaluate_query_engine(query_engine, questions):
loop = asyncio.get_event_loop()
correct, total = loop.run_until_complete(_evaluate_query_engine_async(query_engine, questions))
return correct, total
async def _evaluate_query_engine_async(query_engine, questions):
c = [query_engine.aquery(q) for q in questions]
gathering_future = asyncio.gather(*c)
results = await gathering_future
total_correct = 0
for r in results:
eval_result = (
1 if FaithfulnessEvaluator().evaluate_response(response=r).passing else 0
)
total_correct += eval_result
return total_correct, len(results)
# 生成问题
question_generator = DatasetGenerator.from_documents(documents)
eval_questions = question_generator.generate_questions_from_nodes(1) # 假设生成10个问题
# 打印生成的问题
for i, q in enumerate(eval_questions, start=1):
print(f"问题 {i}: {q}")
# 目标函数
def objective_function(params_dict, documents, questions):
chunk_size = params_dict["chunk_size"]
top_k = params_dict["top_k"]
temperature = params_dict["temperature"]
# 构建索引
vector_index = _build_index(chunk_size, documents)
# 查询引擎
query_engine = vector_index.as_query_engine(
similarity_top_k=top_k, temperature=temperature
)
# 评估查询引擎
correct, total = 0, len(questions)
question_answers = [] # 添加列表来收集问题和答案
for question in questions:
response = query_engine.query(question)
if response is not None:
question_answers.append((question, response.response))
eval_result = FaithfulnessEvaluator().evaluate_response(response=response, query_str=question)
if eval_result.passing:
correct += 1
# 计算分数
score = correct / total if total > 0 else 0
return RunResult(score=score, params=params_dict, question_answers=question_answers)
# 创建 ParamTuner 实例
param_tuner = ParamTuner(
param_fn=lambda params_dict: objective_function(params_dict, documents, eval_questions),
param_dict=param_dict,
show_progress=True,
)
# 调用 tune 方法
results = param_tuner.tune()
best_result = results.best_run_result
best_top_k = best_result.params["top_k"]
best_chunk_size = best_result.params["chunk_size"]
best_temperature = best_result.params["temperature"]
print(f"得分: {best_result.score}")
print(f"Top-k: {best_top_k}")
print(f"文本块大小: {best_chunk_size}")
print(f"温度: {best_temperature}")
# 使用最佳参数再次运行查询引擎，并打印问题与答案
best_vector_index = _build_index(best_chunk_size, documents)
best_query_engine = best_vector_index.as_query_engine(
similarity_top_k=best_top_k, temperature=best_temperature
)
best_question_answers = []
for question in eval_questions:
response = best_query_engine.query(question)
if response is not None:
best_question_answers.append((question, response.response))
# 打印最佳参数下的问题与答案
for i, (question, answer) in enumerate(best_question_answers, start=1):
print(f"最佳参数 - 问题 {i}: {question}\n答案: {answer}\n")
backend/test1/ProjectDivision.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "预制基础的合价是多少?",
"answer": "合价是40567.2639480000"
},
{
"question": "绝缘子串及金具安装的合价是多少?",
"answer": "合价是2897171.9878110001"
},
{
"question": "杆塔工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础防护的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟土石方工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "地基处理的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "灌注桩基础的合价是多少?",
"answer": "合价是43466660.0544390008"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "悬垂绝缘子串及金具安装的合价是多少?",
"answer": "合价是1251465.0340440001"
},
{
"question": "护坡、挡土墙及排洪沟土石方工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "附件安装工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "导地线跨越架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "辅助工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "绝缘子串及金具安装的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟砌筑的合价是多少?",
"answer": "合价是709931.9013930000"
},
{
"question": "锚杆基础的合价是多少?",
"answer": "合价是15344967.9002950005"
},
{
"question": "建筑工程的合价是多少?",
"answer": "合价是25411.2790780000"
},
{
"question": "辅助工程的合价是多少?",
"answer": "合价是1046253.4135240000"
},
{
"question": "导地线跨越架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "电缆工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "输、送电线路试运的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础土石方工程的合价是多少?",
"answer": "合价是32872843180.7429008484"
},
{
"question": "基础永久性围堰的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础永久性围堰的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "混凝土及钢筋混凝土结构的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "输、送电线路试运的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "混合结构的合价是多少?",
"answer": "合价是16967.5193850000"
},
{
"question": "杆塔组立的合价是多少?",
"answer": "合价是2253906.0859830002"
},
{
"question": "附件安装工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "接地工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是27148.0310160000"
},
{
"question": "导地线架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟的合价是多少?",
"answer": "合价是709931.9013930000"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础永久性围堰砌筑的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "(2)拆除后不能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "安装工程的合价是多少?",
"answer": "合价是65324.9496330000"
},
{
"question": "尖峰、施工基面土石方工程的合价是多少?",
"answer": "合价是325205.4178770000"
},
{
"question": "架线工程的合价是多少?",
"answer": "合价是4844399648.0778598785"
},
{
"question": "杆塔组立的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是2088570123.2409000397"
},
{
"question": "导地线架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "耐张绝缘子串及金具安装的合价是多少?",
"answer": "合价是1645706.9537680000"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是2088570123.2409000397"
},
{
"question": "其他基础的合价是多少?",
"answer": "合价是3839666.7656879998"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
}
]
backend/test1/ProjectDivisions_CostPreview.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是440877984.9458540082"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是1086586.9018659999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是51486.7898090000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是3321.8139230000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是78005.0340730000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是3535892767.0972299576"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是24045.2334060000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是336253.7482950000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是142270.1346780000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是933061.7795919999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是182949.5997350000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是21220645.1637400016"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是933061.7795919999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是2501470269.7231497765"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是51486.7898090000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是55265.9111100000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是442897633.6273120046"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是1057484.3306960000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是442897633.6273120046"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是21220645.1637400016"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是336253.7482950000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表（余物清理）(1)的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是24045.2334060000"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是659466.5955000001"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是2501470269.7231497765"
}
]
backend/test1/ProjectProperties.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "降阻剂_数量的属性值是多少?",
"answer": "属性值是f"
},
{
"question": "导线2_单位单价的属性值是多少?",
"answer": "属性值是9"
},
{
"question": "导线_单公里用量的属性值是多少?",
"answer": "属性值是36"
},
{
"question": "线路参数_导地线防震措施的属性值是多少?",
"answer": "属性值是457"
},
{
"question": "合成绝缘子_数量的属性值是多少?",
"answer": "属性值是5"
},
{
"question": "基础垫层的属性值是多少?",
"answer": "属性值是"
},
{
"question": "其中：基础护壁用量的属性值是多少?",
"answer": "属性值是74394.212"
},
{
"question": "铺石加混凝土的属性值是多少?",
"answer": "属性值是0.0"
},
{
"question": "导线用量（西北）的属性值是多少?",
"answer": "属性值是-795976.0855"
},
{
"question": "导线单公里用量（西北）的属性值是多少?",
"answer": "属性值是-159195.2171"
},
{
"question": "灰土垫层单公里用量（西北）的属性值是多少?",
"answer": "属性值是8.0"
},
{
"question": "地线瓷绝缘子单公里用量（西北）的属性值是多少?",
"answer": "属性值是738.253"
},
{
"question": "地形条件_高山的属性值是多少?",
"answer": "属性值是7"
},
{
"question": "流砂坑比例的属性值是多少?",
"answer": "属性值是0.001"
},
{
"question": "碎石_数量的属性值是多少?",
"answer": "属性值是12"
},
{
"question": "线路参数_导地线防震措施的属性值是多少?",
"answer": "属性值是457"
},
{
"question": "灰土垫层的属性值是多少?",
"answer": "属性值是40.0"
},
{
"question": "交叉跨越_弱电线路的属性值是多少?",
"answer": "属性值是45"
},
{
"question": "地线1_根数的属性值是多少?",
"answer": "属性值是12"
},
{
"question": "土质比例_岩石（人凿）的属性值是多少?",
"answer": "属性值是49"
},
{
"question": "耐张混凝土杆基数的属性值是多少?",
"answer": "属性值是26.0"
},
{
"question": "设计单位的属性值是多少?",
"answer": "属性值是3"
},
{
"question": "接地钢的属性值是多少?",
"answer": "属性值是"
},
{
"question": "间隔棒_单公里用量的属性值是多少?",
"answer": "属性值是r"
},
{
"question": "导线其中：跳线和导线弧垂单公里用量（西北）的属性值是多少?",
"answer": "属性值是159203.0171"
},
{
"question": "桩基础的属性值是多少?",
"answer": "属性值是310.0"
},
{
"question": "降阻剂的属性值是多少?",
"answer": "属性值是"
},
{
"question": "可抵扣增值税（万元）的属性值是多少?",
"answer": "属性值是2005241.808822"
},
{
"question": "主要技术经济指标2的属性值是多少?",
"answer": "属性值是"
},
{
"question": "合成绝缘子_数量的属性值是多少?",
"answer": "属性值是5"
},
{
"question": "土质比例_水坑的属性值是多少?",
"answer": "属性值是47"
},
{
"question": "基础_插入式的属性值是多少?",
"answer": "属性值是3"
},
{
"question": "耐张角钢塔比例的属性值是多少?",
"answer": "属性值是250%"
},
{
"question": "地线的属性值是多少?",
"answer": "属性值是"
},
{
"question": "回路数的属性值是多少?",
"answer": "属性值是三回"
},
{
"question": "导线其中：跳线和导线弧垂用量的属性值是多少?",
"answer": "属性值是796015.0855"
},
{
"question": "OPGW用量（西北）的属性值是多少?",
"answer": "属性值是2904.737"
},
{
"question": "现浇混凝土_单公里用量的属性值是多少?",
"answer": "属性值是22"
},
{
"question": "架线工程费用（万元）（含价差）的属性值是多少?",
"answer": "属性值是3203726.0"
},
{
"question": "耐张钢管塔比例的属性值是多少?",
"answer": "属性值是300%"
},
{
"question": "单公里土石方量_基面的属性值是多少?",
"answer": "属性值是8*8"
},
{
"question": "地线2的属性值是多少?",
"answer": "属性值是"
},
{
"question": "降阻剂的属性值是多少?",
"answer": "属性值是"
},
{
"question": "土质比例的属性值是多少?",
"answer": "属性值是"
},
{
"question": "地线1_单位单价的属性值是多少?",
"answer": "属性值是113"
},
{
"question": "绝缘子串型式_悬垂串的属性值是多少?",
"answer": "属性值是48"
},
{
"question": "基坑土石方量（西北）的属性值是多少?",
"answer": "属性值是405403506.156"
},
{
"question": "基坑坚土的属性值是多少?",
"answer": "属性值是25585167.713"
},
{
"question": "基坑普通土的属性值是多少?",
"answer": "属性值是313873965.334"
},
{
"question": "瓷绝缘子单公里用量（西北）的属性值是多少?",
"answer": "属性值是201.0"
}
]
backend/test1/ProjectQuantities.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "电杆坑、塔坑、拉线坑人工挖方（或爆破）及回填 水坑 坑深2.0m以内的编码是多少?",
"answer": "编码是YX2-72"
},
{
"question": "钢筋加工及制作的编码是多少?",
"answer": "编码是YX3-43"
},
{
"question": "船舶运输 线材 每件重400kg以内 运输的编码是多少?",
"answer": "编码是YX1-132"
},
{
"question": "船舶运输 钢管塔材 运输的编码是多少?",
"answer": "编码是YX1-152"
},
{
"question": "碎石的编码是多少?",
"answer": "编码是C10020103"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1001"
},
{
"question": "船舶运输 金具、绝缘子、零星钢材 运输的编码是多少?",
"answer": "编码是YX1-144"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "船舶运输 线材 每件重1000kg以内 运输的编码是多少?",
"answer": "编码是YX1-136"
},
{
"question": "混凝土搅拌及浇制 每基基础联系梁混凝土量20m³以内的编码是多少?",
"answer": "编码是YX3-69"
},
{
"question": "索道运输 循环式 塔材 荷载1t以内 装卸的编码是多少?",
"answer": "编码是YX1-185"
},
{
"question": "人力运输 混凝土预制品 每件重100kg以内的编码是多少?",
"answer": "编码是YX1-6"
},
{
"question": "船舶运输 混凝土杆 每件重1500kg以上 运输的编码是多少?",
"answer": "编码是YX1-118"
},
{
"question": "碎石的编码是多少?",
"answer": "编码是C10020103"
},
{
"question": "电杆坑、塔坑、拉线坑人工挖方（或爆破）及回填 泥水 坑深8.0m以上的编码是多少?",
"answer": "编码是YX2-55"
},
{
"question": "机械施工土方 场地平整的编码是多少?",
"answer": "编码是GT1-1"
},
{
"question": "汽车运输 混凝土预制品 每件重100kg以内 装卸的编码是多少?",
"answer": "编码是YX1-69"
},
{
"question": "汽车运输 其他建筑安装材料 运输的编码是多少?",
"answer": "编码是YX1-108"
},
{
"question": "钻孔灌注桩基础 混凝土搅拌及浇制 孔深10m以内的编码是多少?",
"answer": "编码是YX3-171"
},
{
"question": "线路复测及分坑 直线双杆及拉线塔的编码是多少?",
"answer": "编码是YX2-3"
},
{
"question": "氧化锌避雷器安装 35kV的编码是多少?",
"answer": "编码是YX7-32"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1002"
},
{
"question": "汽车运输 其他建筑安装材料 装卸的编码是多少?",
"answer": "编码是YX1-107"
},
{
"question": "船舶运输 混凝土杆 每件重500kg以内 装卸的编码是多少?",
"answer": "编码是YX1-109"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1001"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "普通硅酸盐水泥的编码是多少?",
"answer": "编码是C09010102"
},
{
"question": "拖拉机运输 钢管塔材 运输的编码是多少?",
"answer": "编码是YX1-44"
},
{
"question": "尖峰及施工基面挖方（或爆破） 普通土的编码是多少?",
"answer": "编码是YX2-226"
},
{
"question": "汽车运输 角钢塔材 装卸的编码是多少?",
"answer": "编码是YX1-103"
},
{
"question": "接地槽挖方（或爆破）及回填 普通土的编码是多少?",
"answer": "编码是YX2-213"
},
{
"question": "水的编码是多少?",
"answer": "编码是C21010101"
},
{
"question": "直线(直线换位、直线转角)杆塔绝缘子串悬挂安装 35kV 针式单联串(悬垂串)的编码是多少?",
"answer": "编码是YX6-21"
},
{
"question": "直线(直线换位、直线转角)杆塔绝缘子串悬挂安装 35kV I型双联串(悬垂串)的编码是多少?",
"answer": "编码是YX6-22"
},
{
"question": "钻孔灌注桩基础 机械推钻成孔 砂砾石 孔深20m以内 孔径1.0m以内的编码是多少?",
"answer": "编码是YX3-117"
},
{
"question": "线路复测及分坑 直线自立塔的编码是多少?",
"answer": "编码是YX2-6"
},
{
"question": "钻孔灌注桩基础 凿桩头 桩径0.8m以上的编码是多少?",
"answer": "编码是YX3-180"
},
{
"question": "线路复测及分坑 耐张（转角）单杆的编码是多少?",
"answer": "编码是YX2-2"
},
{
"question": "中砂的编码是多少?",
"answer": "编码是C10010101"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "带电跨越电力线 被跨线电压等级 35kV的编码是多少?",
"answer": "编码是YX5-186"
},
{
"question": "人工挖土方 普土 深2m以内的编码是多少?",
"answer": "编码是YT1-1"
},
{
"question": "混凝土杆的编码是多少?",
"answer": "编码是"
},
{
"question": "接地模块安装的编码是多少?",
"answer": "编码是YX3-213"
},
{
"question": "拖拉机运输 线材 每件重400kg以内 运输的编码是多少?",
"answer": "编码是YX1-34"
},
{
"question": "拖拉机运输 其他建筑安装材料 装卸的编码是多少?",
"answer": "编码是YX1-45"
},
{
"question": "普通硅酸盐水泥的编码是多少?",
"answer": "编码是C09010102"
},
{
"question": "船舶运输 线材 每件重4000kg以内 装卸的编码是多少?",
"answer": "编码是YX1-139"
},
{
"question": "水的编码是多少?",
"answer": "编码是C21010101"
}
]
backend/test1/TotalCalculateTable.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "价差预备费的金额是多少?",
"answer": "金额是22731130869.6655998230"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "其他费用的金额是多少?",
"answer": "金额是210942912572.8689880371"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "其中：特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "编制基准期价差的金额是多少?",
"answer": "金额是29246752707.1180000305"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "其他费用的金额是多少?",
"answer": "金额是210942912572.8689880371"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "编制基准期价差的金额是多少?",
"answer": "金额是29246752707.1180000305"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "其中：可抵扣增值税额的金额是多少?",
"answer": "金额是20069645492.2888984680"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "其中：建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：可抵扣增值税额的金额是多少?",
"answer": "金额是20069645492.2888984680"
},
{
"question": "价差预备费的金额是多少?",
"answer": "金额是22731130869.6655998230"
},
{
"question": "一般线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "设备购置费的金额是多少?",
"answer": "金额是2567934636.3574500084"
}
]
backend/test1/__init__.py
View File
backend/test1/chromedb.py
-19
View File
@@ -1,19 +0,0 @@
import chromadb
# 创建 ChromaDB 客户端
chroma_client = chromadb.PersistentClient(path="/home/bw/ctr/zjdataai-app/backend/storage_vector-1/")
# 获取已存在的 "default" 集合
collection = chroma_client.get_collection(name="default")
# 获取集合中的所有数据
results = collection.get(
include=['documents', 'metadatas', 'embeddings'] # 只包含允许的选项
)
# 将结果转换为字符串并保存到txt文件中
with open('/home/bw/ctr/zjdataai-app/backend/test1/query_results-1.txt', 'w', encoding='utf-8') as file:
file.write(str(results))
# 打印结果
print("查询结果已保存到 query_results.txt 文件中。")
backend/test1/evaluator.py
-81
View File
@@ -1,81 +0,0 @@
from app.observability import init_observability
from app.settings import init_settings
from dotenv import load_dotenv
import nest_asyncio
nest_asyncio.apply()
load_dotenv()
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core import (
VectorStoreIndex,
SimpleDirectoryReader,
Response,
)
from llama_index.core.evaluation import (
FaithfulnessEvaluator,
DatasetGenerator,
CorrectnessEvaluator,
SemanticSimilarityEvaluator,)
init_settings()
init_observability()
faith_evaluator_qwen = FaithfulnessEvaluator() #诚实度评测
corr_evaluator_qwen = CorrectnessEvaluator() #准确率评测
Seman_evaluator_qwen = SemanticSimilarityEvaluator()#嵌入相似度评估
documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data()
splitter = SentenceSplitter(chunk_size=512)
vector_index = VectorStoreIndex.from_documents(
documents, transformations=[splitter],
)
# # 运行评估
# query_engine = vector_index.as_query_engine()
# response_vector = query_engine.query("工程监理费的金额是多少？")
# eval_result = evaluator_qwen.evaluate_response(response=response_vector)
# print(response_vector)
# print(eval_result)
question_generator = DatasetGenerator.from_documents(documents)
eval_questions = question_generator.generate_questions_from_nodes(5)
print(eval_questions)
import asyncio
async def evaluate_query_engine_async(query_engine, questions):
c = [query_engine.aquery(q) for q in questions]
gathering_future = asyncio.gather(*c)
results = await gathering_future
#print(results)
total_correct = 0
for r in results:
eval_result = (
1 if faith_evaluator_qwen.evaluate_response(response=r).passing else 0
)
total_correct += eval_result
return total_correct, len(results)
def evaluate_query_engine(query_engine, questions):
loop = asyncio.get_event_loop()
correct, total = loop.run_until_complete(evaluate_query_engine_async(query_engine, questions))
return correct, total
# 使用 evaluate_query_engine 函数
vector_query_engine = vector_index.as_query_engine()
correct, total = evaluate_query_engine(vector_query_engine, eval_questions[:5])
print(f"score: {correct}/{total}")
backend/test1/incorrect_answers_log.json
-28
View File
@@ -1,28 +0,0 @@
[]错误问题: 税金的费率是多少?
正确答案: 9.0000000000
查询结果: 9
错误问题: 冬雨季施工增加费的费率是多少?
正确答案: 3.5700000000
查询结果: 未找到有效数字
错误问题: 住房公积金的费率是多少?
正确答案: 15.0000000000
查询结果: 15
错误问题: 税金的费率是多少?
正确答案: 9.0000000000
查询结果: 9
错误问题: 冬雨季施工增加费的费率是多少?
正确答案: 3.5700000000
查询结果: 3
错误问题: 税金的费率是多少?
正确答案: 9.0000000000
查询结果: 9
错误问题: 冬雨季施工增加费的费率是多少?
正确答案: 3.5700000000
查询结果: 未找到有效数字
backend/test1/incorrect_answers_log.txt
-1
View File
@@ -1 +0,0 @@
backend/test1/parameters_results.json
View File
backend/test1/query_results.json
-12
View File
@@ -1,12 +0,0 @@
TOP_K: 5
LLM_TEMPERATURE: 0.1
similarity_top_k: 5.0
问题: 税金的费率是多少?
查询结果: SQL查询结果: 税金的费率是9.0%。请注意，查询结果中有多条重复的记录，但费率都是相同的，为9.0%。
正确答案: 9.0000000000
问题: 冬雨季施工增加费的费率是多少?
查询结果: SQL查询结果: 对于"冬雨季施工增加费"的费率，当前数据库中没有找到具体信息。这可能是因为费率会根据不同的项目、地区或时间有所变化。建议您查阅最新的项目文件或与项目负责人联系以获取准确的费率信息。
正确答案: 3.5700000000
backend/test1/query_results.txt
View File
backend/test1/query_results1.txt
View File
backend/test1/query_test.py
-201
View File
@@ -1,201 +0,0 @@
import re
import os
import sys
import json
from sqlalchemy import create_engine
from llama_index.core import VectorStoreIndex, SQLDatabase
from llama_index.core.indices.struct_store import SQLTableRetrieverQueryEngine
from llama_index.core.objects import SQLTableNodeMapping, ObjectIndex
from app.api.routers.chat import generate_filters
from app.engine import get_index
from app.engine.engine import makeDescriptionByEngine
from app.engine.loaders.db import CustomDatabaseReader
from app.engine.vectordb import get_vector_store
from app.observability import init_observability
from app.settings import init_settings
from dotenv import load_dotenv
load_dotenv()
def read_questions_and_answers(file_path):
questions_and_answers = []
with open(file_path, 'r', encoding='utf-8') as file:
data = json.load(file) # 读取 JSON 数据
for entry in data:
question = entry.get("question", "").strip() # 获取 question
answer = entry.get("answer", "").strip() # 直接获取 answer 而不是提取数字
if question and answer:
questions_and_answers.append((question, answer))
return questions_and_answers
def save_results_to_file(question, query_result, correct_answer, file_path):
# 保存原始查询结果
result_data = {
"问题": question,
"查询结果": str(query_result), # 保存原始查询结果
"正确答案": correct_answer
}
with open(file_path, 'a', encoding='utf-8') as file:
json.dump(result_data, file, ensure_ascii=False)
file.write('\n') # 每个结果条目之间添加换行符
def log_incorrect_answers(question, correct_answer, query_result, log_file_path):
# 保存原始查询结果
incorrect_data = {
"错误问题": question,
"正确答案": correct_answer,
"查询结果": str(query_result) # 保存原始查询结果
}
with open(log_file_path, 'a', encoding='utf-8') as file:
json.dump(incorrect_data, file, ensure_ascii=False)
file.write('\n') # 每个结果条目之间添加换行符
# 提取多个数字
def extract_all_numbers_from_result(result_str):
"""从查询结果字符串中提取所有数字"""
# 使用正则表达式匹配所有数值（包含小数和科学计数法）
numbers = re.findall(r"-?\d+,\d+(\.\d+)?|0E-\d+|\d+(\.\d+)?", result_str)
# 移除逗号并返回所有数字的列表
return [num.replace(',', '') for num in numbers]
# 判断两个浮点数是否接近
def is_close_enough(val1, val2, epsilon=1e-5):
"""判断两个数值是否在指定的误差范围内接近"""
return abs(val1 - val2) < epsilon
def is_answer_correct(query_result_str, correct_answer_str):
"""检查查询结果是否与正确答案匹配"""
# 提取查询结果中的数字或编码
query_result_value = extract_number_or_code_from_result(query_result_str)
# 提取正确答案中的数字或编码
correct_answer_value = extract_number_or_code_from_result(correct_answer_str)
# 对比提取的数字或编码
if query_result_value and correct_answer_value:
try:
# 移除逗号，并转换为浮点数
query_result_float = float(query_result_value.replace(',', ''))
correct_answer_float = float(correct_answer_value.replace(',', ''))
# 处理科学计数法中的零值
if query_result_float == 0.0 and correct_answer_float == 0.0:
return True
# 四舍五入处理到小数点后5位
rounded_query_result = round(query_result_float, 5)
rounded_correct_answer = round(correct_answer_float, 5)
# 比较四舍五入后的浮点数值
return rounded_query_result == rounded_correct_answer
except ValueError:
# 如果无法转换为浮点数，则直接比较字符串
return query_result_value == correct_answer_value
return False # 如果任何一方为空，则认为不匹配
def extract_number_or_code_from_result(result_str):
"""从查询结果字符串中提取数字或编码，并处理逗号、百分号和科学计数法"""
# 使用正则表达式匹配浮点数，包括可能的多位小数、逗号、百分比形式和科学计数法
match = re.search(r"(\d{1,3}(,\d{3})*(\.\d+)?|0E-\d+)", result_str)
if match:
number_str = match.group(1).replace(',', '').replace('%', '') # 移除逗号和百分号
return number_str
# 尝试从结果中提取所有可能的编码格式
potential_codes = re.findall(r"\b[A-Z][A-Za-z\d-]+\b", result_str)
# 返回第一个匹配的编码
return potential_codes[0] if potential_codes else None
def main(questions_file, query_type):
# 获取脚本所在的目录
script_dir = os.path.dirname(os.path.abspath(__file__))
# 将文件扩展名更改为 .json
questions_file_path = os.path.join(script_dir, questions_file)
results_file_path = os.path.join(script_dir, "query_results.json")
log_file_path = os.path.join(script_dir, "incorrect_answers_log.json")
# 如果 .json 文件不存在，则生成一个空的 JSON 文件
if not os.path.exists(questions_file_path):
with open(questions_file_path, 'w', encoding='utf-8') as file:
json.dump([], file) # 写入空数组
# 更新环境变量
os.environ['TOP_K'] = str(5) # 向量的TOP_K值
os.environ['LLM_TEMPERATURE'] = str(0.1) # 温度值
os.environ['similarity_top_k'] = str(5) # SQL的TOP_K值
init_settings()
init_observability()
index = get_index()
top_k = int(os.getenv("TOP_K")) # 向量的TOP_K值
temperature = float(os.getenv("LLM_TEMPERATURE")) # 温度值
similarity_top_k = int(os.getenv("similarity_top_k")) # SQL的TOP_K值
filters = generate_filters([])
engine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
sql_database = SQLDatabase(engine)
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
# 创建SQL查询工具
sql_obj_index = ObjectIndex.from_objects(
table_schema_objs,
table_node_mapping,
index_cls=VectorStoreIndex,
)
sql_query_engine = SQLTableRetrieverQueryEngine(sql_database,
sql_obj_index.as_retriever(similarity_top_k=similarity_top_k))
questions_and_answers = read_questions_and_answers(questions_file_path)
# 如果文件为空，则写入参数值
if os.path.getsize(results_file_path) == 0:
with open(results_file_path, 'w', encoding='utf-8') as file:
json.dump({
"TOP_K": top_k,
"LLM_TEMPERATURE": temperature,
"similarity_top_k": similarity_top_k
}, file, ensure_ascii=False)
file.write('\n')
# 循环执行查询
for i, (question, correct_answer) in enumerate(questions_and_answers):
print(f"执行查询 {i+1}: {question}")
if query_type == "vector":
query_engine = index.as_query_engine(
similarity_top_k=top_k, filters=filters
)
query_result = query_engine.query(question)
print(f"向量查询结果: {query_result}\n")
# 提取向量查询结果中的数字或编码进行匹配
query_result_str = f"The encoding for the query \"{question}\" is {str(query_result)}"
elif query_type == "sql":
sql_query_result = sql_query_engine.query(question)
print(f"SQL查询结果: {sql_query_result}\n")
# 提取SQL查询结果中的数字或编码进行匹配
query_result_str = f"The encoding for the query \"{question}\" is {str(sql_query_result)}"
else:
print("无效的查询类型，请选择 'vector' 或 'sql'")
sys.exit(1)
if is_answer_correct(query_result_str, correct_answer):
# 只在查询结果正确时记录结果
save_results_to_file(question, query_result_str, correct_answer, results_file_path)
else:
# 记录不正确的答案
log_incorrect_answers(question, correct_answer, query_result_str, log_file_path)
if __name__ == "__main__":
if len(sys.argv) < 3:
print("请提供questions.json文件名和查询类型（vector 或 sql）")
sys.exit(1)
questions_file = sys.argv[1]
query_type = sys.argv[2].lower()
main(questions_file, query_type)
backend/test1/question.py
-82
View File
@@ -1,82 +0,0 @@
import os
import json
from sqlalchemy import create_engine, MetaData, Table, func
from sqlalchemy.orm import sessionmaker
from dotenv import load_dotenv
load_dotenv()
def generate_questions_for_table(table_name, file_path, num_questions=50):
engine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
metadata = MetaData()
metadata.reflect(bind=engine)
# 定义每张表的列索引、值列和问题模板
tables_info = {
"ProjectProperties": (0, "Value", "{name_value}的属性值是多少?", "属性值是{answer_value}"),
"OtherFee": (3, "Amount", "{name_value}的金额是多少?", "金额是{answer_value}"),
"FeeCollectionTable": (1, "Rate", "{name_value}的费率是多少?", "费率是{answer_value}"),
"ProjectDivision": (5, "Sum_Price", "{name_value}的合价是多少?", "合价是{answer_value}"),
"ProjectDivisions_CostPreview": (4, "Direct_Cost", "{name_value}的直接费是多少?", "直接费是{answer_value}"),
"TotalCalculateTable": (3, "Amount", "{name_value}的金额是多少?", "金额是{answer_value}"),
"ProjectQuantities": (6, "Encoding", "{name_value}的编码是多少?", "编码是{answer_value}")
}
if table_name not in tables_info:
print(f"未找到表 {table_name} 的配置信息")
return
# 获取表信息
name_index, value_column, question_template, answer_template = tables_info[table_name]
# 加载表
table = Table(table_name, metadata, autoload_with=engine)
# 创建会话
Session = sessionmaker(bind=engine)
session = Session()
# 获取列名
name_column = table.columns.keys()[name_index]
questions_and_answers = []
# 生成指定数量的问题
for _ in range(num_questions):
# 查询表中的随机一行，并获取名称列和值列的值
row = session.query(table).order_by(func.random()).first()
name_value = getattr(row, name_column)
answer_value = getattr(row, value_column)
# 构造问题和答案
question = question_template.format(name_value=name_value)
answer = answer_template.format(answer_value=answer_value)
# 添加到列表中
questions_and_answers.append({
"question": question,
"answer": answer
})
# 将问题和答案以 JSON 格式写入文件
with open(file_path, 'w', encoding='utf-8') as file:
json.dump(questions_and_answers, file, ensure_ascii=False, indent=4)
print(f"已生成表 {table_name} 的问题到文件: {file_path}")
def main():
engine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
metadata = MetaData()
metadata.reflect(bind=engine)
# 获取脚本所在目录
script_dir = os.path.dirname(os.path.abspath(__file__))
# 遍历每张表并生成对应的 JSON 文件
for table_name in metadata.tables.keys():
# 文件路径为：脚本目录 + 表名 + .json
file_path = os.path.join(script_dir, f"{table_name}.json")
generate_questions_for_table(table_name, file_path)
if __name__ == "__main__":
main()
backend/test1/questions.txt
-14
View File
@@ -1,14 +0,0 @@
question：线路参数_转角次数的属性值是多少? answer：线路参数_转角次数的属性值是64
question：接地土石方量的属性值是多少? answer：接地土石方量的属性值是16
question：工程监理费的金额是多少? answer：工程监理费的金额是131009.92
question：矿产压覆评估费用的金额是多少? answer：矿产压覆评估费用的金额是0
question：线路取费表（余物清理）的费率是多少? answer：线路取费表（余物清理）的费率是100
question：线路取费表（拆除）的费率是多少? answer：线路取费表（拆除）的费率是100
question：一般线路本体工程的合价是多少? answer：一般线路本体工程的合价是55105688268.5176
question：基础工程的合价是多少? answer：基础工程的合价是49051649642.9667
question：线路取费表(调试工程)aa的直接费是多少? answer：线路取费表(调试工程)aa的直接费是22411207942.4858
question：线路取费表的直接费是多少? answer：线路取费表的直接费是7314300665.34141
question：一般线路本体工程的金额是多少? answer：一般线路本体工程的金额是55105688268.5176
question：架空输电线路本体工程的金额是多少? answer：架空输电线路本体工程的金额是55105688268.5176
question：截止阀的编码是多少? answer：截止阀的编码是F01010101
question：自定义主材的编码是多少? answer：自定义主材的编码是asd
backend/test1/test_parameters.py
-111
View File
@@ -1,111 +0,0 @@
import os
import json
import sys
from llama_index.core import VectorStoreIndex, SQLDatabase
from llama_index.core.indices.struct_store import SQLTableRetrieverQueryEngine
from llama_index.core.objects import SQLTableNodeMapping, ObjectIndex
from sqlalchemy import create_engine
from app.api.routers.chat import generate_filters
from app.engine import get_index
from app.engine.engine import makeDescriptionByEngine
from app.engine.vectordb import get_vector_store
from app.observability import init_observability
from app.settings import init_settings
from dotenv import load_dotenv
load_dotenv()
def read_questions(file_path):
with open(file_path, 'r', encoding='utf-8') as file:
data = json.load(file)
questions = [item["question"] for item in data]
return questions
def save_results_to_file(question, result, file_path):
result_data = {
"question": question,
"result": result
}
with open(file_path, 'a', encoding='utf-8') as file:
json.dump(result_data, file, ensure_ascii=False)
file.write('\n')
def main(questions_file, query_type):
# 更新环境变量
os.environ['TOP_K'] = str(5) # 向量的TOP_K值
os.environ['similarity_top_k'] = str(1) # SQL的TOP_K值固定为1
init_settings()
init_observability()
index = get_index()
top_k = int(os.getenv("TOP_K")) # 向量的TOP_K值
similarity_top_k = int(os.getenv("similarity_top_k")) # SQL的TOP_K值
filters = generate_filters([])
engine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
sql_database = SQLDatabase(engine)
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
# 创建SQL查询工具
sql_obj_index = ObjectIndex.from_objects(
table_schema_objs,
table_node_mapping,
index_cls=VectorStoreIndex,
)
sql_query_engine = SQLTableRetrieverQueryEngine(sql_database,
sql_obj_index.as_retriever(similarity_top_k=similarity_top_k))
script_dir = os.path.dirname(os.path.abspath(__file__))
questions_file_path = os.path.join(script_dir, questions_file)
results_file_path = os.path.join(script_dir, "parameters_results.json")
questions = read_questions(questions_file_path)
# # 如果文件为空，则写入参数值
# if not os.path.isfile(results_file_path):
# with open(results_file_path, 'w', encoding='utf-8') as file:
# json.dump({
# "TOP_K": top_k,
# "similarity_top_k": similarity_top_k
# }, file, ensure_ascii=False)
# file.write('\n')
# 循环执行查询
for i, question in enumerate(questions):
print(f"Executing query {i+1}: {question}")
# 对于每个问题，测试不同的温度值
for temperature in range(1, 11): # 从1到10
temperature_value = temperature / 10.0 # 从0.1到1.0
os.environ['LLM_TEMPERATURE'] = str(temperature_value)
if query_type == "vector":
query_engine = index.as_query_engine(
similarity_top_k=top_k, filters=filters
)
query_result = query_engine.query(question)
print(f"Vector Query Result: {query_result}\n")
save_results_to_file(question, f"Current parameters: TOP_K={top_k}, similarity_top_k={similarity_top_k}, Temperature: {temperature_value:.1f}, Vector Query Result: {query_result}", results_file_path)
elif query_type == "sql":
sql_query_result = sql_query_engine.query(question)
print(f"SQL Query Result: {sql_query_result}\n")
save_results_to_file(question, f"Current parameters: TOP_K={top_k}, similarity_top_k={similarity_top_k}, Temperature: {temperature_value:.1f}, SQL Query Result: {sql_query_result}", results_file_path)
else:
print("无效的查询类型，请选择 'vector' 或 'sql'")
sys.exit(1)
if __name__ == "__main__":
if len(sys.argv) < 3:
print("请提供questions.json文件的路径和查询类型（vector 或 sql）")
sys.exit(1)
questions_file = sys.argv[1]
query_type = sys.argv[2].lower()
from phoenix.trace import using_project
with using_project(questions_file) as obj:
main(questions_file, query_type)
backend/tests/query.py
+9 -10
View File
@@ -19,13 +19,14 @@ def main():
init_settings()
init_observability()
index = get_index()
indexs = get_index()
if len(indexs) > 0:
index = list(indexs.values())[0]
top_k = 5
filters = generate_filters([])
#question = "从工程属性表中查找工程名称"
#question = "总算表中名称等于架空输电线路本体工程的金额?"
question = "工程监理费的金额是多少？"
question = "总算表中名称等于架空输电线路本体工程的金额?"
# 创建向量检索查询工具
query_engine = index.as_query_engine(
similarity_top_k=top_k, filters=filters
@@ -36,20 +37,18 @@ def main():
engine = create_engine(os.getenv("SQL_DATABASE_URL", ""))
sql_database = SQLDatabase(engine)
loader = CustomDatabaseReader(sql_database)
documents = loader.load_data(query="select * from ProjectProperties")
table_schema_objs = makeDescriptionByEngine(sql_database)
table_node_mapping = SQLTableNodeMapping(sql_database)
vectorIndex = VectorStoreIndex()
# 创建SQL查询工具
# sql_obj_index = ObjectIndex.from_objects(
# table_schema_objs,
# table_node_mapping,
# index_cls=VectorStoreIndex,
# )
sql_obj_index = ObjectIndex.from_objects_and_index(
sql_obj_index = ObjectIndex.from_objects(
table_schema_objs,
vectorIndex,
table_node_mapping,
index_cls=VectorStoreIndex,
)
query_result =vectorIndex.as_query_engine(
backend/unit_test/FeeCollectionTable.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "人工费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是6.3500000000"
},
{
"question": "乙供装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "甲供装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "装置性材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "冬雨季施工增加费的费率是多少?",
"answer": "费率是3.5700000000"
},
{
"question": "材料费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "机械价差的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "规费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是35.7600000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "合计的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "社会保险费的费率是多少?",
"answer": "费率是15.0000000000"
},
{
"question": "间接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "合计的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "利润的费率是多少?",
"answer": "费率是5.2400000000"
},
{
"question": "税金的费率是多少?",
"answer": "费率是9.0000000000"
},
{
"question": "社会保险费的费率是多少?",
"answer": "费率是15.0000000000"
},
{
"question": "直接工程费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "乙供设备不含税价的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是17.1300000000"
},
{
"question": "企业管理费的费率是多少?",
"answer": "费率是35.7600000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "夜间施工增加费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "甲供设备含税价的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "施工机械使用费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "安全文明施工费的费率是多少?",
"answer": "费率是3.5500000000"
},
{
"question": "定额直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "主材费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "直接费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "施工企业配合调试费的费率是多少?",
"answer": "费率是0E-10"
},
{
"question": "施工机械使用费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是6.3500000000"
},
{
"question": "施工工具用具使用费的费率是多少?",
"answer": "费率是3.8200000000"
},
{
"question": "措施费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "材料价差的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "措施费的费率是多少?",
"answer": "费率是100.0000000000"
}
]
backend/unit_test/OtherFee.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "前期工作管理费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程监理费的金额是多少?",
"answer": "金额是131009.9200000000"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "生产准备费的金额是多少?",
"answer": "金额是472373669.4635599852"
},
{
"question": "电力工程技术经济标准编制费的金额是多少?",
"answer": "金额是84352440.9756360054"
},
{
"question": "项目建设技术服务费的金额是多少?",
"answer": "金额是16855957065.4302005768"
},
{
"question": "工程保险费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其他的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "施工图文件评审费的金额是多少?",
"answer": "金额是24940.0000000000"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "项目前期工作费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其他的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "项目法人管理费的金额是多少?",
"answer": "金额是986923559.4149370193"
},
{
"question": "专业爆破服务费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "用地预审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设备材料监造费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设备材料监造费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察费的金额是多少?",
"answer": "金额是12122154260.0000000000"
},
{
"question": "项目法人管理费的金额是多少?",
"answer": "金额是986923559.4149370193"
},
{
"question": "社会稳定风险评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察费的金额是多少?",
"answer": "金额是12122154260.0000000000"
},
{
"question": "环境影响评价费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "使用林地可行性研究费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设计费的金额是多少?",
"answer": "金额是4042055949.4299998283"
},
{
"question": "环境监测及环境保护验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "施工图文件评审费的金额是多少?",
"answer": "金额是24940.0000000000"
},
{
"question": "项目后评价费的金额是多少?",
"answer": "金额是421762204.8781780005"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "勘察设计费的金额是多少?",
"answer": "金额是16164210209.4300003052"
},
{
"question": "前期工作管理费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "节能评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "初步设计文件评审费的金额是多少?",
"answer": "金额是18560.0000000000"
},
{
"question": "特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "初步设计文件评审费的金额是多少?",
"answer": "金额是18560.0000000000"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "矿产压覆评估费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "设计费的金额是多少?",
"answer": "金额是4042055949.4299998283"
},
{
"question": "水土保持方案编审费用的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "电力工程技术经济标准编制费的金额是多少?",
"answer": "金额是84352440.9756360054"
},
{
"question": "桩基检测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "矿产压覆评估费用的金额是多少?",
"answer": "金额是0E-10"
}
]
backend/unit_test/ProjectDivision.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "预制基础的合价是多少?",
"answer": "合价是40567.2639480000"
},
{
"question": "绝缘子串及金具安装的合价是多少?",
"answer": "合价是2897171.9878110001"
},
{
"question": "杆塔工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础防护的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟土石方工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "地基处理的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "灌注桩基础的合价是多少?",
"answer": "合价是43466660.0544390008"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "悬垂绝缘子串及金具安装的合价是多少?",
"answer": "合价是1251465.0340440001"
},
{
"question": "护坡、挡土墙及排洪沟土石方工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "附件安装工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "导地线跨越架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "辅助工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "绝缘子串及金具安装的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟砌筑的合价是多少?",
"answer": "合价是709931.9013930000"
},
{
"question": "锚杆基础的合价是多少?",
"answer": "合价是15344967.9002950005"
},
{
"question": "建筑工程的合价是多少?",
"answer": "合价是25411.2790780000"
},
{
"question": "辅助工程的合价是多少?",
"answer": "合价是1046253.4135240000"
},
{
"question": "导地线跨越架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "电缆工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "输、送电线路试运的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础土石方工程的合价是多少?",
"answer": "合价是32872843180.7429008484"
},
{
"question": "基础永久性围堰的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础永久性围堰的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "混凝土及钢筋混凝土结构的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "输、送电线路试运的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "混合结构的合价是多少?",
"answer": "合价是16967.5193850000"
},
{
"question": "杆塔组立的合价是多少?",
"answer": "合价是2253906.0859830002"
},
{
"question": "附件安装工程的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "接地工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "新增项目名称的合价是多少?",
"answer": "合价是27148.0310160000"
},
{
"question": "导地线架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "护坡、挡土墙及排洪沟的合价是多少?",
"answer": "合价是709931.9013930000"
},
{
"question": "(1)拆除后能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "基础永久性围堰砌筑的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "(2)拆除后不能利用的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "安装工程的合价是多少?",
"answer": "合价是65324.9496330000"
},
{
"question": "尖峰、施工基面土石方工程的合价是多少?",
"answer": "合价是325205.4178770000"
},
{
"question": "架线工程的合价是多少?",
"answer": "合价是4844399648.0778598785"
},
{
"question": "杆塔组立的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是2088570123.2409000397"
},
{
"question": "导地线架设的合价是多少?",
"answer": "合价是0E-10"
},
{
"question": "耐张绝缘子串及金具安装的合价是多少?",
"answer": "合价是1645706.9537680000"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是2088570123.2409000397"
},
{
"question": "其他基础的合价是多少?",
"answer": "合价是3839666.7656879998"
},
{
"question": "架线工程材料工地运输的合价是多少?",
"answer": "合价是0E-10"
}
]
backend/unit_test/ProjectDivisions_CostPreview.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是440877984.9458540082"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是1086586.9018659999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是51486.7898090000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是3321.8139230000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是78005.0340730000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是3535892767.0972299576"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是24045.2334060000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是336253.7482950000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是142270.1346780000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是933061.7795919999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是182949.5997350000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是21220645.1637400016"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是933061.7795919999"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是2501470269.7231497765"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是51486.7898090000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是55265.9111100000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是442897633.6273120046"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是1057484.3306960000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是442897633.6273120046"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是21220645.1637400016"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是336253.7482950000"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表（余物清理）(1)的直接费是多少?",
"answer": "直接费是61049.8665780000"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是24045.2334060000"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（余物清理）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是659466.5955000001"
},
{
"question": "线路取费表（拆除）的直接费是多少?",
"answer": "直接费是0E-10"
},
{
"question": "线路取费表的直接费是多少?",
"answer": "直接费是2501470269.7231497765"
}
]
backend/unit_test/ProjectProperties.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "降阻剂_数量的属性值是多少?",
"answer": "属性值是f"
},
{
"question": "导线2_单位单价的属性值是多少?",
"answer": "属性值是9"
},
{
"question": "导线_单公里用量的属性值是多少?",
"answer": "属性值是36"
},
{
"question": "线路参数_导地线防震措施的属性值是多少?",
"answer": "属性值是457"
},
{
"question": "合成绝缘子_数量的属性值是多少?",
"answer": "属性值是5"
},
{
"question": "基础垫层的属性值是多少?",
"answer": "属性值是"
},
{
"question": "其中：基础护壁用量的属性值是多少?",
"answer": "属性值是74394.212"
},
{
"question": "铺石加混凝土的属性值是多少?",
"answer": "属性值是0.0"
},
{
"question": "导线用量（西北）的属性值是多少?",
"answer": "属性值是-795976.0855"
},
{
"question": "导线单公里用量（西北）的属性值是多少?",
"answer": "属性值是-159195.2171"
},
{
"question": "灰土垫层单公里用量（西北）的属性值是多少?",
"answer": "属性值是8.0"
},
{
"question": "地线瓷绝缘子单公里用量（西北）的属性值是多少?",
"answer": "属性值是738.253"
},
{
"question": "地形条件_高山的属性值是多少?",
"answer": "属性值是7"
},
{
"question": "流砂坑比例的属性值是多少?",
"answer": "属性值是0.001"
},
{
"question": "碎石_数量的属性值是多少?",
"answer": "属性值是12"
},
{
"question": "线路参数_导地线防震措施的属性值是多少?",
"answer": "属性值是457"
},
{
"question": "灰土垫层的属性值是多少?",
"answer": "属性值是40.0"
},
{
"question": "交叉跨越_弱电线路的属性值是多少?",
"answer": "属性值是45"
},
{
"question": "地线1_根数的属性值是多少?",
"answer": "属性值是12"
},
{
"question": "土质比例_岩石（人凿）的属性值是多少?",
"answer": "属性值是49"
},
{
"question": "耐张混凝土杆基数的属性值是多少?",
"answer": "属性值是26.0"
},
{
"question": "设计单位的属性值是多少?",
"answer": "属性值是3"
},
{
"question": "接地钢的属性值是多少?",
"answer": "属性值是"
},
{
"question": "间隔棒_单公里用量的属性值是多少?",
"answer": "属性值是r"
},
{
"question": "导线其中：跳线和导线弧垂单公里用量（西北）的属性值是多少?",
"answer": "属性值是159203.0171"
},
{
"question": "桩基础的属性值是多少?",
"answer": "属性值是310.0"
},
{
"question": "降阻剂的属性值是多少?",
"answer": "属性值是"
},
{
"question": "可抵扣增值税（万元）的属性值是多少?",
"answer": "属性值是2005241.808822"
},
{
"question": "主要技术经济指标2的属性值是多少?",
"answer": "属性值是"
},
{
"question": "合成绝缘子_数量的属性值是多少?",
"answer": "属性值是5"
},
{
"question": "土质比例_水坑的属性值是多少?",
"answer": "属性值是47"
},
{
"question": "基础_插入式的属性值是多少?",
"answer": "属性值是3"
},
{
"question": "耐张角钢塔比例的属性值是多少?",
"answer": "属性值是250%"
},
{
"question": "地线的属性值是多少?",
"answer": "属性值是"
},
{
"question": "回路数的属性值是多少?",
"answer": "属性值是三回"
},
{
"question": "导线其中：跳线和导线弧垂用量的属性值是多少?",
"answer": "属性值是796015.0855"
},
{
"question": "OPGW用量（西北）的属性值是多少?",
"answer": "属性值是2904.737"
},
{
"question": "现浇混凝土_单公里用量的属性值是多少?",
"answer": "属性值是22"
},
{
"question": "架线工程费用（万元）（含价差）的属性值是多少?",
"answer": "属性值是3203726.0"
},
{
"question": "耐张钢管塔比例的属性值是多少?",
"answer": "属性值是300%"
},
{
"question": "单公里土石方量_基面的属性值是多少?",
"answer": "属性值是8*8"
},
{
"question": "地线2的属性值是多少?",
"answer": "属性值是"
},
{
"question": "降阻剂的属性值是多少?",
"answer": "属性值是"
},
{
"question": "土质比例的属性值是多少?",
"answer": "属性值是"
},
{
"question": "地线1_单位单价的属性值是多少?",
"answer": "属性值是113"
},
{
"question": "绝缘子串型式_悬垂串的属性值是多少?",
"answer": "属性值是48"
},
{
"question": "基坑土石方量（西北）的属性值是多少?",
"answer": "属性值是405403506.156"
},
{
"question": "基坑坚土的属性值是多少?",
"answer": "属性值是25585167.713"
},
{
"question": "基坑普通土的属性值是多少?",
"answer": "属性值是313873965.334"
},
{
"question": "瓷绝缘子单公里用量（西北）的属性值是多少?",
"answer": "属性值是201.0"
}
]
backend/unit_test/ProjectQuantities.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "电杆坑、塔坑、拉线坑人工挖方（或爆破）及回填 水坑 坑深2.0m以内的编码是多少?",
"answer": "编码是YX2-72"
},
{
"question": "钢筋加工及制作的编码是多少?",
"answer": "编码是YX3-43"
},
{
"question": "船舶运输 线材 每件重400kg以内 运输的编码是多少?",
"answer": "编码是YX1-132"
},
{
"question": "船舶运输 钢管塔材 运输的编码是多少?",
"answer": "编码是YX1-152"
},
{
"question": "碎石的编码是多少?",
"answer": "编码是C10020103"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1001"
},
{
"question": "船舶运输 金具、绝缘子、零星钢材 运输的编码是多少?",
"answer": "编码是YX1-144"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "船舶运输 线材 每件重1000kg以内 运输的编码是多少?",
"answer": "编码是YX1-136"
},
{
"question": "混凝土搅拌及浇制 每基基础联系梁混凝土量20m³以内的编码是多少?",
"answer": "编码是YX3-69"
},
{
"question": "索道运输 循环式 塔材 荷载1t以内 装卸的编码是多少?",
"answer": "编码是YX1-185"
},
{
"question": "人力运输 混凝土预制品 每件重100kg以内的编码是多少?",
"answer": "编码是YX1-6"
},
{
"question": "船舶运输 混凝土杆 每件重1500kg以上 运输的编码是多少?",
"answer": "编码是YX1-118"
},
{
"question": "碎石的编码是多少?",
"answer": "编码是C10020103"
},
{
"question": "电杆坑、塔坑、拉线坑人工挖方（或爆破）及回填 泥水 坑深8.0m以上的编码是多少?",
"answer": "编码是YX2-55"
},
{
"question": "机械施工土方 场地平整的编码是多少?",
"answer": "编码是GT1-1"
},
{
"question": "汽车运输 混凝土预制品 每件重100kg以内 装卸的编码是多少?",
"answer": "编码是YX1-69"
},
{
"question": "汽车运输 其他建筑安装材料 运输的编码是多少?",
"answer": "编码是YX1-108"
},
{
"question": "钻孔灌注桩基础 混凝土搅拌及浇制 孔深10m以内的编码是多少?",
"answer": "编码是YX3-171"
},
{
"question": "线路复测及分坑 直线双杆及拉线塔的编码是多少?",
"answer": "编码是YX2-3"
},
{
"question": "氧化锌避雷器安装 35kV的编码是多少?",
"answer": "编码是YX7-32"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1002"
},
{
"question": "汽车运输 其他建筑安装材料 装卸的编码是多少?",
"answer": "编码是YX1-107"
},
{
"question": "船舶运输 混凝土杆 每件重500kg以内 装卸的编码是多少?",
"answer": "编码是YX1-109"
},
{
"question": "混凝土（保护帽）的编码是多少?",
"answer": "编码是ZH1001"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "普通硅酸盐水泥的编码是多少?",
"answer": "编码是C09010102"
},
{
"question": "拖拉机运输 钢管塔材 运输的编码是多少?",
"answer": "编码是YX1-44"
},
{
"question": "尖峰及施工基面挖方（或爆破） 普通土的编码是多少?",
"answer": "编码是YX2-226"
},
{
"question": "汽车运输 角钢塔材 装卸的编码是多少?",
"answer": "编码是YX1-103"
},
{
"question": "接地槽挖方（或爆破）及回填 普通土的编码是多少?",
"answer": "编码是YX2-213"
},
{
"question": "水的编码是多少?",
"answer": "编码是C21010101"
},
{
"question": "直线(直线换位、直线转角)杆塔绝缘子串悬挂安装 35kV 针式单联串(悬垂串)的编码是多少?",
"answer": "编码是YX6-21"
},
{
"question": "直线(直线换位、直线转角)杆塔绝缘子串悬挂安装 35kV I型双联串(悬垂串)的编码是多少?",
"answer": "编码是YX6-22"
},
{
"question": "钻孔灌注桩基础 机械推钻成孔 砂砾石 孔深20m以内 孔径1.0m以内的编码是多少?",
"answer": "编码是YX3-117"
},
{
"question": "线路复测及分坑 直线自立塔的编码是多少?",
"answer": "编码是YX2-6"
},
{
"question": "钻孔灌注桩基础 凿桩头 桩径0.8m以上的编码是多少?",
"answer": "编码是YX3-180"
},
{
"question": "线路复测及分坑 耐张（转角）单杆的编码是多少?",
"answer": "编码是YX2-2"
},
{
"question": "中砂的编码是多少?",
"answer": "编码是C10010101"
},
{
"question": "人力运输 混凝土杆 每件重500kg以内的编码是多少?",
"answer": "编码是YX1-1"
},
{
"question": "带电跨越电力线 被跨线电压等级 35kV的编码是多少?",
"answer": "编码是YX5-186"
},
{
"question": "人工挖土方 普土 深2m以内的编码是多少?",
"answer": "编码是YT1-1"
},
{
"question": "混凝土杆的编码是多少?",
"answer": "编码是"
},
{
"question": "接地模块安装的编码是多少?",
"answer": "编码是YX3-213"
},
{
"question": "拖拉机运输 线材 每件重400kg以内 运输的编码是多少?",
"answer": "编码是YX1-34"
},
{
"question": "拖拉机运输 其他建筑安装材料 装卸的编码是多少?",
"answer": "编码是YX1-45"
},
{
"question": "普通硅酸盐水泥的编码是多少?",
"answer": "编码是C09010102"
},
{
"question": "船舶运输 线材 每件重4000kg以内 装卸的编码是多少?",
"answer": "编码是YX1-139"
},
{
"question": "水的编码是多少?",
"answer": "编码是C21010101"
}
]
backend/unit_test/TotalCalculateTable.json
-202
View File
@@ -1,202 +0,0 @@
[
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "价差预备费的金额是多少?",
"answer": "金额是22731130869.6655998230"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "其他费用的金额是多少?",
"answer": "金额是210942912572.8689880371"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "其中：特种设备安全监测费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "编制基准期价差的金额是多少?",
"answer": "金额是29246752707.1180000305"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "其他费用的金额是多少?",
"answer": "金额是210942912572.8689880371"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "编制基准期价差的金额是多少?",
"answer": "金额是29246752707.1180000305"
},
{
"question": "特殊项目的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "其中：可抵扣增值税额的金额是多少?",
"answer": "金额是20069645492.2888984680"
},
{
"question": "小计的金额是多少?",
"answer": "金额是458257942570.3129882812"
},
{
"question": "动态费用的金额是多少?",
"answer": "金额是61246155757.1752014160"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "工程静态投资的金额是多少?",
"answer": "金额是715035853336.3909912109"
},
{
"question": "其中：建设场地征用及清理费的金额是多少?",
"answer": "金额是16831284.2287110016"
},
{
"question": "建设期贷款利息的金额是多少?",
"answer": "金额是38515024887.5095977783"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "架空输电线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "其中：水土保持监测及验收费的金额是多少?",
"answer": "金额是0E-10"
},
{
"question": "工程动态投资的金额是多少?",
"answer": "金额是776282009093.5660400391"
},
{
"question": "其中：可抵扣增值税额的金额是多少?",
"answer": "金额是20069645492.2888984680"
},
{
"question": "价差预备费的金额是多少?",
"answer": "金额是22731130869.6655998230"
},
{
"question": "一般线路本体工程的金额是多少?",
"answer": "金额是55105688268.5176010132"
},
{
"question": "其中：工程建设检测费的金额是多少?",
"answer": "金额是185575370.1463980079"
},
{
"question": "基本预备费的金额是多少?",
"answer": "金额是14020310849.7332000732"
},
{
"question": "设备购置费的金额是多少?",
"answer": "金额是2567934636.3574500084"
}
]
backend/unit_test/corr_test.py
-118
View File
@@ -1,118 +0,0 @@
import json
from dotenv import load_dotenv
import asyncio
import nest_asyncio
nest_asyncio.apply()
from llama_index.core.prompts import (
ChatMessage,
ChatPromptTemplate,
MessageRole
)
DEFAULT_SYSTEM_TEMPLATE = """
您是一个问答聊天机器人的专业评估系统。
您将获得以下信息：
- 用户查询，
- 生成的回答，
也可能提供一个参考答案作为评估的依据。
您的任务是判断生成回答的相关性和正确性。
输出一个代表全面评估的单一分数。
您必须在一行中仅返回该分数。
不要以其他任何格式返回答案。
在单独的一行提供给定分数的理由。
请遵循以下评分指南：
- 您的分数必须在1到5之间，其中1是最差，5是最好的。
-如果生成的回答与用户查询不相关，您应该给出1分。
-如果生成的回答相关但包含错误，您应该给出2到3分之间的分数。
-如果生成的回答相关且完全正确，您应该给出4到5分之间的分数。
示例响应：
4.0
生成的回答与参考答案的指标完全相同，但不够精炼。
"""
DEFAULT_USER_TEMPLATE = """
## User Query
{query}
## Reference Answer
{reference_answer}
## Generated Answer
{generated_answer}
"""
DEFAULT_EVAL_TEMPLATE = ChatPromptTemplate(
message_templates=[
ChatMessage(role=MessageRole.SYSTEM, content=DEFAULT_SYSTEM_TEMPLATE),
ChatMessage(role=MessageRole.USER, content=DEFAULT_USER_TEMPLATE),
]
)
from app.api.routers.models import ChatData, Message
from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore
from llama_index.core.vector_stores.types import MetadataFilter, MetadataFilters
from llama_index.core.evaluation import CorrectnessEvaluator
from app.engine import get_chat_engine
from app.api.routers.chat import generate_filters
from app.engine.index import get_index
from app.observability import init_observability
from app.settings import init_settings
load_dotenv()
init_settings()
init_observability()
index = get_index()
# 初始化聊天引擎和评估器
chat_engine = get_chat_engine()
corr_evaluator_qwen = CorrectnessEvaluator()
# 加载本地问题回答文件
file_path = 'D:/LLM_model/text2sql/zjdataai-app-test/backend/unit_test/test.json'
output_file_path = file_path.replace('.json', '_test.json')
with open(file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
# 异步函数用于评估查询
async def evaluate_query(question, answer, index, output_file):
response = await chat_engine.astream_chat(question)
content_str = str(response.sources[0])
result = corr_evaluator_qwen.evaluate(
query=question,
response=content_str,
reference=answer,
)
result_dict = {
"编号": index,
"问题": question,
"答案": answer,
"回答": result.response,
"得分(0~5)": result.score,
"评价": result.feedback
}
with open(output_file, 'a', encoding='utf-8') as f:
f.write(json.dumps(result_dict, ensure_ascii=False, indent=4))
f.write(',')
# 主异步函数
async def main():
for index, item in enumerate(data, start=1):
await evaluate_query(item['question'], item['answer'], index, output_file_path)
# 运行主协程
asyncio.run(main())
backend/unit_test/test.json
-10
View File
@@ -1,10 +0,0 @@
[
{
"question": "人工费的费率是多少?",
"answer": "费率是100.0000000000"
},
{
"question": "临时设施费的费率是多少?",
"answer": "费率是6.3500000000"
}
]
backend/unit_test/test_test.json
-18
View File
@@ -1,18 +0,0 @@
{
"编号": 1,
"问题": "人工费的费率是多少?",
"答案": "费率是100.0000000000",
"实际回答": "人工费的费率是100.0。",
"得分": 4.5,
"评价": "生成的答案与参考答案一致，只是表达方式略有不同，但完全正确且相关。"
}
********************
{
"编号": 2,
"问题": "临时设施费的费率是多少?",
"答案": "费率是6.3500000000",
"实际回答": "在新的上下文中，临时设施费的计算应当基于取费定额人工费和取费定额机械费，但当前费率被设置为0.0，这意味着需要重新确认或调整费率。在没有具体费率的情况下，不能直接给出确定的费率值。如果需要计算临时设施费，应根据实际的取费定额人工费和取费定额机械费进行计算。参考之前的6.35%的费率可能是一个起点，但请注意，这需要根据项目的具体情况和最新的费用标准进行调整。",
"得分": 1.0,
"评价": "生成的答案与用户查询不相关，用户询问的是临时设施费的费率，而生成的答案提供的是一个关于如何计算临时设施费的解释，且提到了一个不相关的0.0费率，这与用户的问题不符。同时，即使提到了6.35%的费率，也没有明确指出这就是用户想要的答案，反而强调了需要根据项目具体情况调整，这增加了用户的困惑。"
}
********************
backend/util/register.py
-43
View File
@@ -1,43 +0,0 @@
from typing import Dict, List
class ClsRegister:
clsLst:Dict[str,Dict[str,str]] = {}
@classmethod
def add(cls,catalog,name,obj) -> None:
if catalog in cls.clsLst:
registry = cls.clsLst[catalog]
registry[name] = obj
else:
registry:Dict[str,str] = {}
registry[name] = obj
cls.clsLst[catalog] = registry
@classmethod
def get(cls,catalog,name,fuzzy:bool=False) -> None:
if catalog in cls.clsLst:
registry = cls.clsLst[catalog]
for key,value in registry.items():
if fuzzy:
if key in name:
return value
else:
if key == name:
return value
return None
@classmethod
def getClsList(cls,catalog) -> None:
res_Lst = []
if catalog in cls.clsLst:
registry = cls.clsLst[catalog]
for key,value in registry.items():
res_Lst.append(value)
return res_Lst
def register(catalog,name):
def decorator(className):
ClsRegister.add(catalog,name,className)
return className
return decorator
frontend/.env.example
-10
View File
@@ -1,10 +0,0 @@
# The backend API for chat endpoint.
#NEXT_PUBLIC_CHAT_API=http://localhost:8000/api/chat
NEXT_PUBLIC_CHAT_API=http://10.1.6.41:8000/api/chat
#PHOENIX_SERVER_URL=http://localhost:6006/
PHOENIX_SERVER_URL=http://10.1.6.41:6006/
# Let's the user change indexes in LlamaCloud projects
NEXT_PUBLIC_USE_LLAMACLOUD=false
frontend/app/components/header.tsx
+1 -1
View File
@@ -4,7 +4,7 @@ const phoenixUrl = process.env.PHOENIX_SERVER_URL;
export default function Header() {
return (
<div className="z-10 w-full items-center justify-between font-mono text-sm lg:flex">
<div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
<p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
<code className="font-mono font-bold"><a href="javascript:location.reload();">清空当前会话</a></code>
</p>
frontend/app/components/ui/chat/chat-input.tsx
+2 -1
View File
@@ -99,8 +99,9 @@ export default function ChatInput(
</div>
)}
<div className="flex w-full items-start justify-between gap-4 ">
<Input
<textarea
autoFocus
rows={2}
name="message"
placeholder="请输入消息"
className="flex-1"
frontend/app/components/ui/chat/chat-message/chat-sources.tsx
+3 -20
View File
@@ -127,26 +127,9 @@ function NodeInfo({ nodeInfo }: { nodeInfo: NodeInfo }) {
}
// node generated by unknown loader, implement renderer by analyzing logged out metadata
// return (
// <p>
// 对不起, 未知文件类型. 无法打开当前的来源文件。
// </p>
// );
return (
<div className="flex items-center my-2">
<span>{nodeInfo.text}</span>
<Button
onClick={() => copyToClipboard(nodeInfo.url!)}
size="icon"
variant="ghost"
className="h-12 w-12 shrink-0"
>
{isCopied ? (
<Check className="h-4 w-4" />
) : (
<Copy className="h-4 w-4" />
)}
</Button>
</div>
<p>
对不起, 未知文件类型. 无法打开当前的来源文件。
</p>
);
}
frontend/app/components/ui/chat/chat.interface.ts
+1 -1
View File
@@ -10,7 +10,7 @@ export interface ChatHandler {
data?: any;
},
) => void;
handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
handleInputChange: (e: React.ChangeEvent<HTMLTextAreaElement>) => void;
reload?: () => void;
stop?: () => void;
onFileUpload?: (file: File) => Promise<void>;
frontend/package-lock.json
Generated
-16885
View File
File diff suppressed because it is too large Load Diff
phoenixserver/.env.example
-3
View File
@@ -1,3 +0,0 @@
ENV_PHOENIX_HOST=0.0.0.0
ENV_PHOENIX_PORT=6006
PHOENIX_HOST_ROOT_PATH=./.phoenix/
phoenixserver/phoenixserver.py
+2 -3
View File
@@ -1,10 +1,9 @@
import os
from dotenv import load_dotenv
load_dotenv()
import phoenix as px
os.environ['PHOENIX_HOST'] = "0.0.0.0"
session = px.launch_app(use_temp_dir=False)
import msvcrt
phoenixserver/runphoenixserver.bat
+1 -1
View File
@@ -2,4 +2,4 @@ SET ENV_PHOENIX_HOST=0.0.0.0
SET ENV_PHOENIX_PORT=6006
SET PHOENIX_HOST_ROOT_PATH=./.phoenix/
python phoenixserver.py
C:\Users\liuyue\AppData\Local\pypoetry\Cache\virtualenvs\app-pCyqx0Uo-py3.11\Scripts\python phoenixserver.py
webapp
-1
Submodule webapp deleted from 77dbc14a64