ly/zjdataai-app
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

将项目划分表按照业务拆分

Browse Source
This commit is contained in:
chentianrui
2024-08-23 15:05:48 +08:00
parent 5fc8375a06
commit d1117c73c4
5 changed files with 29 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/engine/__init__.py
+6 -6
View File
@@ -4,7 +4,7 @@ from llama_index.core.agent import AgentRunner, ReActChatFormatter
from llama_index.core.settings import Settings
from llama_index.core.tools.query_engine import QueryEngineTool
from app.engine.engine import create_query_engine, create_summary_query_engine, create_sql_query_engine
from app.engine.engine import create_query_engine, create_summary_query_engine
from app.engine.index import get_index
#from app.engine.loaders.db import makeDescriptionByEngine
from app.engine.tools import ToolFactory
@@ -17,11 +17,11 @@ def get_chat_engine(filters=None, params=None):
tools = []
# 创建SQL查询工具
sql_query_engine = create_sql_query_engine()
sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
name="zjdata_query_tool",
description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具"
)
# sql_query_engine = create_summary_query_engine(index)
# sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
# name="zjdata_query_tool",
# description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具"
# )
#tools.append(sql_query_tool)
# Add query tool if index exists
backend/app/engine/engine.py
+2 -2
View File
@@ -52,8 +52,8 @@ def get_Retriever(index,**kwargs):
sql_database = None
sql_obj_index = None
# Create a sql query engine
def create_sql_query_engine(top_k=3, use_reranker=False, filters=None):
# Create a summary query engine
def create_summary_query_engine(top_k=3, use_reranker=False, filters=None):
global sql_obj_index
global sql_database
if sql_obj_index is None or sql_database is None:
backend/app/engine/loaders/__init__.py
+1 -1
View File
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def load_configs():
with open("config/loaders.yaml") as f:
with open("config/loaders.yaml",'r', encoding='utf-8') as f:
configs = yaml.safe_load(f)
return configs
backend/app/engine/loaders/db.py
+15 -30
View File
@@ -2,17 +2,14 @@ import logging
from typing import Any, List, Optional
from llama_index.core import SQLDatabase, Document
from llama_index.core.objects import SQLTableSchema
from llama_index.core.readers.base import BaseReader
from llama_index.readers.database import DatabaseReader
from pydantic import BaseModel
from sqlalchemy import create_engine
from sqlalchemy import text
from sqlalchemy import create_engine, text
from sqlalchemy.engine import Engine
logger = logging.getLogger(__name__)
class CustomDatabaseReader(BaseReader):
class CustomDatabaseReader(DatabaseReader):
"""Simple Database reader.
Concatenates each row into Document used by LlamaIndex.
@@ -76,28 +73,30 @@ class CustomDatabaseReader(BaseReader):
"set of credentials."
)
def load_data(self, query: str) -> List[Document]:
def load_data(self, query: str, explanation: str) -> List[Document]:
"""Query and load data from the Database, returning a list of Documents.
Args:
query (str): Query parameter to filter tables and rows.
explanation (str): Explanation for the query to be included in the document.
Returns:
List[Document]: A list of Document objects.
"""
dco_str = ""
dco_str = explanation + "\n"
with self.sql_database.engine.connect() as connection:
if query is None:
raise ValueError("A query parameter is necessary to filter the data")
else:
result = connection.execute(text(query))
dco_str = ", ".join(
dco_str += ", ".join(
[f"{entry}" for entry in result.keys()]
)
) + "\n"
for item in result.fetchall():
# fetch each item
# Fetch each item
record_str = ", ".join(
[f"{entry}" for col, entry in zip(result.keys(), item)]
)
@@ -111,7 +110,7 @@ class CustomDatabaseReader(BaseReader):
class DBLoaderConfig(BaseModel):
uri: str
queries: List[str]
queries: List[dict]
def get_db_documents(configs: list[DBLoaderConfig]):
docs = []
@@ -123,33 +122,19 @@ def get_db_documents(configs: list[DBLoaderConfig]):
return docs
metadata = {
#'file_name':'',
'file_type':'application/booway.document.zj',
#'file_path':'',
#'file_size':'',
#'creation_date':'',
#'last_modified_date':'',
'file_type': 'application/booway.document.zj',
}
#from llama_index.readers.database import DatabaseReader
for entry in configs:
engine = create_engine(entry.uri)
sql_database = SQLDatabase(engine)
# table_schema_objs = makeDescriptionByEngine(sql_database)
# table_node_mapping = SQLTableNodeMapping(sql_database)
#
# nodes = table_node_mapping.to_nodes(table_schema_objs)
# for node in nodes:
# node.metadata.update(metadata)
#
# docs.extend(nodes)
queries = entry.queries or []
loader = CustomDatabaseReader(sql_database)
for query in queries:
for query_dict in entry.queries:
query = query_dict.get("sql", "")
explanation = query_dict.get("explanation", "")
logger.info(f"Loading data from database with query: {query}")
documents = loader.load_data(query=query)
documents = loader.load_data(query=query, explanation=explanation)
docs.extend(documents)
return docs
backend/app/engine/prompt.py
+5 -4
View File
@@ -39,15 +39,16 @@ refine_template_str = (
"这是原本的问题： {query_str}\n"
"我们已经提供了回答: {existing_answer}\n"
"现在我们有机会改进这个回答 "
"使用以下更多上下文（仅当需要用时）\n"
"使用以下更多上下文（仅当有助于改进回答时使用）\n"
"------------\n"
"{context_msg}\n"
"------------\n"
"根据新的上下文, 请改进原来的回答。"
"如果新的上下文没有用, 直接返回原本的回答。\n"
"如果是表结构或者是数据库的相关内容，只用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
"如果新的上下文对回答没有影响，或者原来的回答已经正确，直接返回原本的回答。\n"
"如果新的上下文有助于改进，请基于它更新回答，但不要引入与问题无关的信息。\n"
"如果是表结构或者是数据库的相关内容，仅用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
"改进的回答: "
)
refine_template = PromptTemplate(refine_template_str)
summary_template_str = (