将项目划分表按照业务拆分

backend/app/engine/__init__.py

@@ -4,7 +4,7 @@ from llama_index.core.agent import AgentRunner, ReActChatFormatter
 from llama_index.core.settings import Settings
 from llama_index.core.tools.query_engine import QueryEngineTool
 
-from app.engine.engine import create_query_engine, create_summary_query_engine, create_sql_query_engine
+from app.engine.engine import create_query_engine, create_summary_query_engine
 from app.engine.index import get_index
 #from app.engine.loaders.db import makeDescriptionByEngine
 from app.engine.tools import ToolFactory
@@ -17,11 +17,11 @@ def get_chat_engine(filters=None, params=None):
     tools = []
 
     # 创建SQL查询工具
-    sql_query_engine = create_sql_query_engine()
+#    sql_query_engine = create_summary_query_engine(index)
-    sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
+    # sql_query_tool = QueryEngineTool.from_defaults(query_engine=sql_query_engine,
-                                                   name="zjdata_query_tool",
+    #                                                name="zjdata_query_tool",
-                                                   description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具"
+    #                                                description="来源于一个由博微公司电力造价软件编制的造价工程文件。该文件以多张表格的形式存储存储了整个工程的全部数据内容。适用于以详细的自然语言查询表格数据方式查询造价工程各项具体属性、费用的数值。请先使用“zj_query_tool”无法解决才使用本工具"
-                                                   )
+    #                                                )
     #tools.append(sql_query_tool)
 
     # Add query tool if index exists

backend/app/engine/engine.py

@@ -52,8 +52,8 @@ def get_Retriever(index,**kwargs):
 sql_database = None
 sql_obj_index = None
 
-# Create a sql query engine
+# Create a summary query engine
-def create_sql_query_engine(top_k=3, use_reranker=False, filters=None):
+def create_summary_query_engine(top_k=3, use_reranker=False, filters=None):
     global sql_obj_index
     global sql_database
     if sql_obj_index is None or sql_database is None:

backend/app/engine/loaders/__init__.py

@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 
 def load_configs():
-    with open("config/loaders.yaml") as f:
+    with open("config/loaders.yaml",'r', encoding='utf-8') as f:
         configs = yaml.safe_load(f)
     return configs
 

backend/app/engine/loaders/db.py

@@ -2,17 +2,14 @@ import logging
 from typing import Any, List, Optional
 
 from llama_index.core import SQLDatabase, Document
-from llama_index.core.objects import SQLTableSchema
-from llama_index.core.readers.base import BaseReader
 from llama_index.readers.database import DatabaseReader
 from pydantic import BaseModel
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
-from sqlalchemy import text
 from sqlalchemy.engine import Engine
 
 logger = logging.getLogger(__name__)
 
-class CustomDatabaseReader(BaseReader):
+class CustomDatabaseReader(DatabaseReader):
     """Simple Database reader.
 
     Concatenates each row into Document used by LlamaIndex.
@@ -76,28 +73,30 @@ class CustomDatabaseReader(BaseReader):
                 "set of credentials."
             )
 
-    def load_data(self, query: str) -> List[Document]:
+    def load_data(self, query: str, explanation: str) -> List[Document]:
         """Query and load data from the Database, returning a list of Documents.
 
         Args:
             query (str): Query parameter to filter tables and rows.
+            explanation (str): Explanation for the query to be included in the document.
 
         Returns:
             List[Document]: A list of Document objects.
         """
-        dco_str = ""
+        dco_str = explanation + "\n"  
+
         with self.sql_database.engine.connect() as connection:
             if query is None:
                 raise ValueError("A query parameter is necessary to filter the data")
             else:
                 result = connection.execute(text(query))
 
-            dco_str = ", ".join(
+            dco_str += ", ".join(
                 [f"{entry}" for entry in result.keys()]
-            )
+            ) + "\n"
 
             for item in result.fetchall():
-                # fetch each item
+                # Fetch each item
                 record_str = ", ".join(
                     [f"{entry}" for col, entry in zip(result.keys(), item)]
                 )
@@ -111,7 +110,7 @@ class CustomDatabaseReader(BaseReader):
 
 class DBLoaderConfig(BaseModel):
     uri: str
-    queries: List[str]
+    queries: List[dict]  
 
 def get_db_documents(configs: list[DBLoaderConfig]):
     docs = []
@@ -123,33 +122,19 @@ def get_db_documents(configs: list[DBLoaderConfig]):
         return docs
 
     metadata = {
-        #'file_name':'',
+        'file_type': 'application/booway.document.zj',
-        'file_type':'application/booway.document.zj',
-        #'file_path':'',
-        #'file_size':'',
-        #'creation_date':'',
-        #'last_modified_date':'',
     }
 
-    #from llama_index.readers.database import DatabaseReader
     for entry in configs:
         engine = create_engine(entry.uri)
         sql_database = SQLDatabase(engine)
 
-        # table_schema_objs = makeDescriptionByEngine(sql_database)
-        # table_node_mapping = SQLTableNodeMapping(sql_database)
-        #
-        # nodes = table_node_mapping.to_nodes(table_schema_objs)
-        # for node in nodes:
-        #     node.metadata.update(metadata)
-        #
-        # docs.extend(nodes)
-
-        queries = entry.queries or []
         loader = CustomDatabaseReader(sql_database)
-        for query in queries:
+        for query_dict in entry.queries:
+            query = query_dict.get("sql", "")
+            explanation = query_dict.get("explanation", "")
             logger.info(f"Loading data from database with query: {query}")
-            documents = loader.load_data(query=query)
+            documents = loader.load_data(query=query, explanation=explanation)
 
             docs.extend(documents)
     return docs

backend/app/engine/prompt.py

@@ -39,15 +39,16 @@ refine_template_str = (
     "这是原本的问题： {query_str}\n"
     "我们已经提供了回答: {existing_answer}\n"
     "现在我们有机会改进这个回答 "
-    "使用以下更多上下文（仅当需要用时）\n"
+    "使用以下更多上下文（仅当有助于改进回答时使用）\n"
     "------------\n"
     "{context_msg}\n"
     "------------\n"
-    "根据新的上下文, 请改进原来的回答。"
+    "如果新的上下文对回答没有影响，或者原来的回答已经正确，直接返回原本的回答。\n"
-    "如果新的上下文没有用, 直接返回原本的回答。\n"
+    "如果新的上下文有助于改进，请基于它更新回答，但不要引入与问题无关的信息。\n"
-    "如果是表结构或者是数据库的相关内容，只用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
+    "如果是表结构或者是数据库的相关内容，仅用于推导问题，不需要告诉用户数据库或表结构等物理信息。\n"
     "改进的回答: "
 )
+
 refine_template = PromptTemplate(refine_template_str)
 
 summary_template_str = (