ly/zjdataai-app
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

将属性图流程同步至主流程中

Browse Source
This commit is contained in:
wanyaokun
2024-09-14 16:35:46 +08:00
parent 6de6cc201e
commit 67b1959d80
8 changed files with 208 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.env.example
+5
View File
@@ -65,6 +65,10 @@ VECTOR_STORE_TYPE=chroma
# The name of the collection in your vector database
VECTOR_STORE_COLLECTION=default
#模型查询方式：graph、rag
LLM_QUERY_WAY = graph
#属性图存储类型:本地属性图库（默认），neo4j
GRAPH_STORE_TYPE =
#---------- neo4j - PropertyGraph ----------------
@@ -120,4 +124,5 @@ CHAT_UPLOAD_FILECACHE = "./output/uploaded"
JIEBA_DATA=./nltk_data
NLTK_DATA=./nltk_data
#IO流默认的编码格式
PYTHONUTF8=1
backend/app/engine/engine.py
+26 -2
View File
@@ -12,6 +12,8 @@ from app.engine.prompt import text_qa_template, refine_template, summary_templat
from app.engine.retriever.HybridRetriever import HybridRetriever
from app.engine.response.treeSummResponse import CustomTreeResponse
from llama_index.core.settings import Settings
from llama_index.core.indices.property_graph import LLMSynonymRetriever,VectorContextRetriever
from llama_index.core import PropertyGraphIndex
ModelPlateCategory = '模型平台'
@@ -117,10 +119,32 @@ def create_query_engine(index, top_k=3, use_reranker=False, filters=None, respon
if use_reranker:
postprocess = get_node_postprocessors()
query_engine = RetrieverQueryEngine.from_args(
get_Retriever(index,
llm_query = os.getenv('LLM_QUERY_WAY','rag')
if llm_query == 'graph':
graphIndex:PropertyGraphIndex = index
synonym_retriver = LLMSynonymRetriever(graphIndex.property_graph_store,
llm=Settings.llm,
include_text=False
)
if graphIndex.property_graph_store.supports_vector_queries:
vector_store = None
else:
vector_store = graphIndex.vector_store
vector_retriver = VectorContextRetriever(graphIndex.property_graph_store,
vector_store = vector_store,
embed_model=Settings.embed_model,
similarity_top_k=top_k,
include_text=False
)
retriever = graphIndex.as_retriever(sub_retrievers=[synonym_retriver,vector_retriver])
else:
retriever = get_Retriever(index,
similarity_top_k=top_k,
filters=filters),
query_engine = RetrieverQueryEngine.from_args(
retriever = retriever,
text_qa_template=text_qa_template,
refine_template=refine_template,
summary_template = summary_template,
backend/app/engine/generate.py
+7 -1
View File
@@ -103,10 +103,12 @@ class PropertyGraphChache:
def simplePropertyGraph(self,prjName:str,prjFlag:str,filePath:str):
documents = get_documents(prjFlag)
storeContext = StorageContext.from_defaults(vector_store=get_vector_store(prjFlag))
index = PropertyGraphIndex(
nodes =documents,
kg_extractors = [PrjGraphExtractor(prjName)],
embed_model = Settings.embed_model,
storage_context = storeContext,
show_progress= True
)
os.makedirs(filePath,exist_ok = True)
@@ -126,7 +128,11 @@ class PropertyGraphChache:
if __name__ == "__main__":
init_settings()
llm_query = os.getenv('LLM_QUERY_WAY','rag')
from phoenix.trace import using_project
with using_project(os.getenv("PHOENIX_PROJECT_NAME") + "_generate") as obj:
generate_datasource()
if llm_query == 'graph':
PropertyGraphChache().generate()
else:
generate_datasource()
backend/app/engine/graph/propertyGraph.py
+4 -4
View File
@@ -9,7 +9,7 @@ import os
from llama_index.core.storage.storage_context import StorageContext
from llama_index.core import load_index_from_storage
from app.observability import init_observability
from app.engine.vectordb import get_Neo4j_Graph_Store
from app.engine.vectordb import get_Neo4j_Graph_Store,get_vector_store
from llama_index.core.response_synthesizers import ResponseMode
from util.register import *
from llama_index.core.query_engine import RetrieverQueryEngine
@@ -44,7 +44,7 @@ class PropertyGraph:
prjCachePath = GRAPH_STORAGE_DIR + f"/{self._prjFlag}"
if not os.path.exists(prjCachePath):
return None
storeContext = StorageContext.from_defaults(persist_dir = prjCachePath)
storeContext = StorageContext.from_defaults(persist_dir = prjCachePath,vector_store = get_vector_store(self._prjFlag))
index = load_index_from_storage(storeContext)
return index
@@ -77,8 +77,8 @@ class PropertyGraph:
if __name__ == "__main__":
init_settings()
init_observability()
# graph = PropertyGraph('projects_1b20bbf4-3243-4ac3-bcf0-8a91e9157521')
# graph.query('代码为XLBT的金额是')
graph = PropertyGraph('projects_1b20bbf4-3243-4ac3-bcf0-8a91e9157521')
graph.query('代码为XLBT的金额是')
backend/app/engine/index.py
+26 -3
View File
@@ -1,15 +1,38 @@
import logging
import logging,os
from llama_index.core.indices import VectorStoreIndex
from app.engine.vectordb import get_vector_store
from app.engine.loaders import get_document_Types
from app.engine.vectordb import get_vector_store,get_Neo4j_Graph_Store
from typing import Dict,Any
from llama_index.core import PropertyGraphIndex
from llama_index.core.storage.storage_context import StorageContext
from llama_index.core import load_index_from_storage
logger = logging.getLogger("uvicorn")
def get_index(prjFlag:str):
if prjFlag is None or prjFlag == '':
raise ValueError('无效的工程标识')
logger.info("Connecting vector store...")
index = None
llm_query = os.getenv('LLM_QUERY_WAY')
if llm_query == 'graph':
index = getPropertyGraphIndex(prjFlag)
else:
store = get_vector_store(prjFlag)
index = VectorStoreIndex.from_vector_store(store)
logger.info("Finished load index from vector store.")
return index
def getPropertyGraphIndex(prjFlag:str):
GRAPH_STORE_TYPE = os.getenv("GRAPH_STORE_TYPE", "")
if GRAPH_STORE_TYPE == 'neo4j':
index = PropertyGraphIndex.from_existing(property_graph_store= get_Neo4j_Graph_Store(prjFlag))
else:
GRAPH_STORAGE_DIR = os.getenv("GRAPH_STORAGE_PATH", "storage_graph")
prjCachePath = GRAPH_STORAGE_DIR + f"/{prjFlag}"
if not os.path.exists(prjCachePath):
return None
storeContext = StorageContext.from_defaults(persist_dir = prjCachePath,vector_store = get_vector_store(prjFlag))
index = load_index_from_storage(storeContext)
return index
backend/app/engine/loaders/markDown.py
+6 -4
View File
@@ -12,7 +12,7 @@ class MarkDown:
colComments:list = []
ignores:List[str] = []
for name,fld in flds.items():
if name =='_id' or name =='nodeType' or name =='relTbId':
if self._table.name() == '工程属性' and (name =='_id' or name =='nodeType' or name =='relTbId'):
ignores.append(name)
continue
@@ -26,7 +26,9 @@ class MarkDown:
if col in ignores:
continue
txt:str = record.value(col)
datas.append(txt.replace('\n'," "))
content = txt.replace('\n',"")
content = content.replace('\r',"")
datas.append(content)
rowdatas.append(datas)
content = self.convert(self._table.name(),self._table.comment(),columns,colComments,rowdatas)
@@ -57,8 +59,8 @@ class MarkDown:
if __name__ == "__main__":
intputDir = ''
outputDir = ''
intputDir = 'C:\\Users\\wanyaokun\\Desktop\\markdown\\Project'
outputDir = 'C:\\Users\\wanyaokun\\Desktop\\markdown\\data'
subdirectories = {}
for dp, dn, fn in os.walk(intputDir):
backend/app/engine/vectordb.py
+4 -4
View File
@@ -8,11 +8,12 @@ qclient = None
def get_qdrant_vector_store(docType:str):
collection_name = docType
llm_query = os.getenv('LLM_QUERY_WAY','rag')
vector_store_path = os.getenv("VECTOR_STORE_PATH")
host=os.getenv("VECTOR_STORE_HOST", "127.0.0.1"),
port=int(os.getenv("VECTOR_STORE_PORT", "6333")),
vector_store_path =os.path.join(vector_store_path,docType)
vector_store_path =os.path.join(vector_store_path,llm_query,docType)
if not vector_store_path or not host:
raise ValueError(
"Please provide either VECTOR_STORE_PATH or VECTOR_STORE_HOST and VECTOR_STORE_PORT"
@@ -36,7 +37,8 @@ def get_qdrant_vector_store(docType:str):
def get_chroma_vector_store(docType:str):
collection_name = docType
vector_store_path =os.path.join(os.getenv("VECTOR_STORE_PATH"),docType)
llm_query = os.getenv('LLM_QUERY_WAY','rag')
vector_store_path =os.path.join(os.getenv("VECTOR_STORE_PATH"),llm_query,docType)
# if VECTOR_STORE_PATH is set, use a local ChromaVectorStore from the path
# otherwise, use a remote ChromaVectorStore (ChromaDB Cloud is not supported yet)
if vector_store_path:
@@ -59,7 +61,6 @@ def get_chroma_vector_store(docType:str):
def get_vector_store(docType:str):
store_type=os.getenv("VECTOR_STORE_TYPE")
store = None
match store_type:
@@ -72,7 +73,6 @@ def get_vector_store(docType:str):
return store
def get_Neo4j_Graph_Store(docType:str):
neo4jStore = Neo4jPropertyGraphStore(
username= os.getenv('NEO4J_USERNAME'),
backend/unit_test/Quetions/projects_0ffaf7fb-8a61-46e2-97a2-8f924e9560a7.json
+124
View File
@@ -0,0 +1,124 @@
{
"test:线路工程查询": [
{
"question": "基础工程项目划分合价",
"answer": "5099350万元 或者 49051649643元"
},
{
"question": "此工程名称",
"answer": "架线南网"
},
{
"question": "此工程电压等级",
"answer": "35kV"
},
{
"question": "基础工程项目划分合价",
"answer": "合价为 49051649642.9667 元"
},
{
"question": "基础工程项目划分合价，输出以万元为单位",
"answer": "合价总计为4905164.96429667万元"
},
{
"question": "建设场地征用及清理费是多少",
"answer": "建设场地征用及清理费的金额为16831284.228711元"
},
{
"question": "建设场地征用及清理费金额多少",
"answer": "建设场地征用及清理费的金额为16831284.228711元"
},
{
"question": "线路亘长",
"answer": "此工程的线路亘长为5.0公里。"
},
{
"question": "工程运距",
"answer": "工程运距包括：人力运距20km，汽车运距30km，拖拉机运距40km，船舶运距40km，海缆船舶运距50km，索道运距7km。"
},
{
"question": "基坑土石方单公里用量",
"answer": "81081630.354"
},
{
"question": "基本预备费费率",
"answer": "基本预备费的费率为2%。"
},
{
"question": "调差文件",
"answer": "调差文件指的是调差系数文件〔2020年14号文〕"
},
{
"question": "工程税率",
"answer": "工程税率是9.0%"
},
{
"question": "新建时间",
"answer": "2020/10/9"
},
{
"question": "工程静态投资，以万元为单位",
"answer": "71503585.3336万元"
},
{
"question": "耐张塔比例",
"answer": "耐张塔的比例为38.462%"
},
{
"question": "耐张杆比例",
"answer": "耐张杆的比例是72.222%。"
},
{
"question": "耐张杆的比例是如何计算出来的？",
"answer": "耐张杆的比例计算方式是将耐张杆基数除以耐张杆和直线杆的总基数，然后将结果转换为百分比。"
},
{
"question": "勘察设计费",
"answer": "勘察设计费的总金额为16164210209.43，其中包含勘察费12122154260.0和设计费4042055949.43。"
},
{
"question": "岩石比例",
"answer": "99"
},
{
"question": "人工挖孔",
"answer": "148"
},
{
"question": "余土运距",
"answer": "余土运距为187.0公里"
},
{
"question": "高压线（含10kV）",
"answer": "3处"
},
{
"question": "基坑普通土",
"answer": "313873965.334m³"
},
{
"question": "尖峰及施工基面普通土",
"answer": "尖峰及施工基面普通土的量为6534.528 m³"
},
{
"question": "节能评估费用",
"answer": "节能评估费用在电力工程造价中被标识为C1A，其费率设置为100.0%，但需要注意的是，在当前工程中此费用的金额为0.0。"
},
{
"question": "工程监理费",
"answer": "工程监理费的代码为B3，其费率是100.0，金额为131009.92。"
},
{
"question": "可行性研究文件评审费",
"answer": "可行性研究文件评审费的代码是C41，其金额为13340.0。"
},
{
"question": "接地工程合价",
"answer": "合价为 121964.914965元"
},
{
"question": "接地工程项目划分合价",
"answer": "合价为 121964.914965元"
}
]
}