Merge branch 'dev-web' of https://git.97id.com/ly/zjdataai-app into dev-web

backend/.env.example

@@ -26,6 +26,7 @@ RERANK_ENABLED=true
 RERANK_PROVIDER=ollama
 RERANK_MODEL= /models/bge-reranker-base
 RERANK_TOP_N=5
+RERANK_THRESHOLD=0.3
 
 #----------   model - Xinference    ----------------
 #MODEL_PROVIDER=xinference

backend/app/engine/engine.py

@@ -7,10 +7,26 @@ from llama_index.core.query_engine import RetrieverQueryEngine
 from llama_index.core.response_synthesizers import ResponseMode
 from llama_index.readers.database import DatabaseReader
 from sqlalchemy import create_engine
-
+from util.register import *
 from app.engine.prompt import text_qa_template, refine_template, summary_template, simple_template
 from app.engine.retriever.HybridRetriever import HybridRetriever
-from app.settings import get_node_postprocessors
+
+ModelPlateCategory = '模型平台'
+
+def get_node_postprocessors():
+    rerank_enabled = os.getenv("RERANK_ENABLED").title()
+    if rerank_enabled is None or rerank_enabled == 'False':
+        return []
+    
+    Rerank_provider = os.getenv("RERANK_PROVIDER")
+    modelPaltCls = ClsRegister.get(ModelPlateCategory,Rerank_provider)
+    postprocess = None
+    if modelPaltCls is not None:        
+        modelPalt = modelPaltCls()
+        postprocess =  modelPalt.rerank()
+    else:
+        raise ValueError(f"Invalid rerank provider: {Rerank_provider}")
+    return postprocess
 
 def makeDescriptionByEngine(sql_database:SQLDatabase):
     reader = DatabaseReader(sql_database)

backend/app/engine/generate.py

@@ -39,7 +39,7 @@ def run_pipeline(docstore, vector_store, documents):
                 #chunk_size=Settings.chunk_size,
                 #chunk_overlap=Settings.chunk_overlap,
             #),
-            MarkdownNodeParser(),
+            #MarkdownNodeParser(),
             Settings.embed_model,
         ],
         docstore=docstore,

backend/app/engine/loaders/markDown.py

@@ -0,0 +1,64 @@
+from app.engine.loaders.projectJson import *
+
+class MarkDown:
+    def __init__(self,table:JsonTable,path:str) -> None:
+        self._table = table
+        self._path = path
+
+    def build(self):        
+        flds:Dict[str,Field] = self._table.fields()
+        records:List[Record] = self._table.records()
+        columns:list = []
+        colComments:list = []
+        ignores:List[str] = []
+        for name,fld in flds.items():
+            if name =='_id' or  name =='nodeType' or name =='relTbId':
+                ignores.append(name)
+                continue
+
+            columns.append(fld.value('name'))
+            colComments.append(fld.value('alias'))
+
+        rowdatas = []
+        for record in records:
+            datas = []
+            for col in columns:
+                if col in ignores:
+                    continue
+                txt:str = record.value(col)
+                datas.append(txt.replace('\n'," "))
+            rowdatas.append(datas)
+        
+        content = self.convert(self._table.name(),self._table.comment(),columns,colComments,rowdatas)
+        with open(self._path, 'w',encoding='utf-8') as file:
+            file.write(content)
+
+    def convert(self,tableName:str,tableComment:str,columns:list,colComments:list,rowdatas:list):
+        strTitle = "# " + tableName + '\n'
+        if tableName!='':
+            strTitle+= f"备注：{tableComment}" + '\n'
+
+        for i in range(len(columns)):
+            strTitle+= f"- 字段名称：{columns[i]}" + '\n'
+            comment = colComments[i]
+            if comment!='':
+                strTitle+= f"  - 备注：{comment}" + '\n'
+
+        markdown_table = "|"
+        # 添加列标题
+        markdown_table += "|".join(columns) + "|\n"
+        # 添加分隔行
+        markdown_table += "|" + "|".join(['---' for _ in columns]) + "|\n"
+        # 遍历每个数据行
+        for row in rowdatas:
+            # 添加数据行
+            markdown_table += "|" + "|".join(row) + "|\n"
+        return strTitle + "\n" + markdown_table
+    
+
+prjSon = ProjectJson('')
+prjSon.parse()
+tables = prjSon.tables()
+for name,table in tables.items():
+    mdObj = MarkDown(table,f'')
+    mdObj.build()

backend/app/engine/loaders/markdownReader.py

@@ -0,0 +1,65 @@
+from llama_index.readers.file.markdown import MarkdownReader
+from typing import Any, Dict, List, Optional, Tuple
+import re
+from llama_index.core.utils import get_tokenizer
+
+
+class ChunkMarkdownReader(MarkdownReader):
+    def __init__(
+        self,
+        *args: Any,
+        chunkSize:int = 2048,
+        **kwargs: Any,
+    ) -> None:
+        self._chunkSize = chunkSize
+        self._tokenizer = get_tokenizer()
+        super().__init__(*args,**kwargs)
+
+    def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
+        markdown_tups: List[Tuple[Optional[str], str]] = []
+        lines = markdown_text.split("\n")
+
+        strTitle = ''
+        tokensNum:int = 0
+        current_lines = []
+        strheader:str = ''
+        headerSize:int = 0
+        for line in lines:  
+            tokensNum += self._token_size(line)
+            if tokensNum > self._chunkSize and len(current_lines) > 0:
+                if len(markdown_tups) == 0:
+                    markdown_tups.append((strTitle + strheader , "\n".join(current_lines)))
+                else:
+                    markdown_tups.append((strheader , "\n".join(current_lines)))
+                tokensNum =  headerSize
+                current_lines.clear()
+            current_lines.append(line)
+
+            if line == '\n' or line == '\r':
+                if tokensNum > self._chunkSize:
+                    raise ValueError('标题Token数大于chunkSize大小')
+                strTitle =  "\n".join(current_lines)
+                #headerSize = headerSize + self._token_size(strTitle)  
+                current_lines.clear()
+                        
+            if line.startswith("|---"):
+                strheader = "\n".join(current_lines) 
+                headerSize= headerSize + self._token_size(strheader)  
+                current_lines.clear() 
+
+        if len(current_lines) > 0:
+            if len(markdown_tups) == 0:
+                markdown_tups.append((strTitle + strheader , "\n".join(current_lines)))
+            else:
+                markdown_tups.append((strheader , "\n".join(current_lines)))
+
+        return [
+            (
+                key if key is None else re.sub(r"#", "", key).strip(),
+                re.sub(r"<.*?>", "", value),
+            )
+            for key, value in markdown_tups
+        ]
+
+    def _token_size(self, text: str) -> int:
+        return len(self._tokenizer(text))

backend/app/engine/loaders/projectJson.py

@@ -24,13 +24,16 @@ class JsonTable:
         self._filePth = filePth
         self._fields:Dict[str,Field] = {}
         self._records:List[Record] = []
+        self._fileName = os.path.splitext(os.path.basename(filePth))[0]
         self._name = ''
+        self._comment = ''
 
     def parse(self):
         with open(self._filePth, 'r',encoding='utf-8') as file:
             jsObj = json.load(file)
             data:dict = jsObj.get('table')
             self._name = data.get('name')
+            self._comment = data.get('comment')
             Jsfields = data.get('fields')
             for jsfiled in Jsfields:
                 field = Field(jsfiled)
@@ -42,6 +45,16 @@ class JsonTable:
 
     def records(self):
         return self._records
+    
+    def fields(self):
+        return self._fields
+    
+    def name(self):
+        return self._fileName
+    
+    def comment(self):
+        return self._comment
+
 
 class ProjectJson:
     def __init__(self,dir:str) -> None:
@@ -59,6 +72,9 @@ class ProjectJson:
 
     def table(self,tableName:str):
         return self._tables[tableName]
+    
+    def tables(self):
+        return self._tables
 
 def getProjectName(dir:str):
     prjJson = ProjectJson(dir)

backend/app/engine/rerank/ollamRerank.py

@@ -0,0 +1,70 @@
+from typing import Any, List, Optional
+from llama_index.core.postprocessor import SentenceTransformerRerank
+from llama_index.core.schema import MetadataMode, NodeWithScore, QueryBundle
+from llama_index.core.callbacks import CBEventType, EventPayload
+from llama_index.core.bridge.pydantic import PrivateAttr
+
+class OllamaRerank(SentenceTransformerRerank):
+    _score_threshold: float = PrivateAttr()
+    def __init__(
+        self,
+        top_n: int = 2,
+        model: str = "cross-encoder/stsb-distilroberta-base",
+        device: Optional[str] = None,
+        keep_retrieval_score: Optional[bool] = False,
+        score_threshold:float = 0.3
+    ):
+        self._score_threshold = score_threshold
+        super().__init__(top_n,model,device,keep_retrieval_score)
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "OllamaRerank"
+
+    def _postprocess_nodes(
+        self,
+        nodes: List[NodeWithScore],
+        query_bundle: Optional[QueryBundle] = None,
+    ) -> List[NodeWithScore]:
+        if query_bundle is None:
+            raise ValueError("Missing query bundle in extra info.")
+        if len(nodes) == 0:
+            return []
+
+        query_and_nodes = [
+            (
+                query_bundle.query_str,
+                node.node.get_content(metadata_mode=MetadataMode.EMBED),
+            )
+            for node in nodes
+        ]
+
+        with self.callback_manager.event(
+            CBEventType.RERANKING,
+            payload={
+                EventPayload.NODES: nodes,
+                EventPayload.MODEL_NAME: self.model,
+                EventPayload.QUERY_STR: query_bundle.query_str,
+                EventPayload.TOP_K: self.top_n,
+            },
+        ) as event:
+            scores = self._model.predict(query_and_nodes)
+
+            assert len(scores) == len(nodes)
+
+            for node, score in zip(nodes, scores):
+                if self.keep_retrieval_score:
+                    node.node.metadata["retrieval_score"] = node.score
+                node.score = score
+
+            for i in range(len(nodes)-1,-1,-1):
+                node = nodes[i]
+                if node.score  < self._score_threshold:
+                    nodes.remove(node)
+
+            new_nodes = sorted(nodes, key=lambda x: -x.score if x.score else 0)[
+                : self.top_n
+            ]
+            event.on_end(payload={EventPayload.NODES: new_nodes})
+
+        return new_nodes

backend/app/settings.py

@@ -18,21 +18,6 @@ from llama_index.core.callbacks import CallbackManager
 
 ModelPlateCategory = '模型平台'
 
-def get_node_postprocessors():
-    rerank_enabled = os.getenv("RERANK_ENABLED").title()
-    if rerank_enabled is None or rerank_enabled == 'False':
-        return []
-    
-    Rerank_provider = os.getenv("RERANK_PROVIDER")
-    modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,Rerank_provider)
-    postprocess = None
-    if modelPaltCls is not None:        
-        modelPalt:ModelPlatform = modelPaltCls()
-        postprocess =  modelPalt.rerank()
-    else:
-        raise ValueError(f"Invalid rerank provider: {Rerank_provider}")
-    return postprocess
-
 def init_settings():
     model_provider = os.getenv("MODEL_PROVIDER")
     modelPaltCls:ModelPlatform = ClsRegister.get(ModelPlateCategory,model_provider)
@@ -91,7 +76,18 @@ class OllamaPlatform(ModelPlatform):
         pass
 
     def rerank(self):
-        pass
+        from app.engine.rerank.ollamRerank import OllamaRerank
+        modelpath = os.getcwd() + os.getenv('RERANK_MODEL')
+        top_n = os.getenv('RERANK_TOP_N',5)
+        threshold =  float(os.getenv('RERANK_THRESHOLD',0.3))
+        rerank = OllamaRerank(
+            model=modelpath,
+            top_n=top_n, 
+            device="cpu",
+            score_threshold= threshold
+        )
+        return [rerank]
+
 
 @register(ModelPlateCategory,'xinference')
 class XinferencePlatform(ModelPlatform):

backend/data/projects/1b20bbf4-3243-4ac3-bcf0-8a91e9157521/余物清理安装_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{AC732533-7DF2-4AE5-B7E5-F44567D78364}","nodeType":"取费表","parentId":"","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"FFZ","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接费"},{"_id":"{634B13BE-3DD0-451B-A6C9-0C9A317CEDA8}","nodeType":"取费表","parentId":"{AC732533-7DF2-4AE5-B7E5-F44567D78364}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"FFZ1","取费基数":"","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{A5043C5F-A326-42D0-8EA7-D8ED1D64495E}","nodeType":"取费表","parentId":"{634B13BE-3DD0-451B-A6C9-0C9A317CEDA8}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"DZF","取费基数":"","备注":"","序号":"1.1","是否隐藏":"否","费率":"100","费用名称":"定额直接费"},{"_id":"{6CE02AA1-5AF2-415C-8120-87B69633E7A6}","nodeType":"取费表","parentId":"{A5043C5F-A326-42D0-8EA7-D8ED1D64495E}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1.1.1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{D06FC2F6-D089-430F-9898-F222D85FF87C}","nodeType":"取费表","parentId":"{A5043C5F-A326-42D0-8EA7-D8ED1D64495E}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"CLF","取费基数":"取费定额乙供材料费不含税+取费定额甲供材料费含税","备注":"","序号":"1.1.2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{A5BAAA84-B337-43F1-BD8B-4C13F1B15A04}","nodeType":"取费表","parentId":"{A5043C5F-A326-42D0-8EA7-D8ED1D64495E}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"1.1.3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{3A5715D5-42FB-41A5-937A-D231277D9916}","nodeType":"取费表","parentId":"{AC732533-7DF2-4AE5-B7E5-F44567D78364}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"FFZ2","取费基数":"","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"措施费"},{"_id":"{AB35557D-593C-4D66-AD1B-17359227B6B5}","nodeType":"取费表","parentId":"{3A5715D5-42FB-41A5-937A-D231277D9916}","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"BZF","取费基数":"FFZ1-甲供消材进项税额","备注":"","序号":"2.1","是否隐藏":"否","费率":"2.93","费用名称":"安全文明施工费"},{"_id":"{15CC6A96-C81A-4AAE-A212-B3DCF573443D}","nodeType":"取费表","parentId":"","relTbId":"{6C5274C1-2091-440D-8E3F-94FA30CE65A2}","代码":"HJ","取费基数":"FFZ+一笔性费用","备注":"","序号":"二","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/1b20bbf4-3243-4ac3-bcf0-8a91e9157521/余物清理建筑_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{1C15B4B2-3B33-40F9-8C1A-F16A585894B4}","nodeType":"取费表","parentId":"","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"FFZ","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接费"},{"_id":"{5E7ADB7A-9C41-4813-89D3-8915182FD91E}","nodeType":"取费表","parentId":"{1C15B4B2-3B33-40F9-8C1A-F16A585894B4}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"FFZ1","取费基数":"","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{80745DAE-6C80-497B-B46A-9B0AD665847A}","nodeType":"取费表","parentId":"{5E7ADB7A-9C41-4813-89D3-8915182FD91E}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1.1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{A7D785CA-D360-4D17-9740-436534F84150}","nodeType":"取费表","parentId":"{5E7ADB7A-9C41-4813-89D3-8915182FD91E}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"CLF","取费基数":"取费定额乙供材料费不含税+取费定额甲供材料费含税","备注":"","序号":"1.2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{2680555F-1FC1-4AFE-BC80-12A16587675B}","nodeType":"取费表","parentId":"{5E7ADB7A-9C41-4813-89D3-8915182FD91E}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"1.3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{1CDC9898-AD38-4B46-BDAC-870F4AC6BE52}","nodeType":"取费表","parentId":"{1C15B4B2-3B33-40F9-8C1A-F16A585894B4}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"FFZ2","取费基数":"","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"措施费"},{"_id":"{B3C816DB-A50E-49B9-B861-98F358A8AD11}","nodeType":"取费表","parentId":"{1CDC9898-AD38-4B46-BDAC-870F4AC6BE52}","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"BZF","取费基数":"FFZ1-甲供消材进项税额","备注":"","序号":"2.1","是否隐藏":"否","费率":"2.93","费用名称":"安全文明施工费"},{"_id":"{A74D1DBF-FA60-4C00-9F2C-659CAB97AD3C}","nodeType":"取费表","parentId":"","relTbId":"{96A61AA6-7110-477F-BD0B-CC0F56658DFE}","代码":"HJ","取费基数":"FFZ+一笔性费用","备注":"","序号":"二","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/1b20bbf4-3243-4ac3-bcf0-8a91e9157521/建筑_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{F68A4F69-59D3-4184-96BE-7B6F662BC60E}","nodeType":"取费表","parentId":"","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"FFZ1","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{12EB546B-D14C-4FFA-9029-54ACEE399D6E}","nodeType":"取费表","parentId":"{F68A4F69-59D3-4184-96BE-7B6F662BC60E}","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{19E92F99-E1E9-4745-8EA2-F525AF331FB0}","nodeType":"取费表","parentId":"{F68A4F69-59D3-4184-96BE-7B6F662BC60E}","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"CLF","取费基数":"取费定额乙供材料费不含税+取费定额甲供材料费含税+乙供取费主材费不含税+甲供取费主材费含税","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{05F2241D-8919-4A1C-96D0-1091DD079E3B}","nodeType":"取费表","parentId":"{F68A4F69-59D3-4184-96BE-7B6F662BC60E}","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{EB338420-823E-47B7-B8DE-640935812A5B}","nodeType":"取费表","parentId":"","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"TZHQF","取费基数":"FFZ1-甲供消材进项税额-甲供主材进项税额","备注":"综合取费费用额（包含措施费、间接费、利润）","序号":"二","是否隐藏":"否","费率":"16.59","费用名称":"综合取费费用额"},{"_id":"{C5834D3B-A4FB-4AEC-812E-B2069F136041}","nodeType":"取费表","parentId":"","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"TFFS","取费基数":"FFZ1+TZHQF-取费定额甲供材料费含税-甲供取费主材费含税+不取费定额费不含税-不取费甲供材料费不含税+不取费乙供主材费不含税","备注":"","序号":"三","是否隐藏":"否","费率":"9","费用名称":"税金"},{"_id":"{71DECC0F-AB3D-4230-9A30-7FC4096BF99C}","nodeType":"取费表","parentId":"","relTbId":"{D8E27F1A-75C4-4005-B49F-BD3402874FB4}","代码":"THJ","取费基数":"FFZ1+TZHQF+TFFS+一笔性费用+不取费定额费不含税+不取费乙供主材费不含税+不取费甲供材料进项税额+不取费甲供主材费含税","备注":"","序号":"四","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/4406164d-b08e-446e-9dba-b544346a0c54/余物清理_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{42DAB3FC-51DF-47E9-8991-7642C6EE885C}","nodeType":"取费表","parentId":"","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"FFZ","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接费"},{"_id":"{80FF74E3-2DA7-46FE-8298-B7A6EE2D0703}","nodeType":"取费表","parentId":"{42DAB3FC-51DF-47E9-8991-7642C6EE885C}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"FFZ1","取费基数":"","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{6ABF558C-0DB4-47C3-9BB6-7ABCBEA910C2}","nodeType":"取费表","parentId":"{80FF74E3-2DA7-46FE-8298-B7A6EE2D0703}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"DZF","取费基数":"","备注":"","序号":"1.1","是否隐藏":"否","费率":"100","费用名称":"定额直接费"},{"_id":"{DDC61D9C-A38B-4D90-90C4-3266EBDE4D77}","nodeType":"取费表","parentId":"{6ABF558C-0DB4-47C3-9BB6-7ABCBEA910C2}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1.1.1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{51CF5ACE-E686-4522-9879-32595AE23830}","nodeType":"取费表","parentId":"{6ABF558C-0DB4-47C3-9BB6-7ABCBEA910C2}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"CLF","取费基数":"取费定额乙供材料费不含税","备注":"","序号":"1.1.2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{A2B54828-4789-4C20-AD42-4DE7F6176747}","nodeType":"取费表","parentId":"{6ABF558C-0DB4-47C3-9BB6-7ABCBEA910C2}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"1.1.3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{02AC3342-44E9-4B16-A718-67A56F6619FF}","nodeType":"取费表","parentId":"{42DAB3FC-51DF-47E9-8991-7642C6EE885C}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"FFZ2","取费基数":"","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"措施费"},{"_id":"{30CFC3DF-FF64-4870-9A93-9EA9F344213A}","nodeType":"取费表","parentId":"{02AC3342-44E9-4B16-A718-67A56F6619FF}","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"BZF","取费基数":"FFZ1","备注":"","序号":"2.1","是否隐藏":"否","费率":"3.25","费用名称":"安全文明施工费"},{"_id":"{E78003F1-AA06-4BF1-BA05-23CBBB4367A0}","nodeType":"取费表","parentId":"","relTbId":"{02A4F179-28A3-476E-9697-CD560E240F7A}","代码":"HJ","取费基数":"FFZ+一笔性费用","备注":"","序号":"二","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/4406164d-b08e-446e-9dba-b544346a0c54/线路_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{D762E40C-F7C1-4A8E-B091-B4BD26454476}","nodeType":"取费表","parentId":"","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"FFZ1","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{82C58340-54B2-4EDB-A62F-A7F786D59B78}","nodeType":"取费表","parentId":"{D762E40C-F7C1-4A8E-B091-B4BD26454476}","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{DE1DDC41-13C3-416B-AD08-253EFAF7FF40}","nodeType":"取费表","parentId":"{D762E40C-F7C1-4A8E-B091-B4BD26454476}","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"CLF","取费基数":"取费定额乙供材料费不含税+乙供取费主材费不含税+甲供取费主材费含税","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{28A95C2F-5516-429E-BAB3-36B6DD9390FC}","nodeType":"取费表","parentId":"{D762E40C-F7C1-4A8E-B091-B4BD26454476}","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{D27F3977-1185-4DAD-A041-186036503DAF}","nodeType":"取费表","parentId":"","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"TZHQF","取费基数":"FFZ1-甲供主材进项税额","备注":"综合取费费用额（包含措施费、间接费、利润）","序号":"二","是否隐藏":"否","费率":"17.79","费用名称":"综合取费费用额"},{"_id":"{695C2D0E-F688-489C-BA71-EABF73E98E41}","nodeType":"取费表","parentId":"","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"TFFS","取费基数":"FFZ1+TZHQF-甲供取费主材费含税+不取费定额费不含税+不取费乙供主材费不含税","备注":"","序号":"三","是否隐藏":"否","费率":"9","费用名称":"税金"},{"_id":"{C91A22B6-896A-4040-948D-DFC25A8D0C77}","nodeType":"取费表","parentId":"","relTbId":"{DCBA7CA2-95BA-458A-BF34-7163010FFEAA}","代码":"THJ","取费基数":"FFZ1+TZHQF+TFFS+一笔性费用+不取费定额费不含税+不取费乙供主材费不含税+不取费甲供主材费含税","备注":"","序号":"四","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/d48f81ce-2727-4128-b795-31fda2e1d88b/余物清理_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{AE260A6F-EF67-4F66-9AA2-742A11A39C78}","nodeType":"取费表","parentId":"","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"FFZ","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接费"},{"_id":"{2EA0D884-AB7F-478B-BCCE-212BF1A5AC87}","nodeType":"取费表","parentId":"{AE260A6F-EF67-4F66-9AA2-742A11A39C78}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"FFZ1","取费基数":"","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{62F8641B-D629-40CD-AB49-34B7FB702E98}","nodeType":"取费表","parentId":"{2EA0D884-AB7F-478B-BCCE-212BF1A5AC87}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"DZF","取费基数":"","备注":"","序号":"1.1","是否隐藏":"否","费率":"100","费用名称":"定额直接费"},{"_id":"{B9DB9995-8BF9-4769-AB7B-E989B4907A08}","nodeType":"取费表","parentId":"{62F8641B-D629-40CD-AB49-34B7FB702E98}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1.1.1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{6F2D0FFB-CAF5-43CD-90EB-2C64A21E6127}","nodeType":"取费表","parentId":"{62F8641B-D629-40CD-AB49-34B7FB702E98}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"CLF","取费基数":"取费定额乙供材料费不含税","备注":"","序号":"1.1.2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{4C54B793-5D91-4762-8813-115218DC72F8}","nodeType":"取费表","parentId":"{62F8641B-D629-40CD-AB49-34B7FB702E98}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"1.1.3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{DBF98998-59F5-4BD6-8575-51CDD2F3EF65}","nodeType":"取费表","parentId":"{AE260A6F-EF67-4F66-9AA2-742A11A39C78}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"FFZ2","取费基数":"","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"措施费"},{"_id":"{1A6B97A1-F64F-4062-9344-92F9DBFEA27C}","nodeType":"取费表","parentId":"{DBF98998-59F5-4BD6-8575-51CDD2F3EF65}","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"BZF","取费基数":"FFZ1","备注":"","序号":"2.1","是否隐藏":"否","费率":"3.55","费用名称":"安全文明施工费"},{"_id":"{61FE5B00-8DA5-4FA5-A6B5-67A506CABBAC}","nodeType":"取费表","parentId":"","relTbId":"{DE579B44-20F0-4A84-A620-3C8D211C8F16}","代码":"HJ","取费基数":"FFZ+一笔性费用","备注":"","序号":"二","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/data/projects/d48f81ce-2727-4128-b795-31fda2e1d88b/线路_取费表.json

@@ -1 +0,0 @@
-{"table":{"alias":"","comment":"取费表是取费设置中各取费表明细。查询示例: SELECT Rate FROM FeeCollectionTable WHERE Name = 'findname'。","fields":[{"alias":"","comment":"","name":"_id","type":"VARCHAR"},{"alias":"编号，代码，代号","comment":"费用代码","name":"Code","type":"VARCHAR"},{"alias":"公式，计算式，表达式","comment":"取费基数","name":"CalculationFormula","type":"VARCHAR"},{"alias":"备注，说明","comment":"费用项备注说明","name":"Remarks","type":"VARCHAR"},{"alias":"费用序号，序号，序列号","comment":"费用表序号","name":"SerialNumber","type":"VARCHAR"},{"alias":"","comment":"","name":"是否隐藏","type":"VARCHAR"},{"alias":"费用利率，费率","comment":"取费费率","name":"Rate","type":"REAL"},{"alias":"项目名，费用名，名称","comment":"费用名称，项目名称","name":"Name","type":"VARCHAR"},{"alias":"","comment":"","name":"nodeType","type":"VARCHAR"},{"alias":"","comment":"","name":"parentId","type":"VARCHAR"},{"alias":"","comment":"","name":"relTbId","type":"VARCHAR"}],"name":"FeeCollectionTable","records":[{"_id":"{ECA21A14-0907-4136-8DCC-B517973334C2}","nodeType":"取费表","parentId":"","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"FFZ1","取费基数":"","备注":"","序号":"一","是否隐藏":"否","费率":"100","费用名称":"直接工程费"},{"_id":"{740AA893-1750-4ECB-B39F-281A67A1E535}","nodeType":"取费表","parentId":"{ECA21A14-0907-4136-8DCC-B517973334C2}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"RGF","取费基数":"取费定额人工费","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"人工费"},{"_id":"{D5ED3338-0D3C-40C6-9578-7664DF58DFE0}","nodeType":"取费表","parentId":"{ECA21A14-0907-4136-8DCC-B517973334C2}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"CLF","取费基数":"取费定额乙供材料费不含税+乙供取费主材费不含税+甲供取费主材费含税","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"材料费"},{"_id":"{EF4424BB-6DA4-4C71-AA3F-0B3A32C2F57D}","nodeType":"取费表","parentId":"{ECA21A14-0907-4136-8DCC-B517973334C2}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"JXF","取费基数":"取费定额机械费","备注":"","序号":"3","是否隐藏":"否","费率":"100","费用名称":"施工机械使用费"},{"_id":"{9BEB4BE3-56A4-4E59-B0F9-912856B078A5}","nodeType":"取费表","parentId":"","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"TZHQF","取费基数":"FFZ1-甲供主材进项税额","备注":"综合取费费用额（包含措施费、间接费、利润）","序号":"二","是否隐藏":"否","费率":"17.79","费用名称":"综合取费费用额"},{"_id":"{463D4D32-D1F0-4A43-B912-5C9B83B4083E}","nodeType":"取费表","parentId":"","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"JC","取费基数":"","备注":"","序号":"三","是否隐藏":"否","费率":"100","费用名称":"编制基准期价差"},{"_id":"{26405AAE-BE51-42E1-8A3A-8E0162C48364}","nodeType":"取费表","parentId":"{463D4D32-D1F0-4A43-B912-5C9B83B4083E}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"RJC","取费基数":"人工价差","备注":"","序号":"1","是否隐藏":"否","费率":"100","费用名称":"人工价差"},{"_id":"{E65ED1F7-384E-4047-891E-79B7A668B42C}","nodeType":"取费表","parentId":"{463D4D32-D1F0-4A43-B912-5C9B83B4083E}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"CJC","取费基数":"乙供材料价差不含税+乙供主材价差不含税+甲供主材价差含税","备注":"","序号":"2","是否隐藏":"否","费率":"100","费用名称":"材料价差"},{"_id":"{A0CD4285-8D60-435A-9CD7-AAF3E249347D}","nodeType":"取费表","parentId":"{463D4D32-D1F0-4A43-B912-5C9B83B4083E}","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"JJC","取费基数":"机械价差","备注":"","序号":"3","是否隐藏":"否","费率":"100","费用名称":"机械价差"},{"_id":"{8AF46722-7815-4A79-A6AA-2AA7D37848CD}","nodeType":"取费表","parentId":"","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"TFFS","取费基数":"FFZ1+TZHQF+JC-甲供取费主材费含税-甲供主材价差含税+不取费定额费不含税+不取费乙供主材费不含税","备注":"","序号":"四","是否隐藏":"否","费率":"9","费用名称":"税金"},{"_id":"{881D2F8E-691E-4D29-88C3-C19AD425AECB}","nodeType":"取费表","parentId":"","relTbId":"{0256D22B-7007-4520-B09E-928BB6A8D16A}","代码":"THJ","取费基数":"FFZ1+TZHQF+TFFS+JC+一笔性费用+不取费定额费不含税+不取费乙供主材费不含税+不取费甲供主材费含税","备注":"","序号":"五","是否隐藏":"否","费率":"100","费用名称":"合计"}]}}

backend/modelProvide/customDashScope.py

@@ -9,7 +9,8 @@ class DashScopeGenerationModels:
     QWEN_MAX = "qwen-max"
     QWEN_MAX_1201 = "qwen-max-1201"
     QWEN_MAX_LONGCONTEXT = "qwen-max-longcontext"
-    QWEN2_MATH_72B_INSTRUCT = 'qwen2-math-72b-instruct'
+    QWEN2_MATH_72B_INSTRUCT = 'qwen2-math-72b-instruct',
+    QWEN2_72B = 'qwen2-72b-instruct'
 
 DASHSCOPE_MODEL_META = {
     DashScopeGenerationModels.QWEN_TURBO: {
@@ -42,6 +43,11 @@ DASHSCOPE_MODEL_META = {
         "num_output": 1024 * 8,
         "is_chat_model": True,
     },
+    DashScopeGenerationModels.QWEN2_72B: {
+        "context_window": 1024 * 2,
+        "num_output": 1024 * 8,
+        "is_chat_model": True,
+    },
 }
 
 

backend/pyproject.toml

@@ -94,4 +94,6 @@ priority = "primary"
 
 [build-system]
 requires = [ "poetry-core" ]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"
+
+tiktoken