更新 .gitignore 文件以忽略特定 Excel 文件，删除多个不再需要的 Excel 文件，优化对话转工单处理类，添加用户问题列表解析功能，增强代码可读性和结构。

rag2_0/dify/dify_tool.py

@@ -160,7 +160,7 @@ class DifyTool:
     应用信息、消息详情以及工作流节点执行情况。
     """
     @staticmethod
-    def get_message_debug_info_id(message_id:str)->dict | None:
+    def get_message_debug_info_by_id(message_id:str)->dict | None:
         """
         根据消息 ID 从 'messages' 表中获取消息信息。
         """
@@ -178,7 +178,7 @@ class DifyTool:
     
 
     @staticmethod
-    def get_message_debug_info(appid:str, query:str)->dict:
+    def get_message_debug_info_by_query(appid:str, query:str)->dict:
         """
         获取指定应用和查询相关的调试信息。
 
@@ -212,4 +212,4 @@ class DifyTool:
 
 
 if __name__ == "__main__":
-    print(DifyTool.get_message_debug_info("ccf92b97-2789-4a3f-90e0-135a869a37c5", "电力建设计价通软件,导入结算后没有暂列金怎么办？要手动添加么？"))
+    print(DifyTool.get_message_debug_info_by_query("ccf92b97-2789-4a3f-90e0-135a869a37c5", "电力建设计价通软件,导入结算后没有暂列金怎么办？要手动添加么？"))

rag2_0/dify/test_dify_chatapi.py

@@ -9,12 +9,27 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 from rag2_0.dify.dify_tool import DifyTool
 import json
+from urllib.parse import unquote
+from rag2_0.tool.WikijsTool import WikijsTool
+from rag2_0.tool.html_to_md import convert_html_to_md
+from rag2_0.tool.ModelTool import OpenAiLLM
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+from langchain.output_parsers import PydanticOutputParser
+from threading import Lock
+
+load_dotenv()
+
+class ContentSource(BaseModel):
+    score:int = Field(description="相关性分数")
+    reason:str = Field(description="评分理由")
 
 class DifyComparisonTester:
     """
-    Dify新旧流程对比测试类，用于比较两个不同流程的问答效果
+    Dify新旧流程对比测试类，用于比较两个不同流程的问答效果并进行评判
     """
-    def __init__(self, excel_path:str, baseurl:str, old_workflow_api_key:str, new_workflow_api_key:str):
+    def __init__(self, excel_path:str, baseurl:str, old_workflow_api_key:str, new_workflow_api_key:str, 
+                 wiki_excel_path:str=None, output_path:str=None, max_workers:int=10):
         """
         初始化对比测试器
         
@@ -23,14 +38,33 @@ class DifyComparisonTester:
             baseurl: Dify API的基础URL
             old_workflow_api_key: 旧流程的API密钥
             new_workflow_api_key: 新流程的API密钥
+            wiki_excel_path: Wiki Excel文件路径，用于获取标准答案
+            output_path: 输出Excel文件路径
+            max_workers: 最大工作线程数
         """
         self.excel_path = excel_path
-        self.baseurl = baseurl
-        self.old_workflow_api_key = old_workflow_api_key
-        self.new_workflow_api_key = new_workflow_api_key
         self.old_chat = ChatClient(api_key=old_workflow_api_key, base_url=baseurl)
         self.new_chat = ChatClient(api_key=new_workflow_api_key, base_url=baseurl)
         
+        # 评判相关参数
+        self.output_path = output_path or os.path.join(os.path.dirname(self.excel_path), "dify问答_综合评判结果.xlsx")
+        self.max_workers = max_workers
+        self.content_source_parser = PydanticOutputParser(pydantic_object=ContentSource)
+        self.results_lock = Lock()
+        
+        # 读取Wiki Excel文件
+        if wiki_excel_path and os.path.exists(wiki_excel_path):
+            self.wiki_excel = pd.read_excel(wiki_excel_path)
+        else:
+            self.wiki_excel = None
+        
+
+    def get_llm(self):
+        api_key = os.getenv("OPENAI_API_KEY")
+        base_url = os.getenv("OPENAI_API_BASE")
+        model = os.getenv("LLM_MODEL_NAME")
+        return OpenAiLLM(api_key=api_key, base_url=base_url, model=model)
+
     def process_question(self, q:str):
         """
         处理单个问题，并行获取新旧流程的回答
@@ -62,40 +96,393 @@ class DifyComparisonTester:
                 new_result = future_new.result()
                 old_message_id = old_result["message_id"]
                 new_message_id = new_result["message_id"]
-                old_message_info = DifyTool.get_message_debug_info_id(message_id=old_message_id)
-                new_message_info = DifyTool.get_message_debug_info_id(message_id=new_message_id)
-                for workflow_node in new_message_info["workflow_node_executions_info"]:
-                    if workflow_node["title"] == "问题优化结果解析":
-                        outputs = json.loads(workflow_node["outputs"])  
-                        rewrite_query = outputs["optimize_query"]
+
                 old_answer = old_result["answer"]
                 new_answer = new_result["answer"]
             except Exception as e:
-                return None
-        return {"问题": q, "问题改写": rewrite_query, "旧流程答案": old_answer, "新流程答案": new_answer}
+                return None, None, None
+        return {"问题": q, "旧流程答案": old_answer, "新流程答案": new_answer}, old_message_id, new_message_id
         
-    def run_comparison(self):
+    def find_wiki_link(self, query) -> str | None:
+        """
+        根据查询找出对应的词条链接
+        
+        Args:
+            query (str): 查询内容
+            
+        Returns:
+            str: 对应的词条链接，如果没有找到则返回None
+        """
+        # 确保query不为空
+        if not query or pd.isna(query):
+            return None
+        if self.wiki_excel is None:
+            return None
+            
+        # 在"新提问"列中查找匹配的行
+        matched_rows = self.wiki_excel[self.wiki_excel['新提问'] == query]
+        
+        # 如果找到了匹配的行，返回对应的词条链接
+        if not matched_rows.empty:
+            return matched_rows.iloc[0]['对应词条链接']
+        
+        # 如果没有完全匹配，尝试部分匹配
+        # 去除软件名称部分（如果有）
+        query_parts = query.split(',', 1)
+        if len(query_parts) > 1:
+            clean_query = query_parts[1].strip()
+            
+            # 在"提问"列中查找包含清理后查询的行
+            for idx, row in self.wiki_excel.iterrows():
+                if pd.notna(row['提问']) and clean_query in row['提问']:
+                    return row['对应词条链接']
+        
+        return None
+
+    def get_wiki_content(self, link) -> str:
+        """
+        获取词条链接的内容
+        
+        Args:
+            link (str): 词条链接
+            
+        Returns:
+            str: 链接内容，如果获取失败则返回错误信息
+        """
+        try:
+            if not link or pd.isna(link):
+                return "链接为空或无效"
+            # 移除域名部分,只保留路径
+            path = link.split('/', 3)[-1]
+            decoded_path = unquote(path)
+            path_parts = decoded_path.split('/')
+            doc_path = "/".join(path_parts[1:])
+            wiki_doc = WikijsTool.get_all_doc_by_path(path=doc_path, path_is_dir=False)
+            html_content = WikijsTool.query_doc_info(wiki_doc[0]["id"]).get('content')
+            if not html_content:
+                return "获取内容失败"
+
+            options = {"heading_style": '', "keep_inline_images_in": ["figure", "img"], "escape_asterisks": True}
+            new_content = (html_content.replace("h6>", "h7>")
+                    .replace("h5>", "h6>")
+                    .replace("h4>", "h5>")
+                    .replace("h3>", "h4>")
+                    .replace("h2>", "h3>")
+                    .replace("h1>", "h2>"))
+            # 将HTML内容转换为Markdown
+            markdown_content = convert_html_to_md(new_content, "", **options)
+            markdown_content = f"# {path_parts[-1]}\n\n{markdown_content}"
+            return markdown_content
+            
+        except Exception as e:
+            raise RuntimeError(f"获取词条内容失败: {str(e)}") from e
+
+    def get_wiki_title(self, link) -> str | None:
+        """
+        获取词条标题
+        
+        Args:
+            link (str): 词条链接
+            
+        Returns:
+            str: 词条标题，如果获取失败则返回None
+        """
+        try:
+            if not link or pd.isna(link):
+                return None
+            # 移除域名部分,只保留路径
+            path = link.split('/', 3)[-1]
+            decoded_path = unquote(path)
+            path_parts = decoded_path.split('/')
+            return path_parts[-1]
+           
+        except Exception as e:
+            raise RuntimeError(f"获取词条内容失败: {str(e)}") from e
+
+    def create_correctness_prompt(self, standard_answer: str, answer_to_check: str) -> str:
+        """
+        创建用于评判答案正确性的prompt
+        
+        Args:
+            standard_answer (str): 标准答案
+            answer_to_check (str): 需要检查的答案
+            
+        Returns:
+            str: 格式化的prompt
+        """
+        return f"""请作为一个专业的答案评判专家，评估以下回答与标准答案的匹配程度。
+        
+标准答案：
+{standard_answer}
+
+待评估的回答：
+{answer_to_check}
+
+请仔细分析两个答案的内容，并给出你的判断。只需要回答"正确"或"错误"，不需要其他解释。
+如果待评估的回答与标准答案在核心内容和关键信息(步骤)上一致，即使表达方式不同，也应判定为"正确"。
+如果待评估的回答存在明显的错误信息或重要信息缺失，应判定为"错误"。
+
+请严格按以下格式输出：【正确】或【错误】："""
+
+    def judge_answer(self, standard_answer: str, answer: str) -> bool | None:
+        """
+        调用LLM判断回答是否正确
+        
+        Args:
+            standard_answer (str): 标准答案（来自Wiki）
+            answer (str): 需评判的回答
+            
+        Returns:
+            bool | None: 判断结果，True表示正确，False表示错误，None表示判断失败
+        """
+            
+        prompt = self.create_correctness_prompt(standard_answer, answer)
+        llm = self.get_llm()
+        try:
+            response = llm.invoke(user_prompt=prompt, need_retry=True)
+            return "正确" in response.content
+        except Exception as e:
+            return None
+
+    def judge_by_standard_answer(self, standard_answer: str, old_answer: str, new_answer: str) -> str | None:
+        """
+        综合判断新旧回答的正确性
+        
+        Args:
+            standard_answer (str): 标准答案（来自Wiki）
+            old_answer (str): 旧流程的回答
+            new_answer (str): 新流程的回答
+            
+        Returns:
+            str | None: 包含新旧回答判断结果的字符串，None表示判断失败
+        """
+        old_result = self.judge_answer(standard_answer, old_answer)
+        new_result = self.judge_answer(standard_answer, new_answer)
+        if old_result is None or new_result is None:
+            return None
+        if new_result and old_result:
+            return "新旧答案均正确"
+        elif new_result and not old_result:
+            return "新答案正确"
+        elif not new_result and old_result:
+            return "旧答案正确"
+        else:
+            return "新旧答案均错误"
+
+    def judge_answer_diff(self, old_answer: str, new_answer: str) -> str | None:
+        """
+        判断新旧回答是否存在较大差异
+        
+        Args:
+            old_answer (str): 旧流程的回答
+            new_answer (str): 新流程的回答
+            
+        Returns:
+            str | None: 差异判断结果，None表示判断失败
+        """
+            
+        prompt = f"""请判断以下两个回答是否存在较大差异:
+        
+        旧回答: {old_answer}
+        
+        新回答: {new_answer}
+        
+        主要是主要步骤、主要信息、或者主要主体的差异
+        请仅回答"存在较大差异"或"差异较小"。"""
+        llm = self.get_llm()
+        try:
+            response = llm.invoke(user_prompt=prompt, need_retry=True)
+            return "缺乏标准答案无法判断准确性,但答案差异较大" if "存在较大差异" in response.content else "缺乏标准答案无法判断准确性,但答案基本相同"
+        except Exception as e:
+            return None
+
+    def calculate_score(self, query:str, content:str) -> int:
+        """
+        使用LLM判断query与content之间的相关性分数
+        
+        Args:
+            query (str): 用户问题
+            content (str): 检索内容
+            
+        Returns:
+            int: 相关性分数，1-10分，10代表完全相关，1代表完全不相关；-1表示评分失败
+        """
+            
+        try:
+            prompt = f"""你是一个专业的信息相关性评估助手。请根据以下标准对用户query和检索内容的相关性进行1-10评分（10=完全相关，1=完全不相关），并按指定格式输出JSON结果。
+
+【评分标准】
+10分：完全契合，主题/意图完全一致且涵盖所有关键信息
+8-9分：高度相关，核心要素匹配但存在少量信息缺失
+6-7分：部分相关，涉及相同主题但存在重要信息缺失
+4-5分：弱相关，仅次要信息点匹配
+1-3分：完全不相关或信息冲突
+
+【评估维度】
+1. 主题一致性：核心主题/意图的匹配程度
+2. 内容覆盖度：是否涵盖query的关键要素
+3. 信息准确性：是否存在矛盾/错误信息
+4. 细节丰富度：是否提供query要求的详细信息
+
+【输出格式】
+{{
+  "score": 评分,
+  "reason": "简明扼要的评分理由（中文）"
+}}
+
+【示例】
+query: "新冠疫苗的常见副作用"
+内容: "辉瑞疫苗常见反应包括注射部位疼痛（84.1%）、疲劳（62.9%）"
+输出: {{"score":8,"reason":"主题完全匹配，涵盖主要副作用但未提及发热等常见反应"}}
+
+现在评估：
+query: "{query}"
+content: "{content}"
+"""
+            llm = self.get_llm()    
+            response = llm.invoke(user_prompt=prompt, need_retry=True)
+
+            # 解析JSON响应
+            try:
+                parsed_output = self.content_source_parser.parse(response.content)
+                return parsed_output.score
+            except Exception as e:
+                return -1
+        except Exception as e:
+            return -1
+        
+    def get_retrieve_info(self, query:str, outputs:dict) -> tuple:
+        """
+        获取检索信息并计算分数
+        
+        Args:
+            query (str): 用户问题
+            outputs (dict): 检索输出结果
+            
+        Returns:
+            tuple: (检索内容列表, 最高分, 最低分, 平均分)
+        """
+        max_score = 0
+        min_score = 10
+        total_score = 0
+        valid_scores = 0
+        retrieve_content = []
+        
+        for result in outputs["result"]:
+            content = result["content"].strip()
+            score = self.calculate_score(query=query, content=content)
+            if score != -1:
+                max_score = max(max_score, score)
+                min_score = min(min_score, score)
+                total_score += score
+                valid_scores += 1
+            content_title = content.split("\n")[0]
+            if content_title:
+                retrieve_content.append(content_title + f"--得分({score}分)")
+                
+        avg_score = total_score / valid_scores if valid_scores > 0 else 0
+        return retrieve_content, max_score, min_score, avg_score
+
+    def process_question_with_judge(self, q:str):
+        """
+        处理单个问题，获取新旧流程的回答并进行评判
+        
+        Args:
+            q: 问题内容
+            
+        Returns:
+            dict: 包含问题、回答和评判结果的字典
+        """
+        # 获取基本的问题和回答
+        basic_result, old_message_id, new_message_id = self.process_question(q)
+        if basic_result is None:
+            return None
+            
+        query = basic_result["问题"]
+        old_answer = basic_result["旧流程答案"]
+        new_answer = basic_result["新流程答案"]
+            
+        # 获取词条链接和标准答案
+        wiki_url = self.find_wiki_link(query)
+        standard_answer = ""
+        answer_title = ""
+        
+        try:
+            if wiki_url and not pd.isna(wiki_url):
+                standard_answer = self.get_wiki_content(wiki_url)
+                answer_title = self.get_wiki_title(wiki_url)
+        except Exception as e:
+            print(f"处理问题 '{query}' 获取标准答案时发生错误: {str(e)}")
+        
+        # 判断答案正确性
+        if standard_answer:
+            judge_result = self.judge_by_standard_answer(standard_answer, old_answer, new_answer)
+        else:
+            judge_result = self.judge_answer_diff(old_answer, new_answer)
+            
+        if judge_result is None:
+            judge_result = ""
+        
+        # 获取检索内容评分
+        retrieve_content = []
+        max_score = 0
+        min_score = 0
+        avg_score = 0
+        rewrite_query = ""
+        try:
+            new_message_info = DifyTool.get_message_debug_info_by_id(message_id=new_message_id)
+            for workflow_node in new_message_info["workflow_node_executions_info"]:
+                if workflow_node["title"] == "知识检索结果后处理":
+                    outputs = json.loads(workflow_node["outputs"])  
+                    retrieve_content, max_score, min_score, avg_score = self.get_retrieve_info(query=query, outputs=outputs)
+                elif workflow_node["title"] == "问题优化结果解析":
+                    outputs = json.loads(workflow_node["outputs"])  
+                    rewrite_query = outputs["optimize_query"]
+        except Exception as e:
+            print(f"处理问题 '{query}' 获取检索内容时发生错误: {str(e)}")
+        
+        # 返回结果
+        return {
+            "问题": query,
+            "问题改写": rewrite_query,
+            "旧流程答案": old_answer,
+            "新流程答案": new_answer,
+            "回答判断": judge_result,
+            "答案词条": answer_title if answer_title else "", 
+            "检索得到词条": "\n".join(retrieve_content) if retrieve_content else "未检索知识库",
+            "检索最高分": max_score,
+            "检索最低分": min_score,
+            "检索平均分": avg_score
+        }
+        
+    def run_comparison(self, with_judge=False):
         """
         运行对比测试，处理所有问题并生成结果Excel
         
+        Args:
+            with_judge: 是否进行答案评判
+            
         Returns:
             str: 输出Excel文件的路径
         """
         # 读取Excel文件中的问题
         df = pd.read_excel(self.excel_path)
-        questions = df.iloc[:,0].tolist()
+        questions = df['补全后的提问'].tolist()
         results = []
 
+        # 选择处理函数
+        process_func = self.process_question_with_judge if with_judge else self.process_question
+
         # 按顺序处理问题
         with tqdm(total=len(questions), desc="处理问题进度") as pbar:
             for q in questions:
-                result = self.process_question(q)
+                result = process_func(q)
                 if result is not None:
                     results.append(result)
                 pbar.update(1)
 
         # 生成输出Excel文件
-        out_path = os.path.join(os.path.dirname(self.excel_path), "dify问答_对比结果.xlsx")
+        out_path = self.output_path if with_judge else os.path.join(os.path.dirname(self.excel_path), "dify问答_对比结果.xlsx")
         df_results = pd.DataFrame(results)
         
         # 使用ExcelWriter设置格式
@@ -107,16 +494,16 @@ class DifyComparisonTester:
             worksheet = writer.sheets['Sheet1']
             
             # 设置列宽
-            worksheet.set_column('A:A', 50)  # 问题列宽 50个Excel单位
-            worksheet.set_column('B:B', 70)  # 旧流程答案列宽 70个Excel单位
-            worksheet.set_column('C:C', 70)  # 新流程答案列宽 70个Excel单位
+            for col_idx, col_name in enumerate(df_results.columns):
+                max_len = max(df_results[col_name].astype(str).map(len).max(), len(col_name))
+                worksheet.set_column(col_idx, col_idx, min(max_len + 2, 70))
             
         return out_path
 
 
 if __name__ == "__main__":
     # 定义Excel路径
-    excel_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".." ,"data/excel/历史提问数据(dislike)_1000条_软件明确.xlsx")
+    excel_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".." ,"data/excel/历史提问数据(like)_提问明确.xlsx")
     
     if not os.path.exists(excel_path):
         print(f"错误：Excel文件不存在: {excel_path}")
@@ -127,10 +514,21 @@ if __name__ == "__main__":
     old_workflow_api_key = "app-wUdkWJx5zeOvmvBUZizMoSw3"
     new_workflow_api_key = "app-Lf1pQ1NVwdMfCRVNTBCOTPHT"
     
+    # Wiki Excel路径和Dify应用ID（用于评判）
+    wiki_excel_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".." ,"data/excel/部分提问_软件名称明确.xlsx")
+    
     # 创建测试器并运行
-    tester = DifyComparisonTester(excel_path, baseurl, old_workflow_api_key, new_workflow_api_key)
-    output_file = tester.run_comparison()
-    print(f"对比结果已保存至: {output_file}")
+    tester = DifyComparisonTester(
+        excel_path=excel_path, 
+        baseurl=baseurl, 
+        old_workflow_api_key=old_workflow_api_key, 
+        new_workflow_api_key=new_workflow_api_key,
+        wiki_excel_path=wiki_excel_path,
+    )
+    
+    # 运行对比测试（带评判）
+    output_file = tester.run_comparison(with_judge=True)
+    print(f"对比评判结果已保存至: {output_file}")
 
 # 单个问题测试示例
 # c = DifyChat(baseurl="http://172.20.0.145/v1", api_key="app-LjJaeLoAfqa6aoGzqU9UvxSf")