更新日志目录创建逻辑，调整对话到工单的日期参数默认值，新增对话记录分析功能，优化API密钥管理器中的购买余额计算，并添加多个API密钥。同时，新增数据处理和分析模块以支持工单问答数据的上传和处理。

2025-07-15 09:27:44 +08:00
parent 5b5a2f2b16
commit 82724d206b
6 changed files with 163 additions and 62 deletions
@@ -0,0 +1,116 @@
+import pandas as pd
+import json
+
+from regex import search
+
+import ijson
+
+df = pd.read_excel("data/excel/已分析数据汇总（第一轮）.xlsx")
+df=df[df["评价"]=="dislike"]
+
+msg_id_list = df["msg_id"].tolist()
+msg_debug_list = {}
+# 流式解析 JSON 数组
+with open("data/excel/msg_debug_list.json", "r", encoding="utf-8") as f:
+    # 使用ijson.items直接获取顶层键值对
+    for msg_id, data in ijson.kvitems(f, ''):
+        if msg_id in msg_id_list:
+            msg_debug_list[msg_id] = data
+
+def get_rewrite_query(intent_node_execution_info)->str:
+    outputs_result =json.loads(intent_node_execution_info['outputs'])
+    return outputs_result['optimize_query']
+
+def judge_error_node_and_reason(intent_node_execution_info, knowledge_filter_node_execution_info_list, answer_wiki_name)->dict:
+    result = {"问题改写结果":None, "错误环节":None, "错误原因":None, "具体描述":None}
+    if answer_wiki_name is None or pd.isna(answer_wiki_name):
+        return result
+    
+    outputs_result =json.loads(intent_node_execution_info['outputs'])
+    result["问题改写结果"] = outputs_result['optimize_query']
+    if outputs_result['is_complete'] == False:
+        result["错误环节"] = "槽点填充"
+        result["错误原因"] = f"槽点缺失"
+        result["具体描述"] = f"缺失内容：{outputs_result['missing_slots']}"
+        return result
+
+    if len(knowledge_filter_node_execution_info_list) == 0:
+        return result
+
+    knowledge_filter_node_execution_info=knowledge_filter_node_execution_info_list[0]
+    # 获取检索到的所有词条
+    knowledge_filter_outputs = json.loads(knowledge_filter_node_execution_info['outputs'])
+    source_knowledge = knowledge_filter_outputs['source_kno']
+    source_knowledge_title ="\n".join([item['title'] for item in source_knowledge])
+    if answer_wiki_name not in source_knowledge_title:
+        result["错误环节"] = "知识检索"
+        result["错误原因"] = f"未检索到对应词条"
+
+        # 获取词条名称及对应评分
+        result["具体描述"] = "检索到的词条如下：\n"
+        for index, item in enumerate(source_knowledge):
+            result["具体描述"] += f"词条名称：{item['title'].split('/')[-1]}，重排评分：{item['metadata']['score']:.2f}\n"
+        return result
+    
+    # 获取检索到的词条的metadata
+    knowledge_filter = knowledge_filter_outputs['knowledge_list_metadata']
+    knowledge_filter_title ="\n".join([item['title'] for item in knowledge_filter])
+    if answer_wiki_name not in knowledge_filter_title:
+        result["错误环节"] = "知识过滤"
+        result["错误原因"] = f"词条被过滤"
+        result["具体描述"] = "检索到的词条如下：\n"
+        for index, item in enumerate(source_knowledge):
+            result["具体描述"] += f"词条名称：{item['title'].split('/')[-1]}，重排评分：{item['metadata']['score']:.2f}\n"
+        return result
+    
+    # 检索正确，回答错误
+    result["错误环节"] = "生成错误"
+    result["错误原因"] = f""
+    result["具体描述"] = f""
+    return result
+
+df["问题改写结果"] = None
+df["错误环节"] = None
+df["错误原因"] = None
+df["具体描述"] = None
+
+for index, row in df.iterrows():
+    try:
+        msg_id = row["msg_id"]
+        answer = row["回答"]
+        query = row["提问"]
+        rating = row["评价"]
+        class_type = row["问题分类"]
+        dislike_reason = row["点踩原因"]
+        if dislike_reason is None or pd.isna(dislike_reason):
+            continue
+        
+        answer_wiki_name = row["关联词条"]
+        search_wiki = row["检索到的词条"]
+        node_executions_info = msg_debug_list[msg_id]
+        intent_node_execution_info = [node_execution_info for node_execution_info in node_executions_info 
+                                    if node_execution_info["title"] == "意图识别结果解析"]
+        
+        knowledge_filter_node_execution_info_list = [node_execution_info for node_execution_info in node_executions_info 
+                                                if node_execution_info["title"] == "提取处理后的知识"]
+        if len(intent_node_execution_info) == 0:
+            print(f"msg_id: {msg_id} 缺少节点信息")
+            continue
+        
+        rewrite_query = get_rewrite_query(intent_node_execution_info[0])
+        df.loc[index, "问题改写结果"] = rewrite_query
+        if "有词条" not in dislike_reason:
+            continue
+        result = judge_error_node_and_reason(intent_node_execution_info[0], knowledge_filter_node_execution_info_list, answer_wiki_name)
+        for key, value in result.items():
+            df.loc[index, key] = value
+
+    except Exception as e:
+        print(f"msg_id: {msg_id} 处理失败: {e}")
+        continue
+
+df.to_excel("data/excel/已分析数据汇总（第一轮）_分析.xlsx", index=False)
+
+    
+
+
@@ -0,0 +1,29 @@
+import os
+import sys
+
+sys.path.append(os.getcwd())
+import rag2_0.dify.dify_client.dify_api as DifyApi
+
+import pandas as pd
+pd_data = pd.read_excel("data/excel/2025年5月30日到6月10号对话记录_转工单.xlsx")
+
+
+dify_api = DifyApi.DifyApi()
+dataset_id = dify_api.get_or_create_dataset_by_name("工单问答数据")
+document_id = dify_api.upload_text_to_document(text_name="5月30日到6月10号对话工单", text="", dataset_id=dataset_id)
+
+segments_list=[]
+for index, row in pd_data.iterrows():
+    query = row["客户问题"]
+    answer = row["解决方案"]
+    if "存在抱怨" in answer:
+        answer = answer.split("存在抱怨")[0]
+    
+    content = f"问题：{query}\n解决方案：{answer}"
+    segments_list.append({
+        "content": str(content),
+        "answer": "",
+        "keywords": []
+    })
+
+dify_api.add_document_segments(dataset_id=dataset_id, document_id=document_id, segments_list=segments_list)