ouyangyouzhang/QueryRewrite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整启动脚本的权限，优化screen会话检查逻辑，修复DifyQueryRetrieval_api.py中的top_k参数，删除IntentRecognition.py中的冗余代码，优化异步处理流程，移除不必要的注释和日志记录。

Browse Source
This commit is contained in:
ouyangyouzhang
2025-07-25 13:55:06 +08:00
parent 2cbdc23fc0
commit 3f6f5d038c
6 changed files with 37 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rag2_0/dify/DifyQueryRetrieval.py
+1 -2
View File
@@ -189,8 +189,7 @@ class DifyQueryRetrieval:
if doc_title not in hit_doc_titles_set: # 确保不添加重复的标题
hit_doc_titles_set.add(doc_title)
query_hit_stats[query_type].append(doc_title)
logging.info(f"查询命中统计: {json.dumps(query_hit_stats, ensure_ascii=False)}")
return {
"documents": processed_documents,
rag2_0/dify/DifyQueryRetrieval_api.py
+1 -1
View File
@@ -99,7 +99,7 @@ async def retrieve(request: RetrieveRequest):
query_list,
data_set_list,
query_expand_dict=query_expand_dict,
top_k=5
top_k=4
)
end_time = time.time()
rag2_0/intent_recognition/IntentRecognition.py
+25 -127
View File
@@ -22,14 +22,14 @@ import threading
from .PromptTemplates import (classification_prompt, query_rewrite_prompt_pro,
extract_nouns_prompt, classification_info,
slot_filling_prompt, step_back_prompt,
follow_up_questions_prompt, hyde_prompt, multi_questions_prompt)
hyde_prompt)
from .DataModels import (
Classification, QueryRewrite, Term, TermList,
SoftwareFunctionSlots, SoftwareTroubleShootingSlots, ProfessionalConsultingSlots,
DataProblemSlots, FileExtensionConsultingSlots, SoftwareLockSlots,
InstallationDownloadSlots, ProblemDiagnosisSlots, OtherSlots, IntentAndSlotResult,
StepBackPrompt, FollowUpQuestions, HypotheticalDocument, MultiQuestions
StepBackPrompt, HypotheticalDocument
)
from .ProfessionalNounVector import ProfessionalNounRetriever, AsyncProfessionalNounRetriever
from rag2_0.tool.ModelTool import XinferenceReRankerModel, OpenAiLLM, SiliconFlowReRankerModel
@@ -147,7 +147,6 @@ class AsyncIntentRecognizer:
Returns:
分类结果
"""
classification_start_time = time.time()
classification_parser = PydanticOutputParser(pydantic_object=Classification)
formatted_prompt = classification_prompt.format(user_input=query,
classification_info=classification_info,
@@ -159,10 +158,6 @@ class AsyncIntentRecognizer:
# 异步调用LLM
response = await self._llm.invoke_async(formatted_prompt, False)
classification_end_time = time.time()
classification_time = classification_end_time - classification_start_time
logging.info(f"异步意图分类耗时统计 - 总耗时: {classification_time:.2f}秒")
# 尝试直接解析JSON响应
response.content = response.content.strip()
clean_output = re.sub(r'<think>.*?</think>', '', response.content, flags=re.DOTALL)
@@ -356,7 +351,6 @@ class AsyncIntentRecognizer:
改写结果
"""
rewrite_start_time = time.time()
# 准备问题改写提示
terms_dict = [term.model_dump(exclude={"description"}) for term in keywords.terms]
keywords_str = json.dumps(terms_dict, ensure_ascii=False)
@@ -373,9 +367,6 @@ class AsyncIntentRecognizer:
response.content = response.content.strip()
clean_output = re.sub(r'<think>.*?</think>', '', response.content, flags=re.DOTALL)
parsed_output = query_rewrite_parser.parse(clean_output)
rewrite_end_time = time.time()
rewrite_time = rewrite_end_time - rewrite_start_time
logging.info(f"异步问题改写耗时统计 - 总耗时: {rewrite_time:.2f}秒")
return parsed_output
except Exception as e:
raise RuntimeError(f"解析问题改写结果时出错: {e}") from e
@@ -393,11 +384,11 @@ class AsyncIntentRecognizer:
async def process_query_async(self, query: str, conversation_context: Dict = None,
chat_history: List[Dict[str, str]] = None,
previous_slots: Dict[str, Any] = None,
use_jieba: bool = False,
enable_query_expansion: bool = False,
cur_soft_name: str = "") -> Dict[str, Any]:
chat_history: List[Dict[str, str]] = None,
previous_slots: Dict[str, Any] = None,
use_jieba: bool = False,
enable_query_expansion: bool = False,
cur_soft_name: str = "") -> Dict[str, Any]:
"""
异步处理用户问题的完整流程
@@ -423,26 +414,20 @@ class AsyncIntentRecognizer:
if enable_query_expansion:
# 创建异步任务并立即开始执行
query_expand_tasks = [
# 5.1: 后退提示
# 后退提示
asyncio.create_task(self._generate_step_back_prompt_async(query, chat_history, conversation_context)),
# 5.2: Follow Up Questions
asyncio.create_task(self._generate_follow_up_questions_async(query, chat_history, conversation_context)),
# 5.3: 文档查询
# 文档查询
asyncio.create_task(self._find_matching_software_docs_async(query, cur_soft_name, chat_history)),
# 5.4: 多问题查询
asyncio.create_task(self._generate_multi_questions_async(query, chat_history, conversation_context))
]
# 步骤1-3: 并行执行关键词匹配、问题改写和意图分类
# 执行关键词匹配
keywords_task = self._match_keywords_async(query, use_jieba)
# 等待关键词匹配完成
keywords_terms, query_keys = await keywords_task
# 步骤2: 问题改写
# 步骤2-3: 并行执行问题改写和意图分类
rewrite_task = self._rewrite_query_async(
query=query,
keywords=keywords_terms,
@@ -450,14 +435,14 @@ class AsyncIntentRecognizer:
chat_history=chat_history,
context=conversation_context
)
classification_task = self._classify_intent_async(query, conversation_context, chat_history, previous_slots)
# 等待问题改写完成
rewrite = await rewrite_task
# 并行等待问题改写和意图分类完成
start_time = time.time()
rewrite, classification = await asyncio.gather(rewrite_task, classification_task)
end_time = time.time()
logging.info(f"意图分类耗时统计 - 总耗时: {end_time - start_time:.2f}秒")
# 步骤3: 进行意图分类
classification_task = self._classify_intent_async(rewrite.rewrite, conversation_context, chat_history, previous_slots)
classification = await classification_task
# 特殊处理 锁相关咨询
if classification.vertical_classification == "安装下载注册" and classification.sub_classification == "软件锁类":
process_lock_start_time = time.time()
@@ -465,12 +450,9 @@ class AsyncIntentRecognizer:
process_lock_end_time = time.time()
process_lock_time = process_lock_end_time - process_lock_start_time
logging.info(f"锁相关咨询正则匹配 - 总耗时: {process_lock_time:.2f}秒")
# 步骤4: 进行槽位填充
# 如果是有效分类，进行槽位填充
slot_filling_result = {}
# if classification.vertical_classification not in ["其他", "闲聊"] and classification.sub_classification not in ["其他", "闲聊"]:
# slot_filling_result = await self._fill_slots_async(rewrite.rewrite, classification, conversation_context, chat_history, previous_slots)
if not enable_query_expansion:
return {
"classification": classification.model_dump(),
@@ -484,28 +466,22 @@ class AsyncIntentRecognizer:
start_time = time.time()
query_expand_results = await asyncio.gather(*query_expand_tasks)
end_time = time.time()
logging.info(f"异步问题扩展环节耗时统计 - 总耗时: {end_time - start_time:.2f}秒")
logging.info(f"问题扩展环节耗时统计 - 总耗时: {end_time - start_time:.2f}秒")
# 收集结果
step_back_result = query_expand_results[0] if query_expand_results[0] else StepBackPrompt(original_query=query, can_use_back_prompt=False, step_back_query=[query])
follow_up_result = query_expand_results[1] if query_expand_results[1] else FollowUpQuestions(original_query=query, follow_up_query=query)
wiki_result = query_expand_results[2] if query_expand_results[2] else []
multi_questions_result = query_expand_results[3] if query_expand_results[3] else MultiQuestions(original_query=query, sub_questions=[query])
wiki_result = query_expand_results[1] if query_expand_results[1] else []
all_questions=[]
all_questions.append(query)
all_questions.append(rewrite.rewrite)
all_questions.extend(wiki_result)
all_questions.extend(step_back_result.step_back_query)
all_questions.append(follow_up_result.follow_up_query)
all_questions.extend(multi_questions_result.sub_questions)
all_questions = list(set(all_questions))
query_expand = {
"all": all_questions,
"step_back": step_back_result.step_back_query,
"follow_up": [follow_up_result.follow_up_query],
"multi_questions": multi_questions_result.sub_questions,
"wiki_title": wiki_result,
"original_query":query,
"rewrite_query":rewrite.rewrite
@@ -670,52 +646,13 @@ class AsyncIntentRecognizer:
parsed_output = step_back_parser.parse(clean_output)
step_back_end_time = time.time()
step_back_time = step_back_end_time - step_back_start_time
logging.debug(f"异步后退提示生成耗时统计 - 总耗时: {step_back_time:.2f}秒")
logging.info(f"后退提示生成耗时统计 - 总耗时: {step_back_time:.2f}秒")
return parsed_output
except Exception as e:
# 如果解析失败，返回原始查询作为后退提示
logging.error(f"异步后退提示生成失败: {e}", exc_info=True)
logging.error(f"后退提示生成失败: {e}", exc_info=True)
return StepBackPrompt(original_query=query, can_use_back_prompt=False, step_back_query=[query])
async def _generate_follow_up_questions_async(self, query: str, chat_history: List[Dict[str, str]] = None, conversation_context: str = "") -> FollowUpQuestions:
"""
异步生成后续问题
Args:
query: 用户原始问题
chat_history: 历史对话记录
conversation_context: 会话背景信息
Returns:
后续问题结果
"""
follow_up_start_time = time.time()
# 准备提示词
follow_up_parser = PydanticOutputParser(pydantic_object=FollowUpQuestions)
formatted_prompt = follow_up_questions_prompt.format(
query=query,
chat_history=json.dumps(chat_history, ensure_ascii=False) if chat_history else "[]",
# conversation_context=conversation_context,
output_format=follow_up_parser.get_format_instructions()
)
try:
# 异步调用LLM
response = await self._llm.invoke_async(formatted_prompt, False)
# 解析输出
response.content = response.content.strip()
clean_output = re.sub(r'<think>.*?</think>', '', response.content, flags=re.DOTALL)
parsed_output = follow_up_parser.parse(clean_output)
follow_up_end_time = time.time()
follow_up_time = follow_up_end_time - follow_up_start_time
logging.debug(f"异步后续问题生成耗时统计 - 总耗时: {follow_up_time:.2f}秒")
return parsed_output
except Exception as e:
# 如果解析失败，返回原始查询作为后续问题
logging.error(f"异步后续问题生成失败: {e}", exc_info=True)
return FollowUpQuestions(original_query=query, follow_up_query=query)
async def _find_matching_software_docs_async(self, query: str, soft_name: str,
chat_history: List[Dict[str, str]] = None,
top_k: int = 3) -> List[str]:
@@ -771,7 +708,7 @@ class AsyncIntentRecognizer:
json_response = json_parser.parse(response.content)
for match in json_response:
wiki_names.append(match["content"])
logging.debug(f"软件文档匹配耗时: {end_time - start_time:.2f}秒")
logging.info(f"软件文档匹配耗时: {end_time - start_time:.2f}秒")
return wiki_names
except json.JSONDecodeError as e:
@@ -781,43 +718,4 @@ class AsyncIntentRecognizer:
except Exception as e:
logging.error(f"查找匹配软件文档时出错: {e}", exc_info=True)
# 出错时返回空列表
return []
async def _generate_multi_questions_async(self, query: str, chat_history: List[Dict[str, str]] = None, conversation_context: str = "") -> MultiQuestions:
"""
异步生成多角度问题
Args:
query: 用户原始问题
chat_history: 历史对话记录
conversation_context: 会话背景信息
Returns:
多角度问题结果
"""
multi_questions_start_time = time.time()
# 准备提示词
multi_questions_parser = PydanticOutputParser(pydantic_object=MultiQuestions)
formatted_prompt = multi_questions_prompt.format(
query=query,
chat_history=json.dumps(chat_history, ensure_ascii=False) if chat_history else "[]",
# conversation_context=conversation_context,
output_format=multi_questions_parser.get_format_instructions()
)
try:
# 异步调用LLM
response = await self._llm.invoke_async(formatted_prompt, False)
# 解析输出
response.content = response.content.strip()
clean_output = re.sub(r'<think>.*?</think>', '', response.content, flags=re.DOTALL)
parsed_output = multi_questions_parser.parse(clean_output)
multi_questions_end_time = time.time()
multi_questions_time = multi_questions_end_time - multi_questions_start_time
logging.debug(f"异步多角度问题生成耗时统计 - 总耗时: {multi_questions_time:.2f}秒")
return parsed_output
except Exception as e:
# 如果解析失败，返回原始查询作为唯一子问题
logging.error(f"异步多角度问题生成失败: {e}", exc_info=True)
return MultiQuestions(original_query=query, sub_questions=[query])
return []