上传问题改写、意图识别模块代码

2025-05-27 09:48:03 +08:00
commit 99017f0cb0
66 changed files with 111493 additions and 0 deletions
@@ -0,0 +1,289 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File: IntentRecognition.py
+Author: oyyz
+Date: 2025-05-13
+Description: 意图分类、改写核心逻辑
+"""
+
+import os
+from langchain_openai import ChatOpenAI
+from langchain.output_parsers import PydanticOutputParser
+import json
+from typing import List, Tuple
+import re
+from .PromptTemplates import classification_prompt, query_rewrite_prompt, extract_nouns_prompt, classification_info
+from .DataModels import Classification, QueryRewrite, Term, TermList
+from .ProfessionalNounVector import ProfessionalNounRetriever
+from rag2_0.tool.ModelTool import XinferenceReRankerModel, OpenAiLLM
+
+
+class IntentRecognizer:
+    """
+    意图识别和问题改写类
+    """
+    def __init__(self, api_key: str = None, base_url: str = None, model_name: str = "gpt-3.5-turbo", vector_index_dir: str = None):
+        """
+        初始化意图识别器
+        
+        Args:
+            api_key: OpenAI API密钥，如果为None则从环境变量获取
+            base_url: OpenAI API基础URL，如果为None则使用默认URL
+            model_name: 要使用的模型名称
+            vector_index_dir: 向量索引目录，如果为None则使用默认目录
+        """
+        # 初始化LLM
+        llm_params = {
+            "temperature": 0.2,  # 降低随机性，使结果更确定
+            "model": model_name
+        }
+        
+        # 如果提供了API密钥，则使用提供的密钥
+        if api_key:
+            llm_params["api_key"] = api_key
+        
+        # 如果提供了自定义URL，则使用提供的URL
+        if base_url:
+            llm_params["base_url"] = base_url
+        
+        self.llm = OpenAiLLM(**llm_params) 
+        
+        # 准备分类解析器
+        self.classification_parser = PydanticOutputParser(pydantic_object=Classification)
+        
+        # 准备问题改写解析器
+        self.query_rewrite_parser = PydanticOutputParser(pydantic_object=QueryRewrite)
+        
+        # 准备术语列表解析器
+        self.terms_list_parser = PydanticOutputParser(pydantic_object=TermList)
+        
+        # 加载suffix关键词
+        self.suffix_keywords = self._load_suffix_keywords()
+        
+        # 初始化向量检索器
+        self.noun_retriever = ProfessionalNounRetriever(api_key=api_key, index_dir=vector_index_dir)
+    
+    def _load_suffix_keywords(self, filepath: str = None) -> List[str]:
+        """
+        加载后缀关键词列表
+        
+        Args:
+            filepath: 后缀关键词文件路径，默认为None使用默认路径
+            
+        Returns:
+            后缀关键词列表
+        """
+        try:
+            # 如果未指定路径，使用默认路径
+            if filepath is None:
+                current_dir = os.path.dirname(os.path.abspath(__file__))
+                filepath = os.path.join(current_dir, "..", "..", "data", "nouns", "suffix_keywords.json")
+            
+            # 读取JSON文件
+            with open(filepath, "r", encoding="utf-8") as f:
+                suffix_data = json.load(f)
+            
+            # 添加额外的固定后缀
+            return suffix_data
+            
+        except Exception as e:
+            raise RuntimeError(f"加载后缀关键词失败: {e}") from e
+    
+    def classify_intent(self, query: str, keywords: TermList) -> Classification:
+        """
+        对用户输入进行意图分类
+        
+        Args:
+            content: 用户输入内容
+            keywords: 匹配到的关键词列表
+            rewrite: 重写的问题
+        Returns:
+            分类结果
+        """
+        formatted_prompt = classification_prompt.replace("{user_input}", query)
+        formatted_prompt = formatted_prompt.replace("{classification_info}", classification_info)
+        formatted_prompt = formatted_prompt.replace("{output_format}", self.classification_parser.get_format_instructions())
+        # 将关键词列表转换为JSON字符串
+        terms_dict = [term.model_dump() for term in keywords.terms] 
+        keywords_str = json.dumps(terms_dict, ensure_ascii=False)
+        formatted_prompt = formatted_prompt.replace("{keywords}", keywords_str)
+        # 调用LLM
+        response = self.llm.invoke(formatted_prompt, False)
+        
+        # 解析输出
+        try:
+            # 尝试直接解析JSON响应
+            parsed_output = self.classification_parser.parse(response.content.strip())
+            return parsed_output
+        except Exception as e:
+            raise RuntimeError(f"解析分类结果时出错: {e}") from e
+    
+    def extract_keywords_with_llm(self, query: str) -> List[Term]:
+        """
+        使用LLM从用户查询中提取专业关键词
+        
+        Args:
+            query: 用户查询
+            
+        Returns:
+            提取的术语列表
+        """
+        # 准备提示词
+        formatted_prompt = extract_nouns_prompt.replace("{content}", query)
+        formatted_prompt = formatted_prompt.replace("{output_format}", self.terms_list_parser.get_format_instructions())
+        
+        # 调用LLM
+        response = self.llm.invoke(formatted_prompt, False)
+        
+        try:
+            # 尝试使用Pydantic解析器解析TermList
+            parsed_output = self.terms_list_parser.parse(response.content)
+            return parsed_output.terms
+        except Exception as e:
+            raise RuntimeError(f"无法解析LLM关键词提取响应: {e}") from e
+        
+    def match_keywords(self, query: str) -> Tuple[TermList, List[str]]:
+        """
+        从用户问题中匹配关键词，结合LLM提取和向量检索
+        
+        Args:
+            query: 用户问题
+            
+        Returns:
+            匹配到的关键词列表
+        """
+        matched_terms = set()  # 存储匹配到的Term对象
+        query_keys=[]
+        # 步骤2: 使用LLM提取查询中的关键词
+        try:
+            extracted_terms = self.extract_keywords_with_llm(query)
+            for term in extracted_terms:
+                query_keys.append(term.name)
+        except Exception as e:
+            raise RuntimeError(f"LLM关键词提取失败: {e}") from e
+        
+        # 步骤3: 使用向量检索找到相似的专业名词
+        try:
+            # 对matched_terms中的每个关键字进行向量检索
+            for current_key in query_keys:
+                vector_results = self.noun_retriever.query(current_key, top_k=3, use_intersection=True)
+                
+                # 添加向量检索结果
+                for result in vector_results:
+                    term = Term(
+                        name=result.get('name'),
+                        synonymous=result.get('synonymous', []),
+                        description=result.get('description', '')
+                    )
+                    matched_terms.add(term)
+
+        except Exception as e:
+            raise RuntimeError(f"向量检索关键词时出错: {e}") from e
+        
+        if len(matched_terms) != 0:
+            txts = ["名称:" + term.name + "|" + "同义词:" + ";".join(term.synonymous) + "|" + "描述:" + term.description for term in matched_terms]
+            # txts = [term.name for term in matched_terms]
+            xinference_reranker = XinferenceReRankerModel()
+            rerank_results = xinference_reranker.rerank(query, txts, top_k=5)
+            matched_terms_list = list(matched_terms)
+            matched_terms = [matched_terms_list[result["index"]] for result in rerank_results]
+        # 提取所有Term对象的名称并排序
+        # 将set类型的matched_terms转换为TermList类型
+        term_list = TermList(terms=list(matched_terms))
+        return term_list, query_keys
+    
+    def rewrite_query(self, query: str, keywords: TermList) -> QueryRewrite:
+        """
+        对用户问题进行改写
+        
+        Args:
+            query: 用户原始问题
+            keywords: 匹配到的关键词列表
+            
+        Returns:
+            改写结果
+        """
+        # 准备问题改写提示
+        terms_dict = [term.model_dump(exclude={"description"}) for term in keywords.terms] 
+        keywords_str = json.dumps(terms_dict, ensure_ascii=False)
+        formatted_prompt = query_rewrite_prompt.format(query=query, output_format=self.query_rewrite_parser.get_format_instructions(),keywords=keywords_str)
+
+        
+        # 调用LLM
+        response = self.llm.invoke(formatted_prompt, False)
+        
+        # 解析输出
+        try:
+            # 尝试直接解析JSON响应
+            parsed_output = self.query_rewrite_parser.parse(response.content)
+            return parsed_output
+        except Exception as e:
+            raise RuntimeError(f"解析问题改写结果时出错: {e}") from e
+    
+    def judge_define_suffix(self, input_str: str) -> Tuple[bool, List[str]]:
+        """
+        判断输入字符串是否包含定义的后缀，并返回所有匹配到的后缀名列表
+        
+        Args:
+            input_str: 输入字符串
+            
+        Returns:
+            Tuple[bool, List[str]]: (是否包含定义的后缀, 匹配到的后缀名列表)
+        """
+        
+        # 构建正则表达式模式，匹配大小写不敏感且前面可能带有.
+        pattern = r'(?:\.?)(' + '|'.join(re.escape(field.get('name')) for field in self.suffix_keywords) + r')'
+        
+        # 使用 re.IGNORECASE 标志来忽略大小写，findall找到所有匹配
+        matches = re.finditer(pattern, input_str, re.IGNORECASE)
+        matched_suffixes = [match.group(1) for match in matches]
+        
+        return bool(matched_suffixes), matched_suffixes
+
+    def process_query(self, query: str) -> Tuple[Classification, TermList, QueryRewrite, List[str]]:
+        """
+        处理用户问题的完整流程
+        
+        Args:
+            query: 用户原始问题
+            
+        Returns:
+            (意图分类结果, 匹配的关键词列表, 问题改写结果)的元组
+        """
+        # 是否是扩展名
+        # is_suffix, matched_suffixes = self.judge_define_suffix(query)
+        # if is_suffix:
+        #     # 将所有匹配到的后缀名作为Term添加到结果中
+        #     suffix_terms = []
+        #     for suffix in matched_suffixes:
+        #         term_dict = next((item for item in self.suffix_keywords if item['name'].lower() == suffix.lower()), None)
+        #         if term_dict:
+        #             suffix_term = Term(
+        #                 name=term_dict.get('name'),
+        #                 synonymous=term_dict.get('synonymous', []),
+        #                 description=json.dumps(term_dict.get('description', ''), ensure_ascii=False)
+        #             )
+        #         suffix_terms.append(suffix_term)
+   
+        #     return Classification(vertical_classification="安装下载", sub_classification="查询"), TermList(terms=suffix_terms), QueryRewrite(rewrite=query), matched_suffixes
+
+        # 步骤1: 匹配关键词
+        keywords_terms, query_keys = self.match_keywords(query)
+
+        # 步骤2: 问题改写
+        rewrite = self.rewrite_query(
+            query=query, 
+            keywords=keywords_terms
+        )
+
+        # 步骤3: 进行意图分类
+        classification = self.classify_intent(query, keywords_terms)
+        if classification.vertical_classification == "其他" or classification.sub_classification == "其他":
+            return classification, TermList(terms=[]), QueryRewrite(rewrite=query), []
+
+        if classification.vertical_classification == "闲聊" or classification.sub_classification == "闲聊":
+            return classification, TermList(terms=[]), QueryRewrite(rewrite=query),[]
+
+        # rewrite = QueryRewrite(rewrite=query)
+        return classification, keywords_terms, rewrite, query_keys