更新API密钥管理逻辑，优化意图识别示例，增强Excel数据验证功能，移除冗余名词条目，更新多个二进制索引文件

api_key.txt

@@ -1,31 +1,5 @@
-sk-kvgfuqeqvpmfsccykyoohheshclcrtvjlnewratvrjpkpbkc
-sk-aulumxzhvaladchcwgmsxidtdsvzytbpvzqgfuvcxlwbwcgl
-sk-otxxemniwhxkdvroszmmkitswwuykosnqoldrkzdoflqpgvw
-sk-zlruqobfdbjebyyvkmehakpcvfgnlfbdlbfrepusazzckbnv
-sk-rczjqufgdisqplkrmvhaxmdgcboluvxympvzljlreuqeeviq
-sk-xfnvcksdgwufsktvmhpqrwpgovsxxtaeehtxnaqjtxmubqzl
-sk-wwonvjnowbcxmoyoluynnkjwerghspzdulyidskunkordaft
 sk-qmrkfvvbbfssuoreyvwqawoveyowuvxviqzqknotyweqmuog
-sk-nprpuknjmikvoaxnwgyshwwwtnifvixpuqtzkzmcacdnvoib
-sk-xanwnicepdxfqrfejzuxjcrhdsglfypkoxlcmmtamrtjkork
-sk-lvtdgodiaurqyiwdxtdrgxifguychhccqlqkhqctscvqbfgi
-sk-aedlbtlmqcttxwnvlfmxzaysamocamqxjceoyqjfgpcowybw
-sk-fahdvndjblyjlizamvwcrxnilsgmbgbvwssxgquhkezgpqne
-sk-tzludgttzxvpvwayazdbppbauvathdtccafjrhojpemucgyi
-sk-ylgoiqxmtxeojdnonthxtweungyzldaqarvjxlqyztlvyrff
-sk-asuqbqwdhjcqnvtjlwufyrkrwkobnrbmukzarvcctsgjipdp
-sk-nbksjgcngsayoumnsdbkcpnqivnvxjenwpzuazzrkhnsgeoo
-sk-jgybgyayxlwoxeijgrjcneqlyusleohgbliuwpsuhocrjsmk
-sk-wzjsmwxcbbpcrqivqfzjwufqqjtlwejtncnvbpeicznkwiuh
-sk-izdjicdoyillktsihkiapuvwebisehtlgykozrvzfkgncwsc
-sk-fcsfmyivfuojsqsditvobfqprdpeunukycpcfnoxkraqevpx
-sk-szyjgyxrcvyxpvzfwgmbxnflxngxvcplitcctsdvvrqjgftk
-sk-jzbodthsnvjwbyrnynsxrudtqfnbdbrcxebjwjgajocnzqse
-sk-fxepossfzpmccibfwqpkluorzqlbtcaplepeugtfzfsctcbl
-sk-flhqvziknntednkcgjaxlyzzsrfzjhrzrmteqonajpbiinni
-sk-xfregpbbquqbxpiobjzanydsjivrjrnbokzxcqtnhxhyghhe
 sk-jrdzerhmvrtvzawkksowbgkggkubwfquplmrxbdhespqgtis
-sk-oqehupcveovkjqqtxypqyifidcdissuyehwrkdwgruoyjkpq
 sk-jnnmltwtqwuoyagoogzzeraczmyfxhoairiddgayksqdfnbr
 sk-eghuepxnbcollzrjwbzqvbnhiiwagkejaclyhvaodeqgwrog
 sk-poszkbjdmamimconjustnrxxqusuzlryxkrzkpronlenrmen
@@ -36,76 +10,6 @@ sk-gzdqfoyvulrqscdpjlwlufdecrsyjpmwpkknuhnjsvtyftox
 sk-bkcufidsebujopqqwexwxwpmevrpelmvxzdymncvllcyojce
 sk-olabhscekudzkyudypkcjvehwqunagubwdmtppugrjmcptwv
 sk-zpdqyocliebhqpkuwvebpgcnfjdkvavdltimllmgkthwnwph
-sk-gvhchlfelocjniuydusyhhwacnomxnvucjonzkhtqoplnbcr
-sk-lzneagvdxhisodndnxnpkntghpkimjmjsebiqdzaoqzuhbla
-sk-xotcfdkigykevngedupitbcatjqppxmcibjtcebyoglykuxz
-sk-ufydqsdqnwsegaqwtappzwdyzqnoblyunfvslomnnmykedgk
-sk-jwasykftbkyjzdqlwcxuicrwzxsbhttilxfefbrozrznpwlv
-sk-xngteojwkxmftyaabjdwwgyoadspsowmcpcqobteutdcfmnr
-sk-akzkgniebruqrtuqskvlibkpcxjuazhcatysptkfyqivldfn
-sk-vpqkxtmcgkggllexchzysuewyfaoexzasoumxngdplzgwksw
-sk-fvcsqdbqmdlwxzjyofrilusqcypbfyczogaqwqrjrwvojmer
-sk-htjprscvfgskjtjzpxxxjhyymshagogykpawxekrrfbgftyx
-sk-vtlgznzanapdinnlpuuygdidejxvlvlziolumckxwntzvxcn
-sk-npwaykpqovytxhihxepeaxuwgnzvqvscirwbiffudyebibwe
-sk-kduozskjwsbxytajaedemqemimflzoqdrfbbuopuogbxptos
-sk-cmyaaofqksxeccaiswwholizxtdhqoszpilbqhmwwablwtwl
-sk-zltwpyvzodfszrvqcmjxycfshshmqmgtpghiaulcrtrilqup
-sk-svpublkronxohqmjvtdemtyqopuwyknynmxvurkmmoqsfesk
-sk-tindfeurhffenyetmblplxgytcgqnbbruatchpddtstnajvq
-sk-ttwgosljnfdqbevvfkjewspoasountxeyiesuuwwpuoseatw
-sk-ikzwjzfxxnocfqvkhbolmmnojdqfvdxzmlruynfoneahgraq
-sk-zomatpdlatgtzvuoeabofeandiiespmsinosenjlmdigkvjk
-sk-xypcnibvqgebhwvsudjuxlycwombofshifpltstezflountf
-sk-vhtbtqsdyhwsrsitpppjpulbpnwlhjeeaujccggdjvjtucru
-sk-ygqyapfsdpxxzihadttugvofrcgwlnsrqjvrqfrsrlgqidch
-sk-cczurtilupwyrnqnjpdjltzamjbpyaxnwgzqoykvfwecezhs
-sk-hsalsmvnzmvlknpzldciqligtjrvhcjdbjtcqhorptsgoisr
-sk-ierjdccoycmipzflzvzjqdmjazmbczxhwejwrjmuouxpcqev
-sk-rlthtjzrxkdlfsemijumxiwwsoxkwnjovobtqwprhnducsqi
-sk-doocedvfjsmwysxyzpptojnwwuoucxvrknlrdsnomffljems
-sk-tgxnplgtwawfnnetghvlzoinksickbynzsgovbrmlrkzzrsy
-sk-aggkwzdteggbcgyzimiivphpzibnpmqfbkmgtwyggsjfdymg
-sk-tzynazayiglcpzejedpfdftqvtuqynygluorjbbcvrnmsbfl
-sk-mcaqucnscgznmtvtzhooxbdafzhcgjgpusnjfbftroedzujk
-sk-mlfnxfmwwhturavnsbyedkgrhxfspbfitfyqkllezgdtzhwn
-sk-meaqykzphtqqoiswdtbxfqndddmtgvwrdnzdxprozlumrojs
-sk-vrtfrtioocmrylrrqpburhkgwmyxfipnissumgufmjpzjqoo
-sk-rhknxievjjjzcmkvfwqosnaiglsoufytfkbhhgconiibwriy
-sk-wzsvxjfnvpceeoapbzdimpwecckpmcsqmflocttnqyvqvrsr
-sk-lflfajincbdljrfwkhrjunyolzuxtcfdbzmspyfizpbdwvvh
-sk-siujvueppvhhokbofqlcdxudpswdfwpetvxyittpbarddhsu
-sk-avpyhivanabvdyorypdmlqnfetivlwoovfvgsgklumxeqgzy
-sk-fvmpnspfeleqrgumwnvzevbtgmdqvftlbvswoghrtxfengfa
-sk-wxaeehghpdettyrsnhjrawddnqzxghfjbkkokbwtnbdocsmi
-sk-yighqmadjhvixkhjpcknrkapqsyxmphcqvjhlbfzgigfghft
-sk-orepiytzucdtxgnxuxhekaivcpzwnunpiauccujrhbkifodc
-sk-qsfrireujzrpomdjcfcgbfqncnferyzfcsralnszvurfvkgz
-sk-tjienchcxamgldtzjtigpitklsonwcuhyhltswwcnsfhmois
-sk-gdtqulaciovvnzwklpckwyheysaplirhgimzreqzkhfxotjh
-sk-rqhynqmafsydevjctxeuqizuiithschhiztmqnpawqjjbhyv
-sk-idrdvmcgmjmoxowxswfcorfdjmprfukrumimaljjnlwqkazu
-sk-hrrmqrhscocbzhpfkzuoptnubgfiedololapvyfftapaawdb
-sk-rtqzxuylhryzqnotvkzelvhomuovmbvpxrytydvqevisciwg
-sk-ttgsojzjchifbrpwfmcajfujbxexbyvlovgbgpfdvkzqjzpu
-sk-tvclqjccfylijwcujuxpppvnumkvmxbctevfqfxjryyygoep
-sk-qyazelyawtgyohimgbswwetajekuvmyteweviovvqsjybtwq
-sk-zowapfiiqrpeuahmnkopcyccwiyzweigvekiusiuirtsfmbj
-sk-qwanpokfsquwwrpxceavasxwmxjarfaxheraamqhsyegfgum
-sk-smqpfdbcnhpifwsrpwutwzpscbkeinkmfeoabesztmnicvez
-sk-cnvevlfppneiztcgayxiydukhexujtiqjajpdhbrtnenizxe
-sk-jmjgjraheompbftegtdzrwcnlyxsfhtrjootwuqirgmodxdm
-sk-tfkyyzbhdowskndsnqkniivwhmmfmhnjdzitgvwhmhfrobeh
-sk-spjtulrgqkvepawufkloxktfmkyjjhffnmzcththdmnlqkwp
-sk-xafrjoiwnundkubilpoxmzpmbsizljfgrerckqgpcnvguqma
-sk-dtjwhkzhljzzmchfvoyywjbsdpvgtgnlpzdkgegwnjjnsesc
-sk-lurglxyefchyowvybancbtxdavndwvcengfhcrzcdouqsdmp
-sk-nckmscferrehepawtytvvzzcexsgbxszmrerkwvzsbvnwckq
-sk-hpgusoznbejkugxugsstyezaihqatwmwiwelwjwudekoxlbq
-sk-yyqtojpqfrtkyvmhbjvnzhuujzgvitqpuxkytgxspwqscptz
-sk-vcwenoaiegiwqhdvalxrvttwqmcrudttpfqhlvtdocsxvvob
-sk-npnyztbrdaqtbnldlvfelkfbfozykuwykkkfsrsisytusymj
-sk-rjaadeqsgoskclkuqkkyjxdknngrdsjwlgucqutiskroprzk
 sk-uollmeyatyiwfzszvxkpyndmzfrbqjpyixewmrastbmaqbhy
 sk-xdlsjytiwilvodadkjxvwdgulhhdytkqvfpyrcnllclgzqkb
 sk-ffkltifkylutornjhwmnmfjsqsywrjibvujhjtjctzgnkvlp
@@ -115,4 +19,44 @@ sk-duchutcxmygrnkhzmmlykvtzwaylqtdxfbbuhvfvzuapazii
 sk-nlddwexmjxqtgdvahwvlotnomrzcgskxeakxkxauicknzfkp
 sk-lopwluipwvilwpwztvaxfebueeyilefwgncgpeprqvwazxom
 sk-rgwrklpvhhrluokkbgavzukuhhpfhqzmozpjzoezfhkxyorc
-sk-cdrpglnfmyeeqyhtvxvkpcpwscsbfouwkagjpphuksfzeipy
+sk-cdrpglnfmyeeqyhtvxvkpcpwscsbfouwkagjpphuksfzeipy
+sk-eyktixexxjqvwufezcmdrazcedtsphyiunhqpamlkrcaxwtg
+sk-euzbguamsxqspfdjnrpbchkjkouncqipjvhnkkbvoihgwspe
+sk-qlpoqleqaodseswzqklbwjwwcdjrqthbmvweuablibiszpnw
+sk-gqjtkwjmrupugviflhsffhkpzxxmjcewviqsneurxnlfqewy
+sk-hkxgjpdyxuxjklksunfzaetrhveelucmrldfjnlztibxgjgl
+sk-fqjkatqvpkmvlkbqhjfkzmiifmiodayututyprdtldszmacd
+sk-wrybirbfwtbfdijjrfpdfxlxzmvcrhfgqqbhyuysibmcmcez
+sk-vgkpsmbcchymktakjsheqnjlkopiqcvntqcvxuxmxlllifod
+sk-afvrxowsmjmlhguhfpfhmefcldzsohmnyumjdwwkrvzwjrym
+sk-jwfemfolekojghqwfwvjxwmzdvdnhaznngqeurvzgwqepddp
+sk-vxzwcizvmykzwbxaypvshdkbgjymnmelryyqftkxzxnnpibr
+sk-bwcbyczfakqkwrsvmmraeikjfpiurhhanflfqkunpzezvagv
+sk-dicoheyoylocrtjsacbvwqzwwibnnasuwhnhsudoemieengi
+sk-mvuzqvuqmryqrqolztwthhzcnzibygjycdbeplbbcnbxkncq
+sk-qavyzqmnfjbhqhuesfkjcyszkzmjsykpnaebuwzcxhkitoir
+sk-jhrbmqmzyrlrqyehkkachwhztkuisphqbmoyvchmaqrcfxtf
+sk-mtcfttyljuzteasdjbcrtwwwmqxetmaonbwvojzfijzfiplq
+sk-iooturknmbpxhbiulovovtzpyayovyidifzpzkjgqacroxjt
+sk-jgjhasobxeuzfgutdyuhcejocwdiwonhkdithchkonhxnawc
+sk-mvyqkciotllopyozsfzwtjeicuhnvoihnrrxadfsfiakperx
+sk-wticrisyjehvnlrlhmmxhrnlzknqpnkfxowlzvnozskvtvzc
+sk-hiniqrfvuqlsgmhrqlezlribsaqdefuhpxxfavoqtszxtasw
+sk-zlfncovfrpzczmjquirolpogdrzfarkcwwqkluvifwcvrezq
+sk-blhnvpenedysngftlghrkxhweoququkvduikziuypzilpyrp
+sk-rynvudqktwvnjleiahwdpeqqdkncsvawvjyicyiojoviiges
+sk-ncskqkwomgqpfnfnachehkeaczzgiiuripeyzjrpnuzeosnn
+sk-ijqdmtyeuqrbndqjggxyicfjrmpsgbsjwwkitgsmxqvcjrri
+sk-ajgisqnkpgoiwxigrnjachusupagqpukteuknemhmnxsasre
+sk-zsskrhjnoepgjngcsseklxfpwpozhenurtrluxlstxujdsti
+sk-lubrliuefgmrxpfafdwrhzletyvhemqkpvriuuqncivlewgx
+sk-pavwfgrpftdtzeiiaoousmkdptwujoeocyzzoudeqkfyoxks
+sk-jfwhzkhpgxacedwxzkbwrwvlfqvnlhxaeyghjcmtshogsqub
+sk-sqmaankkcsqpbtktdfcmpuxjxgarfzgvygdgxgztlmyxfpkp
+sk-xhlgjmwmtkahrpdncwoqynjdrkekmsyftwqmomsodbggvbdd
+sk-ynqqptobbeazmjyrmaytsvyczsqwrukpezizrlcloncxtwvc
+sk-wswttgfrxrwijvqhctfilhvlxgdkgogrjhvjkdbzvqrocofa
+sk-jdijeubeygjmqtxwryrbwmrpvqawinzwpcxodpolhcupzmpa
+sk-xbloemctsowwicjvrtrrewreosnfojoijtygsfxfnjntridv
+sk-isovavcefvkzlbjewnumeqqevmnoucojsxwskkitfktkemtq
+sk-vxrlvvdzgythgyycuqehdloubxcdwhgojpowgxvgxsstjtvk

data/nouns/merged_nouns.json

@@ -26843,14 +26843,6 @@
     "synonymous": [],
     "description": ""
   },
-  {
-    "name": "zwzj工程",
-    "synonymous": [
-      "博微电力建设计价通的概预算工程、建安预算工程或南网规约工程",
-      "18主网定额概预算工程、建安预算工程"
-    ],
-    "description": "博微电力建设计价通软件编制的定额计价的电力建设工程概预算工程、国网施工图预算综合单价法工程"
-  },
   {
     "name": "其他费用的输出复选框",
     "synonymous": [

rag2_0/demo/intent_recognition_example.py

@@ -63,7 +63,11 @@ def process_query(recognizer: IntentRecognizer, query: str, conversation_context
         try:
             # 使用新的process_query_with_slots方法处理查询
             # result = recognizer.process_query_with_slots(query)
-            result = recognizer.process_query(query, conversation_context=conversation_context, chat_history=chat_history, previous_slots=previous_slots)
+            result = recognizer.process_query(query, 
+                                              conversation_context=conversation_context, 
+                                              chat_history=chat_history, 
+                                              previous_slots=previous_slots, 
+                                              enable_query_expansion=True)
             # 提取分类信息
             classification = result["classification"]
             
@@ -176,7 +180,7 @@ def save_results_to_excel(results, output_file, is_final=False):
     logging.info(f"已保存{len(valid_results)}条结果至: {temp_output_file}")
 
 # 示例查询
-examples_query = """D3软件结算工程怎么解锁清单"""
+examples_query = """主网电力建设计价通软件, 35kV的软件   土质比例不能一起设置吗"""
 conversation_context=""
 chat_history=[
         {

rag2_0/demo/validate_excel_data_batch.py

@@ -98,7 +98,7 @@ class ExcelDataValidator:
             logging.error(f"读取Excel文件时出错: {e}")
             return None
     
-    def validate_classification(self, llm, query, vertical_class, sub_class):
+    def validate_classification(self, llm:OpenAiLLM , query:str, vertical_class:str, sub_class:str):
         """
         验证问题分类是否正确
         

rag2_0/tool/APIKeyManager.py

@@ -1,7 +1,7 @@
 import os
 import random
 import time
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict,Tuple
 from threading import Lock
 import requests
 
@@ -75,11 +75,12 @@ class APIKeyManager:
         return instance._get_random_api_key()
     
     @classmethod
-    def get_valid_api_keys(cls) -> List[str]:
+    def get_valid_api_keys(cls,api_key:str) -> Tuple[bool,str]:
         """
         静态方法：获取有效的API密钥列表
         
         Returns:
+            Tuple[bool,str]: 是否有效，报错信息
     """
          # 验证每一个apikey是否有效，无效则删除并打印日志。地址https://api.siliconflow.cn/v1/
         import requests
@@ -97,19 +98,17 @@ class APIKeyManager:
             ],
             "max_tokens": 1
         }
-        instance = cls.get_instance()
+
-        for key in instance.api_keys:
+        headers = headers_template.copy()
-            headers = headers_template.copy()
+        headers["Authorization"] = f"Bearer {api_key}"
-            headers["Authorization"] = f"Bearer {key}"
+        try:
-            try:
+            resp = requests.post(url, headers=headers, json=data, timeout=8)
-                resp = requests.post(url, headers=headers, json=data, timeout=8)
+            if resp.status_code == 200:
-                if resp.status_code == 200:
+                return True,""
-                    valid_api_keys.append(key)
+            else:
-                else:
+                return False,resp.text
-                    logging.warning(f"API密钥无效（被移除）: {key}, 状态码: {resp.status_code}, 响应: {resp.text}")
+        except Exception as e:
-            except Exception as e:
+            return False,str(e)
-                logging.warning(f"API密钥验证异常（被移除）: {key}, 错误: {e}")
-        return valid_api_keys
 
     @classmethod
     def count(cls) -> int:
@@ -269,19 +268,30 @@ if __name__ == "__main__":
     
     # 获取实例并查看使用统计
     instance = APIKeyManager.get_instance()
+
+    # 获取有效的API密钥列表
+    # valid_api_keys = APIKeyManager.get_valid_api_keys()
+    # print(f"有效的API密钥列表：{valid_api_keys}")
+
     stats = instance.get_usage_stats()
     all_balance=0.0
-    buy_balance=17 * 10 * 14 # 购买16次，一次10条api_key，每个api_key有14元
+    buy_balance=14 * 10 * 14 # 购买18次，一次10条api_key，每个api_key有14元
     invalid_api_keys = []
     for key, data in stats.items():
         usage_stats = APIKeyManager.get_key_usage_stats(key)
         all_balance+=float(usage_stats['data']['balance'])
-        print(f"api key:{key}---赠送余额：{usage_stats['data']['balance']}元")
+        valid,err_info = APIKeyManager.get_valid_api_keys(key)
+        if not valid:
+            print(f"api_key:{key}---赠送余额：{usage_stats['data']['balance']}元---报错信息：{err_info}")
+            # invalid_api_keys.append(key)
+        else:
+            print(f"api_key:{key}---赠送余额：{usage_stats['data']['balance']}元")
         if float(usage_stats['data']['balance']) == 0:  
             invalid_api_keys.append(key)
-    print(f"剩余总赠送余额：{all_balance}元，累计消耗：{buy_balance-all_balance}元")
 
-    print(f"无效的API密钥：{invalid_api_keys}")
+    invalid_api_keys = list(set(invalid_api_keys))
+    print(f"剩余总赠送余额：{all_balance}元，累计消耗：{buy_balance-all_balance}元")
+    print(f"无效的API密钥,数量：{len(invalid_api_keys)}，列表：{invalid_api_keys}")
     print(f"开始移除无效的API密钥，并重新保存")
     APIKeyManager.remove_invalid_api_keys(invalid_api_keys)
     APIKeyManager.save_api_keys()

rag2_0/tool/ModelTool.py

@@ -165,20 +165,23 @@ class OpenAiLLM:
                 except Exception as e:
                     retry_count += 1
                     if retry_count == max_retries:
-                        raise RuntimeError(f"调用LLM失败，已重试{max_retries}次: {str(e)}")
+                        raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e
                     else:
                         time.sleep(5*retry_count)  # 重试前等待5秒*重试次数
         else:
-            # 创建 Completion 请求. 超时120s
+            try:
-            api_key = APIKeyManager.get_api_key()  
+                # 创建 Completion 请求. 超时120s
-            # 使用with语句创建客户端，确保资源会被正确释放
+                api_key = APIKeyManager.get_api_key()  
-            with OpenAI(api_key=api_key, base_url=self._url) as client:
+                # 使用with语句创建客户端，确保资源会被正确释放
-                completion = client.chat.completions.create(
+                with OpenAI(api_key=api_key, base_url=self._url) as client:
-                    model=self._model,
+                    completion = client.chat.completions.create(
-                    messages=[{'role': 'user', 'content': user_prompt}],
+                        model=self._model,
-                    **self._kwargs
+                        messages=[{'role': 'user', 'content': user_prompt}],
-                )
+                        **self._kwargs
-                return completion.choices[0].message
+                    )
+                    return completion.choices[0].message
+            except Exception as e:
+                raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e
 
 if __name__ == "__main__":
     # 测试重排模型