diff --git a/api_key.txt b/api_key.txt index 5ee6e0e..7b3d78d 100644 --- a/api_key.txt +++ b/api_key.txt @@ -1,31 +1,5 @@ -sk-kvgfuqeqvpmfsccykyoohheshclcrtvjlnewratvrjpkpbkc -sk-aulumxzhvaladchcwgmsxidtdsvzytbpvzqgfuvcxlwbwcgl -sk-otxxemniwhxkdvroszmmkitswwuykosnqoldrkzdoflqpgvw -sk-zlruqobfdbjebyyvkmehakpcvfgnlfbdlbfrepusazzckbnv -sk-rczjqufgdisqplkrmvhaxmdgcboluvxympvzljlreuqeeviq -sk-xfnvcksdgwufsktvmhpqrwpgovsxxtaeehtxnaqjtxmubqzl -sk-wwonvjnowbcxmoyoluynnkjwerghspzdulyidskunkordaft sk-qmrkfvvbbfssuoreyvwqawoveyowuvxviqzqknotyweqmuog -sk-nprpuknjmikvoaxnwgyshwwwtnifvixpuqtzkzmcacdnvoib -sk-xanwnicepdxfqrfejzuxjcrhdsglfypkoxlcmmtamrtjkork -sk-lvtdgodiaurqyiwdxtdrgxifguychhccqlqkhqctscvqbfgi -sk-aedlbtlmqcttxwnvlfmxzaysamocamqxjceoyqjfgpcowybw -sk-fahdvndjblyjlizamvwcrxnilsgmbgbvwssxgquhkezgpqne -sk-tzludgttzxvpvwayazdbppbauvathdtccafjrhojpemucgyi -sk-ylgoiqxmtxeojdnonthxtweungyzldaqarvjxlqyztlvyrff -sk-asuqbqwdhjcqnvtjlwufyrkrwkobnrbmukzarvcctsgjipdp -sk-nbksjgcngsayoumnsdbkcpnqivnvxjenwpzuazzrkhnsgeoo -sk-jgybgyayxlwoxeijgrjcneqlyusleohgbliuwpsuhocrjsmk -sk-wzjsmwxcbbpcrqivqfzjwufqqjtlwejtncnvbpeicznkwiuh -sk-izdjicdoyillktsihkiapuvwebisehtlgykozrvzfkgncwsc -sk-fcsfmyivfuojsqsditvobfqprdpeunukycpcfnoxkraqevpx -sk-szyjgyxrcvyxpvzfwgmbxnflxngxvcplitcctsdvvrqjgftk -sk-jzbodthsnvjwbyrnynsxrudtqfnbdbrcxebjwjgajocnzqse -sk-fxepossfzpmccibfwqpkluorzqlbtcaplepeugtfzfsctcbl -sk-flhqvziknntednkcgjaxlyzzsrfzjhrzrmteqonajpbiinni -sk-xfregpbbquqbxpiobjzanydsjivrjrnbokzxcqtnhxhyghhe sk-jrdzerhmvrtvzawkksowbgkggkubwfquplmrxbdhespqgtis -sk-oqehupcveovkjqqtxypqyifidcdissuyehwrkdwgruoyjkpq sk-jnnmltwtqwuoyagoogzzeraczmyfxhoairiddgayksqdfnbr sk-eghuepxnbcollzrjwbzqvbnhiiwagkejaclyhvaodeqgwrog sk-poszkbjdmamimconjustnrxxqusuzlryxkrzkpronlenrmen @@ -36,76 +10,6 @@ sk-gzdqfoyvulrqscdpjlwlufdecrsyjpmwpkknuhnjsvtyftox sk-bkcufidsebujopqqwexwxwpmevrpelmvxzdymncvllcyojce sk-olabhscekudzkyudypkcjvehwqunagubwdmtppugrjmcptwv sk-zpdqyocliebhqpkuwvebpgcnfjdkvavdltimllmgkthwnwph -sk-gvhchlfelocjniuydusyhhwacnomxnvucjonzkhtqoplnbcr -sk-lzneagvdxhisodndnxnpkntghpkimjmjsebiqdzaoqzuhbla -sk-xotcfdkigykevngedupitbcatjqppxmcibjtcebyoglykuxz -sk-ufydqsdqnwsegaqwtappzwdyzqnoblyunfvslomnnmykedgk -sk-jwasykftbkyjzdqlwcxuicrwzxsbhttilxfefbrozrznpwlv -sk-xngteojwkxmftyaabjdwwgyoadspsowmcpcqobteutdcfmnr -sk-akzkgniebruqrtuqskvlibkpcxjuazhcatysptkfyqivldfn -sk-vpqkxtmcgkggllexchzysuewyfaoexzasoumxngdplzgwksw -sk-fvcsqdbqmdlwxzjyofrilusqcypbfyczogaqwqrjrwvojmer -sk-htjprscvfgskjtjzpxxxjhyymshagogykpawxekrrfbgftyx -sk-vtlgznzanapdinnlpuuygdidejxvlvlziolumckxwntzvxcn -sk-npwaykpqovytxhihxepeaxuwgnzvqvscirwbiffudyebibwe -sk-kduozskjwsbxytajaedemqemimflzoqdrfbbuopuogbxptos -sk-cmyaaofqksxeccaiswwholizxtdhqoszpilbqhmwwablwtwl -sk-zltwpyvzodfszrvqcmjxycfshshmqmgtpghiaulcrtrilqup -sk-svpublkronxohqmjvtdemtyqopuwyknynmxvurkmmoqsfesk -sk-tindfeurhffenyetmblplxgytcgqnbbruatchpddtstnajvq -sk-ttwgosljnfdqbevvfkjewspoasountxeyiesuuwwpuoseatw -sk-ikzwjzfxxnocfqvkhbolmmnojdqfvdxzmlruynfoneahgraq -sk-zomatpdlatgtzvuoeabofeandiiespmsinosenjlmdigkvjk -sk-xypcnibvqgebhwvsudjuxlycwombofshifpltstezflountf -sk-vhtbtqsdyhwsrsitpppjpulbpnwlhjeeaujccggdjvjtucru -sk-ygqyapfsdpxxzihadttugvofrcgwlnsrqjvrqfrsrlgqidch -sk-cczurtilupwyrnqnjpdjltzamjbpyaxnwgzqoykvfwecezhs -sk-hsalsmvnzmvlknpzldciqligtjrvhcjdbjtcqhorptsgoisr -sk-ierjdccoycmipzflzvzjqdmjazmbczxhwejwrjmuouxpcqev -sk-rlthtjzrxkdlfsemijumxiwwsoxkwnjovobtqwprhnducsqi -sk-doocedvfjsmwysxyzpptojnwwuoucxvrknlrdsnomffljems -sk-tgxnplgtwawfnnetghvlzoinksickbynzsgovbrmlrkzzrsy -sk-aggkwzdteggbcgyzimiivphpzibnpmqfbkmgtwyggsjfdymg -sk-tzynazayiglcpzejedpfdftqvtuqynygluorjbbcvrnmsbfl -sk-mcaqucnscgznmtvtzhooxbdafzhcgjgpusnjfbftroedzujk -sk-mlfnxfmwwhturavnsbyedkgrhxfspbfitfyqkllezgdtzhwn -sk-meaqykzphtqqoiswdtbxfqndddmtgvwrdnzdxprozlumrojs -sk-vrtfrtioocmrylrrqpburhkgwmyxfipnissumgufmjpzjqoo -sk-rhknxievjjjzcmkvfwqosnaiglsoufytfkbhhgconiibwriy -sk-wzsvxjfnvpceeoapbzdimpwecckpmcsqmflocttnqyvqvrsr -sk-lflfajincbdljrfwkhrjunyolzuxtcfdbzmspyfizpbdwvvh -sk-siujvueppvhhokbofqlcdxudpswdfwpetvxyittpbarddhsu -sk-avpyhivanabvdyorypdmlqnfetivlwoovfvgsgklumxeqgzy -sk-fvmpnspfeleqrgumwnvzevbtgmdqvftlbvswoghrtxfengfa -sk-wxaeehghpdettyrsnhjrawddnqzxghfjbkkokbwtnbdocsmi -sk-yighqmadjhvixkhjpcknrkapqsyxmphcqvjhlbfzgigfghft -sk-orepiytzucdtxgnxuxhekaivcpzwnunpiauccujrhbkifodc -sk-qsfrireujzrpomdjcfcgbfqncnferyzfcsralnszvurfvkgz -sk-tjienchcxamgldtzjtigpitklsonwcuhyhltswwcnsfhmois -sk-gdtqulaciovvnzwklpckwyheysaplirhgimzreqzkhfxotjh -sk-rqhynqmafsydevjctxeuqizuiithschhiztmqnpawqjjbhyv -sk-idrdvmcgmjmoxowxswfcorfdjmprfukrumimaljjnlwqkazu -sk-hrrmqrhscocbzhpfkzuoptnubgfiedololapvyfftapaawdb -sk-rtqzxuylhryzqnotvkzelvhomuovmbvpxrytydvqevisciwg -sk-ttgsojzjchifbrpwfmcajfujbxexbyvlovgbgpfdvkzqjzpu -sk-tvclqjccfylijwcujuxpppvnumkvmxbctevfqfxjryyygoep -sk-qyazelyawtgyohimgbswwetajekuvmyteweviovvqsjybtwq -sk-zowapfiiqrpeuahmnkopcyccwiyzweigvekiusiuirtsfmbj -sk-qwanpokfsquwwrpxceavasxwmxjarfaxheraamqhsyegfgum -sk-smqpfdbcnhpifwsrpwutwzpscbkeinkmfeoabesztmnicvez -sk-cnvevlfppneiztcgayxiydukhexujtiqjajpdhbrtnenizxe -sk-jmjgjraheompbftegtdzrwcnlyxsfhtrjootwuqirgmodxdm -sk-tfkyyzbhdowskndsnqkniivwhmmfmhnjdzitgvwhmhfrobeh -sk-spjtulrgqkvepawufkloxktfmkyjjhffnmzcththdmnlqkwp -sk-xafrjoiwnundkubilpoxmzpmbsizljfgrerckqgpcnvguqma -sk-dtjwhkzhljzzmchfvoyywjbsdpvgtgnlpzdkgegwnjjnsesc -sk-lurglxyefchyowvybancbtxdavndwvcengfhcrzcdouqsdmp -sk-nckmscferrehepawtytvvzzcexsgbxszmrerkwvzsbvnwckq -sk-hpgusoznbejkugxugsstyezaihqatwmwiwelwjwudekoxlbq -sk-yyqtojpqfrtkyvmhbjvnzhuujzgvitqpuxkytgxspwqscptz -sk-vcwenoaiegiwqhdvalxrvttwqmcrudttpfqhlvtdocsxvvob -sk-npnyztbrdaqtbnldlvfelkfbfozykuwykkkfsrsisytusymj -sk-rjaadeqsgoskclkuqkkyjxdknngrdsjwlgucqutiskroprzk sk-uollmeyatyiwfzszvxkpyndmzfrbqjpyixewmrastbmaqbhy sk-xdlsjytiwilvodadkjxvwdgulhhdytkqvfpyrcnllclgzqkb sk-ffkltifkylutornjhwmnmfjsqsywrjibvujhjtjctzgnkvlp @@ -115,4 +19,44 @@ sk-duchutcxmygrnkhzmmlykvtzwaylqtdxfbbuhvfvzuapazii sk-nlddwexmjxqtgdvahwvlotnomrzcgskxeakxkxauicknzfkp sk-lopwluipwvilwpwztvaxfebueeyilefwgncgpeprqvwazxom sk-rgwrklpvhhrluokkbgavzukuhhpfhqzmozpjzoezfhkxyorc -sk-cdrpglnfmyeeqyhtvxvkpcpwscsbfouwkagjpphuksfzeipy \ No newline at end of file +sk-cdrpglnfmyeeqyhtvxvkpcpwscsbfouwkagjpphuksfzeipy +sk-eyktixexxjqvwufezcmdrazcedtsphyiunhqpamlkrcaxwtg +sk-euzbguamsxqspfdjnrpbchkjkouncqipjvhnkkbvoihgwspe +sk-qlpoqleqaodseswzqklbwjwwcdjrqthbmvweuablibiszpnw +sk-gqjtkwjmrupugviflhsffhkpzxxmjcewviqsneurxnlfqewy +sk-hkxgjpdyxuxjklksunfzaetrhveelucmrldfjnlztibxgjgl +sk-fqjkatqvpkmvlkbqhjfkzmiifmiodayututyprdtldszmacd +sk-wrybirbfwtbfdijjrfpdfxlxzmvcrhfgqqbhyuysibmcmcez +sk-vgkpsmbcchymktakjsheqnjlkopiqcvntqcvxuxmxlllifod +sk-afvrxowsmjmlhguhfpfhmefcldzsohmnyumjdwwkrvzwjrym +sk-jwfemfolekojghqwfwvjxwmzdvdnhaznngqeurvzgwqepddp +sk-vxzwcizvmykzwbxaypvshdkbgjymnmelryyqftkxzxnnpibr +sk-bwcbyczfakqkwrsvmmraeikjfpiurhhanflfqkunpzezvagv +sk-dicoheyoylocrtjsacbvwqzwwibnnasuwhnhsudoemieengi +sk-mvuzqvuqmryqrqolztwthhzcnzibygjycdbeplbbcnbxkncq +sk-qavyzqmnfjbhqhuesfkjcyszkzmjsykpnaebuwzcxhkitoir +sk-jhrbmqmzyrlrqyehkkachwhztkuisphqbmoyvchmaqrcfxtf +sk-mtcfttyljuzteasdjbcrtwwwmqxetmaonbwvojzfijzfiplq +sk-iooturknmbpxhbiulovovtzpyayovyidifzpzkjgqacroxjt +sk-jgjhasobxeuzfgutdyuhcejocwdiwonhkdithchkonhxnawc +sk-mvyqkciotllopyozsfzwtjeicuhnvoihnrrxadfsfiakperx +sk-wticrisyjehvnlrlhmmxhrnlzknqpnkfxowlzvnozskvtvzc +sk-hiniqrfvuqlsgmhrqlezlribsaqdefuhpxxfavoqtszxtasw +sk-zlfncovfrpzczmjquirolpogdrzfarkcwwqkluvifwcvrezq +sk-blhnvpenedysngftlghrkxhweoququkvduikziuypzilpyrp +sk-rynvudqktwvnjleiahwdpeqqdkncsvawvjyicyiojoviiges +sk-ncskqkwomgqpfnfnachehkeaczzgiiuripeyzjrpnuzeosnn +sk-ijqdmtyeuqrbndqjggxyicfjrmpsgbsjwwkitgsmxqvcjrri +sk-ajgisqnkpgoiwxigrnjachusupagqpukteuknemhmnxsasre +sk-zsskrhjnoepgjngcsseklxfpwpozhenurtrluxlstxujdsti +sk-lubrliuefgmrxpfafdwrhzletyvhemqkpvriuuqncivlewgx +sk-pavwfgrpftdtzeiiaoousmkdptwujoeocyzzoudeqkfyoxks +sk-jfwhzkhpgxacedwxzkbwrwvlfqvnlhxaeyghjcmtshogsqub +sk-sqmaankkcsqpbtktdfcmpuxjxgarfzgvygdgxgztlmyxfpkp +sk-xhlgjmwmtkahrpdncwoqynjdrkekmsyftwqmomsodbggvbdd +sk-ynqqptobbeazmjyrmaytsvyczsqwrukpezizrlcloncxtwvc +sk-wswttgfrxrwijvqhctfilhvlxgdkgogrjhvjkdbzvqrocofa +sk-jdijeubeygjmqtxwryrbwmrpvqawinzwpcxodpolhcupzmpa +sk-xbloemctsowwicjvrtrrewreosnfojoijtygsfxfnjntridv +sk-isovavcefvkzlbjewnumeqqevmnoucojsxwskkitfktkemtq +sk-vxrlvvdzgythgyycuqehdloubxcdwhgojpowgxvgxsstjtvk \ No newline at end of file diff --git a/data/nouns/merged_nouns.json b/data/nouns/merged_nouns.json index 47c7dd0..4acedef 100644 --- a/data/nouns/merged_nouns.json +++ b/data/nouns/merged_nouns.json @@ -26843,14 +26843,6 @@ "synonymous": [], "description": "" }, - { - "name": "zwzj工程", - "synonymous": [ - "博微电力建设计价通的概预算工程、建安预算工程或南网规约工程", - "18主网定额概预算工程、建安预算工程" - ], - "description": "博微电力建设计价通软件编制的定额计价的电力建设工程概预算工程、国网施工图预算综合单价法工程" - }, { "name": "其他费用的输出复选框", "synonymous": [ diff --git a/data/nouns/professional_nouns_index/index.faiss b/data/nouns/professional_nouns_index/index.faiss index a6877d9..fe256ce 100644 Binary files a/data/nouns/professional_nouns_index/index.faiss and b/data/nouns/professional_nouns_index/index.faiss differ diff --git a/data/nouns/professional_nouns_index/index.pkl b/data/nouns/professional_nouns_index/index.pkl index e293680..df153a9 100644 Binary files a/data/nouns/professional_nouns_index/index.pkl and b/data/nouns/professional_nouns_index/index.pkl differ diff --git a/rag2_0/demo/intent_recognition_example.py b/rag2_0/demo/intent_recognition_example.py index 679fc89..1decac8 100755 --- a/rag2_0/demo/intent_recognition_example.py +++ b/rag2_0/demo/intent_recognition_example.py @@ -63,7 +63,11 @@ def process_query(recognizer: IntentRecognizer, query: str, conversation_context try: # 使用新的process_query_with_slots方法处理查询 # result = recognizer.process_query_with_slots(query) - result = recognizer.process_query(query, conversation_context=conversation_context, chat_history=chat_history, previous_slots=previous_slots) + result = recognizer.process_query(query, + conversation_context=conversation_context, + chat_history=chat_history, + previous_slots=previous_slots, + enable_query_expansion=True) # 提取分类信息 classification = result["classification"] @@ -176,7 +180,7 @@ def save_results_to_excel(results, output_file, is_final=False): logging.info(f"已保存{len(valid_results)}条结果至: {temp_output_file}") # 示例查询 -examples_query = """D3软件结算工程怎么解锁清单""" +examples_query = """主网电力建设计价通软件, 35kV的软件 土质比例不能一起设置吗""" conversation_context="" chat_history=[ { diff --git a/rag2_0/demo/validate_excel_data_batch.py b/rag2_0/demo/validate_excel_data_batch.py index dcb003b..33d1c55 100755 --- a/rag2_0/demo/validate_excel_data_batch.py +++ b/rag2_0/demo/validate_excel_data_batch.py @@ -98,7 +98,7 @@ class ExcelDataValidator: logging.error(f"读取Excel文件时出错: {e}") return None - def validate_classification(self, llm, query, vertical_class, sub_class): + def validate_classification(self, llm:OpenAiLLM , query:str, vertical_class:str, sub_class:str): """ 验证问题分类是否正确 diff --git a/rag2_0/tool/APIKeyManager.py b/rag2_0/tool/APIKeyManager.py index e506a18..3de3979 100755 --- a/rag2_0/tool/APIKeyManager.py +++ b/rag2_0/tool/APIKeyManager.py @@ -1,7 +1,7 @@ import os import random import time -from typing import List, Optional, Dict +from typing import List, Optional, Dict,Tuple from threading import Lock import requests @@ -75,11 +75,12 @@ class APIKeyManager: return instance._get_random_api_key() @classmethod - def get_valid_api_keys(cls) -> List[str]: + def get_valid_api_keys(cls,api_key:str) -> Tuple[bool,str]: """ 静态方法：获取有效的API密钥列表 Returns: + Tuple[bool,str]: 是否有效，报错信息 """ # 验证每一个apikey是否有效，无效则删除并打印日志。地址https://api.siliconflow.cn/v1/ import requests @@ -97,19 +98,17 @@ class APIKeyManager: ], "max_tokens": 1 } - instance = cls.get_instance() - for key in instance.api_keys: - headers = headers_template.copy() - headers["Authorization"] = f"Bearer {key}" - try: - resp = requests.post(url, headers=headers, json=data, timeout=8) - if resp.status_code == 200: - valid_api_keys.append(key) - else: - logging.warning(f"API密钥无效（被移除）: {key}, 状态码: {resp.status_code}, 响应: {resp.text}") - except Exception as e: - logging.warning(f"API密钥验证异常（被移除）: {key}, 错误: {e}") - return valid_api_keys + + headers = headers_template.copy() + headers["Authorization"] = f"Bearer {api_key}" + try: + resp = requests.post(url, headers=headers, json=data, timeout=8) + if resp.status_code == 200: + return True,"" + else: + return False,resp.text + except Exception as e: + return False,str(e) @classmethod def count(cls) -> int: @@ -269,19 +268,30 @@ if __name__ == "__main__": # 获取实例并查看使用统计 instance = APIKeyManager.get_instance() + + # 获取有效的API密钥列表 + # valid_api_keys = APIKeyManager.get_valid_api_keys() + # print(f"有效的API密钥列表：{valid_api_keys}") + stats = instance.get_usage_stats() all_balance=0.0 - buy_balance=17 * 10 * 14 # 购买16次，一次10条api_key，每个api_key有14元 + buy_balance=14 * 10 * 14 # 购买18次，一次10条api_key，每个api_key有14元 invalid_api_keys = [] for key, data in stats.items(): usage_stats = APIKeyManager.get_key_usage_stats(key) all_balance+=float(usage_stats['data']['balance']) - print(f"api key:{key}---赠送余额：{usage_stats['data']['balance']}元") + valid,err_info = APIKeyManager.get_valid_api_keys(key) + if not valid: + print(f"api_key:{key}---赠送余额：{usage_stats['data']['balance']}元---报错信息：{err_info}") + # invalid_api_keys.append(key) + else: + print(f"api_key:{key}---赠送余额：{usage_stats['data']['balance']}元") if float(usage_stats['data']['balance']) == 0: invalid_api_keys.append(key) - print(f"剩余总赠送余额：{all_balance}元，累计消耗：{buy_balance-all_balance}元") - print(f"无效的API密钥：{invalid_api_keys}") + invalid_api_keys = list(set(invalid_api_keys)) + print(f"剩余总赠送余额：{all_balance}元，累计消耗：{buy_balance-all_balance}元") + print(f"无效的API密钥,数量：{len(invalid_api_keys)}，列表：{invalid_api_keys}") print(f"开始移除无效的API密钥，并重新保存") APIKeyManager.remove_invalid_api_keys(invalid_api_keys) APIKeyManager.save_api_keys() diff --git a/rag2_0/tool/ModelTool.py b/rag2_0/tool/ModelTool.py index c0ade81..3af2004 100755 --- a/rag2_0/tool/ModelTool.py +++ b/rag2_0/tool/ModelTool.py @@ -165,20 +165,23 @@ class OpenAiLLM: except Exception as e: retry_count += 1 if retry_count == max_retries: - raise RuntimeError(f"调用LLM失败，已重试{max_retries}次: {str(e)}") + raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e else: time.sleep(5*retry_count) # 重试前等待5秒*重试次数 else: - # 创建 Completion 请求. 超时120s - api_key = APIKeyManager.get_api_key() - # 使用with语句创建客户端，确保资源会被正确释放 - with OpenAI(api_key=api_key, base_url=self._url) as client: - completion = client.chat.completions.create( - model=self._model, - messages=[{'role': 'user', 'content': user_prompt}], - **self._kwargs - ) - return completion.choices[0].message + try: + # 创建 Completion 请求. 超时120s + api_key = APIKeyManager.get_api_key() + # 使用with语句创建客户端，确保资源会被正确释放 + with OpenAI(api_key=api_key, base_url=self._url) as client: + completion = client.chat.completions.create( + model=self._model, + messages=[{'role': 'user', 'content': user_prompt}], + **self._kwargs + ) + return completion.choices[0].message + except Exception as e: + raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e if __name__ == "__main__": # 测试重排模型