Compare commits

...

2 Commits

11 changed files with 129 additions and 138 deletions
+1
View File
@@ -9,3 +9,4 @@ rag2_0/demo/Test.py
data/excel/*.xlsx
rag2_0/demo/ProfessionalTermAnalyzer.py
data/logs/*
rag2_0/dify/Test.py
+40 -96
View File
@@ -1,31 +1,5 @@
sk-kvgfuqeqvpmfsccykyoohheshclcrtvjlnewratvrjpkpbkc
sk-aulumxzhvaladchcwgmsxidtdsvzytbpvzqgfuvcxlwbwcgl
sk-otxxemniwhxkdvroszmmkitswwuykosnqoldrkzdoflqpgvw
sk-zlruqobfdbjebyyvkmehakpcvfgnlfbdlbfrepusazzckbnv
sk-rczjqufgdisqplkrmvhaxmdgcboluvxympvzljlreuqeeviq
sk-xfnvcksdgwufsktvmhpqrwpgovsxxtaeehtxnaqjtxmubqzl
sk-wwonvjnowbcxmoyoluynnkjwerghspzdulyidskunkordaft
sk-qmrkfvvbbfssuoreyvwqawoveyowuvxviqzqknotyweqmuog
sk-nprpuknjmikvoaxnwgyshwwwtnifvixpuqtzkzmcacdnvoib
sk-xanwnicepdxfqrfejzuxjcrhdsglfypkoxlcmmtamrtjkork
sk-lvtdgodiaurqyiwdxtdrgxifguychhccqlqkhqctscvqbfgi
sk-aedlbtlmqcttxwnvlfmxzaysamocamqxjceoyqjfgpcowybw
sk-fahdvndjblyjlizamvwcrxnilsgmbgbvwssxgquhkezgpqne
sk-tzludgttzxvpvwayazdbppbauvathdtccafjrhojpemucgyi
sk-ylgoiqxmtxeojdnonthxtweungyzldaqarvjxlqyztlvyrff
sk-asuqbqwdhjcqnvtjlwufyrkrwkobnrbmukzarvcctsgjipdp
sk-nbksjgcngsayoumnsdbkcpnqivnvxjenwpzuazzrkhnsgeoo
sk-jgybgyayxlwoxeijgrjcneqlyusleohgbliuwpsuhocrjsmk
sk-wzjsmwxcbbpcrqivqfzjwufqqjtlwejtncnvbpeicznkwiuh
sk-izdjicdoyillktsihkiapuvwebisehtlgykozrvzfkgncwsc
sk-fcsfmyivfuojsqsditvobfqprdpeunukycpcfnoxkraqevpx
sk-szyjgyxrcvyxpvzfwgmbxnflxngxvcplitcctsdvvrqjgftk
sk-jzbodthsnvjwbyrnynsxrudtqfnbdbrcxebjwjgajocnzqse
sk-fxepossfzpmccibfwqpkluorzqlbtcaplepeugtfzfsctcbl
sk-flhqvziknntednkcgjaxlyzzsrfzjhrzrmteqonajpbiinni
sk-xfregpbbquqbxpiobjzanydsjivrjrnbokzxcqtnhxhyghhe
sk-jrdzerhmvrtvzawkksowbgkggkubwfquplmrxbdhespqgtis
sk-oqehupcveovkjqqtxypqyifidcdissuyehwrkdwgruoyjkpq
sk-jnnmltwtqwuoyagoogzzeraczmyfxhoairiddgayksqdfnbr
sk-eghuepxnbcollzrjwbzqvbnhiiwagkejaclyhvaodeqgwrog
sk-poszkbjdmamimconjustnrxxqusuzlryxkrzkpronlenrmen
@@ -36,76 +10,6 @@ sk-gzdqfoyvulrqscdpjlwlufdecrsyjpmwpkknuhnjsvtyftox
sk-bkcufidsebujopqqwexwxwpmevrpelmvxzdymncvllcyojce
sk-olabhscekudzkyudypkcjvehwqunagubwdmtppugrjmcptwv
sk-zpdqyocliebhqpkuwvebpgcnfjdkvavdltimllmgkthwnwph
sk-gvhchlfelocjniuydusyhhwacnomxnvucjonzkhtqoplnbcr
sk-lzneagvdxhisodndnxnpkntghpkimjmjsebiqdzaoqzuhbla
sk-xotcfdkigykevngedupitbcatjqppxmcibjtcebyoglykuxz
sk-ufydqsdqnwsegaqwtappzwdyzqnoblyunfvslomnnmykedgk
sk-jwasykftbkyjzdqlwcxuicrwzxsbhttilxfefbrozrznpwlv
sk-xngteojwkxmftyaabjdwwgyoadspsowmcpcqobteutdcfmnr
sk-akzkgniebruqrtuqskvlibkpcxjuazhcatysptkfyqivldfn
sk-vpqkxtmcgkggllexchzysuewyfaoexzasoumxngdplzgwksw
sk-fvcsqdbqmdlwxzjyofrilusqcypbfyczogaqwqrjrwvojmer
sk-htjprscvfgskjtjzpxxxjhyymshagogykpawxekrrfbgftyx
sk-vtlgznzanapdinnlpuuygdidejxvlvlziolumckxwntzvxcn
sk-npwaykpqovytxhihxepeaxuwgnzvqvscirwbiffudyebibwe
sk-kduozskjwsbxytajaedemqemimflzoqdrfbbuopuogbxptos
sk-cmyaaofqksxeccaiswwholizxtdhqoszpilbqhmwwablwtwl
sk-zltwpyvzodfszrvqcmjxycfshshmqmgtpghiaulcrtrilqup
sk-svpublkronxohqmjvtdemtyqopuwyknynmxvurkmmoqsfesk
sk-tindfeurhffenyetmblplxgytcgqnbbruatchpddtstnajvq
sk-ttwgosljnfdqbevvfkjewspoasountxeyiesuuwwpuoseatw
sk-ikzwjzfxxnocfqvkhbolmmnojdqfvdxzmlruynfoneahgraq
sk-zomatpdlatgtzvuoeabofeandiiespmsinosenjlmdigkvjk
sk-xypcnibvqgebhwvsudjuxlycwombofshifpltstezflountf
sk-vhtbtqsdyhwsrsitpppjpulbpnwlhjeeaujccggdjvjtucru
sk-ygqyapfsdpxxzihadttugvofrcgwlnsrqjvrqfrsrlgqidch
sk-cczurtilupwyrnqnjpdjltzamjbpyaxnwgzqoykvfwecezhs
sk-hsalsmvnzmvlknpzldciqligtjrvhcjdbjtcqhorptsgoisr
sk-ierjdccoycmipzflzvzjqdmjazmbczxhwejwrjmuouxpcqev
sk-rlthtjzrxkdlfsemijumxiwwsoxkwnjovobtqwprhnducsqi
sk-doocedvfjsmwysxyzpptojnwwuoucxvrknlrdsnomffljems
sk-tgxnplgtwawfnnetghvlzoinksickbynzsgovbrmlrkzzrsy
sk-aggkwzdteggbcgyzimiivphpzibnpmqfbkmgtwyggsjfdymg
sk-tzynazayiglcpzejedpfdftqvtuqynygluorjbbcvrnmsbfl
sk-mcaqucnscgznmtvtzhooxbdafzhcgjgpusnjfbftroedzujk
sk-mlfnxfmwwhturavnsbyedkgrhxfspbfitfyqkllezgdtzhwn
sk-meaqykzphtqqoiswdtbxfqndddmtgvwrdnzdxprozlumrojs
sk-vrtfrtioocmrylrrqpburhkgwmyxfipnissumgufmjpzjqoo
sk-rhknxievjjjzcmkvfwqosnaiglsoufytfkbhhgconiibwriy
sk-wzsvxjfnvpceeoapbzdimpwecckpmcsqmflocttnqyvqvrsr
sk-lflfajincbdljrfwkhrjunyolzuxtcfdbzmspyfizpbdwvvh
sk-siujvueppvhhokbofqlcdxudpswdfwpetvxyittpbarddhsu
sk-avpyhivanabvdyorypdmlqnfetivlwoovfvgsgklumxeqgzy
sk-fvmpnspfeleqrgumwnvzevbtgmdqvftlbvswoghrtxfengfa
sk-wxaeehghpdettyrsnhjrawddnqzxghfjbkkokbwtnbdocsmi
sk-yighqmadjhvixkhjpcknrkapqsyxmphcqvjhlbfzgigfghft
sk-orepiytzucdtxgnxuxhekaivcpzwnunpiauccujrhbkifodc
sk-qsfrireujzrpomdjcfcgbfqncnferyzfcsralnszvurfvkgz
sk-tjienchcxamgldtzjtigpitklsonwcuhyhltswwcnsfhmois
sk-gdtqulaciovvnzwklpckwyheysaplirhgimzreqzkhfxotjh
sk-rqhynqmafsydevjctxeuqizuiithschhiztmqnpawqjjbhyv
sk-idrdvmcgmjmoxowxswfcorfdjmprfukrumimaljjnlwqkazu
sk-hrrmqrhscocbzhpfkzuoptnubgfiedololapvyfftapaawdb
sk-rtqzxuylhryzqnotvkzelvhomuovmbvpxrytydvqevisciwg
sk-ttgsojzjchifbrpwfmcajfujbxexbyvlovgbgpfdvkzqjzpu
sk-tvclqjccfylijwcujuxpppvnumkvmxbctevfqfxjryyygoep
sk-qyazelyawtgyohimgbswwetajekuvmyteweviovvqsjybtwq
sk-zowapfiiqrpeuahmnkopcyccwiyzweigvekiusiuirtsfmbj
sk-qwanpokfsquwwrpxceavasxwmxjarfaxheraamqhsyegfgum
sk-smqpfdbcnhpifwsrpwutwzpscbkeinkmfeoabesztmnicvez
sk-cnvevlfppneiztcgayxiydukhexujtiqjajpdhbrtnenizxe
sk-jmjgjraheompbftegtdzrwcnlyxsfhtrjootwuqirgmodxdm
sk-tfkyyzbhdowskndsnqkniivwhmmfmhnjdzitgvwhmhfrobeh
sk-spjtulrgqkvepawufkloxktfmkyjjhffnmzcththdmnlqkwp
sk-xafrjoiwnundkubilpoxmzpmbsizljfgrerckqgpcnvguqma
sk-dtjwhkzhljzzmchfvoyywjbsdpvgtgnlpzdkgegwnjjnsesc
sk-lurglxyefchyowvybancbtxdavndwvcengfhcrzcdouqsdmp
sk-nckmscferrehepawtytvvzzcexsgbxszmrerkwvzsbvnwckq
sk-hpgusoznbejkugxugsstyezaihqatwmwiwelwjwudekoxlbq
sk-yyqtojpqfrtkyvmhbjvnzhuujzgvitqpuxkytgxspwqscptz
sk-vcwenoaiegiwqhdvalxrvttwqmcrudttpfqhlvtdocsxvvob
sk-npnyztbrdaqtbnldlvfelkfbfozykuwykkkfsrsisytusymj
sk-rjaadeqsgoskclkuqkkyjxdknngrdsjwlgucqutiskroprzk
sk-uollmeyatyiwfzszvxkpyndmzfrbqjpyixewmrastbmaqbhy
sk-xdlsjytiwilvodadkjxvwdgulhhdytkqvfpyrcnllclgzqkb
sk-ffkltifkylutornjhwmnmfjsqsywrjibvujhjtjctzgnkvlp
@@ -116,3 +20,43 @@ sk-nlddwexmjxqtgdvahwvlotnomrzcgskxeakxkxauicknzfkp
sk-lopwluipwvilwpwztvaxfebueeyilefwgncgpeprqvwazxom
sk-rgwrklpvhhrluokkbgavzukuhhpfhqzmozpjzoezfhkxyorc
sk-cdrpglnfmyeeqyhtvxvkpcpwscsbfouwkagjpphuksfzeipy
sk-eyktixexxjqvwufezcmdrazcedtsphyiunhqpamlkrcaxwtg
sk-euzbguamsxqspfdjnrpbchkjkouncqipjvhnkkbvoihgwspe
sk-qlpoqleqaodseswzqklbwjwwcdjrqthbmvweuablibiszpnw
sk-gqjtkwjmrupugviflhsffhkpzxxmjcewviqsneurxnlfqewy
sk-hkxgjpdyxuxjklksunfzaetrhveelucmrldfjnlztibxgjgl
sk-fqjkatqvpkmvlkbqhjfkzmiifmiodayututyprdtldszmacd
sk-wrybirbfwtbfdijjrfpdfxlxzmvcrhfgqqbhyuysibmcmcez
sk-vgkpsmbcchymktakjsheqnjlkopiqcvntqcvxuxmxlllifod
sk-afvrxowsmjmlhguhfpfhmefcldzsohmnyumjdwwkrvzwjrym
sk-jwfemfolekojghqwfwvjxwmzdvdnhaznngqeurvzgwqepddp
sk-vxzwcizvmykzwbxaypvshdkbgjymnmelryyqftkxzxnnpibr
sk-bwcbyczfakqkwrsvmmraeikjfpiurhhanflfqkunpzezvagv
sk-dicoheyoylocrtjsacbvwqzwwibnnasuwhnhsudoemieengi
sk-mvuzqvuqmryqrqolztwthhzcnzibygjycdbeplbbcnbxkncq
sk-qavyzqmnfjbhqhuesfkjcyszkzmjsykpnaebuwzcxhkitoir
sk-jhrbmqmzyrlrqyehkkachwhztkuisphqbmoyvchmaqrcfxtf
sk-mtcfttyljuzteasdjbcrtwwwmqxetmaonbwvojzfijzfiplq
sk-iooturknmbpxhbiulovovtzpyayovyidifzpzkjgqacroxjt
sk-jgjhasobxeuzfgutdyuhcejocwdiwonhkdithchkonhxnawc
sk-mvyqkciotllopyozsfzwtjeicuhnvoihnrrxadfsfiakperx
sk-wticrisyjehvnlrlhmmxhrnlzknqpnkfxowlzvnozskvtvzc
sk-hiniqrfvuqlsgmhrqlezlribsaqdefuhpxxfavoqtszxtasw
sk-zlfncovfrpzczmjquirolpogdrzfarkcwwqkluvifwcvrezq
sk-blhnvpenedysngftlghrkxhweoququkvduikziuypzilpyrp
sk-rynvudqktwvnjleiahwdpeqqdkncsvawvjyicyiojoviiges
sk-ncskqkwomgqpfnfnachehkeaczzgiiuripeyzjrpnuzeosnn
sk-ijqdmtyeuqrbndqjggxyicfjrmpsgbsjwwkitgsmxqvcjrri
sk-ajgisqnkpgoiwxigrnjachusupagqpukteuknemhmnxsasre
sk-zsskrhjnoepgjngcsseklxfpwpozhenurtrluxlstxujdsti
sk-lubrliuefgmrxpfafdwrhzletyvhemqkpvriuuqncivlewgx
sk-pavwfgrpftdtzeiiaoousmkdptwujoeocyzzoudeqkfyoxks
sk-jfwhzkhpgxacedwxzkbwrwvlfqvnlhxaeyghjcmtshogsqub
sk-sqmaankkcsqpbtktdfcmpuxjxgarfzgvygdgxgztlmyxfpkp
sk-xhlgjmwmtkahrpdncwoqynjdrkekmsyftwqmomsodbggvbdd
sk-ynqqptobbeazmjyrmaytsvyczsqwrukpezizrlcloncxtwvc
sk-wswttgfrxrwijvqhctfilhvlxgdkgogrjhvjkdbzvqrocofa
sk-jdijeubeygjmqtxwryrbwmrpvqawinzwpcxodpolhcupzmpa
sk-xbloemctsowwicjvrtrrewreosnfojoijtygsfxfnjntridv
sk-isovavcefvkzlbjewnumeqqevmnoucojsxwskkitfktkemtq
sk-vxrlvvdzgythgyycuqehdloubxcdwhgojpowgxvgxsstjtvk
-8
View File
@@ -26843,14 +26843,6 @@
"synonymous": [],
"description": ""
},
{
"name": "zwzj工程",
"synonymous": [
"博微电力建设计价通的概预算工程、建安预算工程或南网规约工程",
"18主网定额概预算工程、建安预算工程"
],
"description": "博微电力建设计价通软件编制的定额计价的电力建设工程概预算工程、国网施工图预算综合单价法工程"
},
{
"name": "其他费用的输出复选框",
"synonymous": [
Binary file not shown.
Binary file not shown.
+6 -2
View File
@@ -63,7 +63,11 @@ def process_query(recognizer: IntentRecognizer, query: str, conversation_context
try:
# 使用新的process_query_with_slots方法处理查询
# result = recognizer.process_query_with_slots(query)
result = recognizer.process_query(query, conversation_context=conversation_context, chat_history=chat_history, previous_slots=previous_slots)
result = recognizer.process_query(query,
conversation_context=conversation_context,
chat_history=chat_history,
previous_slots=previous_slots,
enable_query_expansion=True)
# 提取分类信息
classification = result["classification"]
@@ -176,7 +180,7 @@ def save_results_to_excel(results, output_file, is_final=False):
logging.info(f"已保存{len(valid_results)}条结果至: {temp_output_file}")
# 示例查询
examples_query = """D3软件结算工程怎么解锁清单"""
examples_query = """主网电力建设计价通软件, 35kV的软件 土质比例不能一起设置吗"""
conversation_context=""
chat_history=[
{
+1 -1
View File
@@ -98,7 +98,7 @@ class ExcelDataValidator:
logging.error(f"读取Excel文件时出错: {e}")
return None
def validate_classification(self, llm, query, vertical_class, sub_class):
def validate_classification(self, llm:OpenAiLLM , query:str, vertical_class:str, sub_class:str):
"""
验证问题分类是否正确
+37
View File
@@ -457,3 +457,40 @@ class KnowledgeBaseClient(DifyClient):
data = {"segment": segment_data}
url = f"/datasets/{self._get_dataset_id()}/documents/{document_id}/segments/{segment_id}"
return self._send_request("POST", url, json=data, **kwargs)
def retrieve(
self,
query: str,
retrieval_model: dict | None = None,
external_retrieval_model: dict | None = None,
**kwargs
):
"""
Retrieve from knowledge base.
:param query: Query keyword for retrieval
:param retrieval_model: Retrieval parameters (optional), example:
{
"search_method": "semantic_search", # One of: "keyword_search", "semantic_search", "full_text_search", "hybrid_search"
"reranking_enable": True, # Optional, whether to enable reranking
"reranking_mode": { # Optional, required if reranking_enable is True
"reranking_provider_name": "provider_name",
"reranking_model_name": "model_name"
},
"weights": 0.5, # Optional, weight for semantic search in hybrid search mode
"top_k": 10, # Optional, number of results to return
"score_threshold_enabled": True, # Optional, whether to enable score threshold
"score_threshold": 0.5 # Optional, score threshold value
}
:param external_retrieval_model: Unused field
:return: Response from the API
"""
data = {"query": query}
if retrieval_model is not None:
data["retrieval_model"] = retrieval_model
if external_retrieval_model is not None:
data["external_retrieval_model"] = external_retrieval_model
url = f"/datasets/{self._get_dataset_id()}/retrieve"
return self._send_request("POST", url, json=data, **kwargs)
+23 -13
View File
@@ -1,7 +1,7 @@
import os
import random
import time
from typing import List, Optional, Dict
from typing import List, Optional, Dict,Tuple
from threading import Lock
import requests
@@ -75,11 +75,12 @@ class APIKeyManager:
return instance._get_random_api_key()
@classmethod
def get_valid_api_keys(cls) -> List[str]:
def get_valid_api_keys(cls,api_key:str) -> Tuple[bool,str]:
"""
静态方法:获取有效的API密钥列表
Returns:
Tuple[bool,str]: 是否有效,报错信息
"""
# 验证每一个apikey是否有效,无效则删除并打印日志。地址https://api.siliconflow.cn/v1/
import requests
@@ -97,19 +98,17 @@ class APIKeyManager:
],
"max_tokens": 1
}
instance = cls.get_instance()
for key in instance.api_keys:
headers = headers_template.copy()
headers["Authorization"] = f"Bearer {key}"
headers["Authorization"] = f"Bearer {api_key}"
try:
resp = requests.post(url, headers=headers, json=data, timeout=8)
if resp.status_code == 200:
valid_api_keys.append(key)
return True,""
else:
logging.warning(f"API密钥无效(被移除): {key}, 状态码: {resp.status_code}, 响应: {resp.text}")
return False,resp.text
except Exception as e:
logging.warning(f"API密钥验证异常(被移除): {key}, 错误: {e}")
return valid_api_keys
return False,str(e)
@classmethod
def count(cls) -> int:
@@ -269,19 +268,30 @@ if __name__ == "__main__":
# 获取实例并查看使用统计
instance = APIKeyManager.get_instance()
# 获取有效的API密钥列表
# valid_api_keys = APIKeyManager.get_valid_api_keys()
# print(f"有效的API密钥列表:{valid_api_keys}")
stats = instance.get_usage_stats()
all_balance=0.0
buy_balance=17 * 10 * 14 # 购买16次,一次10条api_key,每个api_key有14元
buy_balance=14 * 10 * 14 # 购买18次,一次10条api_key,每个api_key有14元
invalid_api_keys = []
for key, data in stats.items():
usage_stats = APIKeyManager.get_key_usage_stats(key)
all_balance+=float(usage_stats['data']['balance'])
print(f"api key:{key}---赠送余额:{usage_stats['data']['balance']}")
valid,err_info = APIKeyManager.get_valid_api_keys(key)
if not valid:
print(f"api_key:{key}---赠送余额:{usage_stats['data']['balance']}元---报错信息:{err_info}")
# invalid_api_keys.append(key)
else:
print(f"api_key:{key}---赠送余额:{usage_stats['data']['balance']}")
if float(usage_stats['data']['balance']) == 0:
invalid_api_keys.append(key)
print(f"剩余总赠送余额:{all_balance}元,累计消耗:{buy_balance-all_balance}")
print(f"无效的API密钥:{invalid_api_keys}")
invalid_api_keys = list(set(invalid_api_keys))
print(f"剩余总赠送余额:{all_balance}元,累计消耗:{buy_balance-all_balance}")
print(f"无效的API密钥,数量:{len(invalid_api_keys)},列表:{invalid_api_keys}")
print(f"开始移除无效的API密钥,并重新保存")
APIKeyManager.remove_invalid_api_keys(invalid_api_keys)
APIKeyManager.save_api_keys()
+4 -1
View File
@@ -165,10 +165,11 @@ class OpenAiLLM:
except Exception as e:
retry_count += 1
if retry_count == max_retries:
raise RuntimeError(f"调用LLM失败,已重试{max_retries}次: {str(e)}")
raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e
else:
time.sleep(5*retry_count) # 重试前等待5秒*重试次数
else:
try:
# 创建 Completion 请求. 超时120s
api_key = APIKeyManager.get_api_key()
# 使用with语句创建客户端,确保资源会被正确释放
@@ -179,6 +180,8 @@ class OpenAiLLM:
**self._kwargs
)
return completion.choices[0].message
except Exception as e:
raise RuntimeError(f"OpenAiLLM:invoke:error:{str(e)}.api_key:{api_key}") from e
if __name__ == "__main__":
# 测试重排模型