更新.gitignore以忽略临时文件，修改api_key文件，重构合并名词的逻辑，删除不再使用的脚本，优化对话到工单的处理流程，添加会话结果保存为JSON的功能，调整API调用参数，修复部分代码中的错误。
This commit is contained in:
@@ -14,3 +14,4 @@ rag2_0/dify/Test.py
|
||||
data/query_logs/*
|
||||
data/conversations/*
|
||||
data/test*
|
||||
data/temp*
|
||||
-381
@@ -1,10 +1,3 @@
|
||||
sk-fqiorgwugzcrnwrkcnfsiaikkthgzzupzrvnwidbdmehpidn
|
||||
sk-zdhbhipwbffoqmoxfxjsynfyjximahraxodzadiirnpbuzsg
|
||||
sk-wsqrpkdpkszwdebckaxxbsjhovfwlosctzfpujxxpvhqkvyz
|
||||
sk-krrffrphigolboeegpogoksdyeuxbaohiaxtflandfclmxsk
|
||||
sk-hsbehxbxnpigxoootnjqlpckcezhlhypdgnabpbywnmquiqe
|
||||
sk-fonqeeexzdgocliiwffopgdesvcmnlqtditsafklqgnqyxvm
|
||||
sk-hjhftlgfeswvcsuowabthdkriaevhppvbgoriyiydgahfrgk
|
||||
sk-cxlvgeuxavxfcajprxietuqyqjngtbrwrmrmrioxmgtbkpci
|
||||
sk-vjjsuzntqbhcmelfsuquqyoxjivxcfwyxnrhpwzobgxlpmrv
|
||||
sk-hbgctnpvntsnelveaudpekyncfgstdfazezboxmcgjvudzyg
|
||||
@@ -15,26 +8,6 @@ sk-kdfrkixwoeiuzdsdtystcxbhvwzeschacrnvhfiupnilnpoo
|
||||
sk-kelmbfsruwmjgrzzbjkqpqasabrnuskongtqxwzrfrpefdgz
|
||||
sk-mtnlhcosddoztgmdgjmtlniqsrkfiskafmthtmnyzqmdfcib
|
||||
sk-cnrleygygpuqogpqyvuqnhubkwschrtcpjlldsdhxrkhgttf
|
||||
sk-hycclhscssbpyorblvaalhugdacpsfhwppihzlxckiqvqqdp
|
||||
sk-qpwzzrvywgblfzsvwachxpuawfuffyqichnkfxwnotgzxvic
|
||||
sk-qhbearldhnfaiipdnysouxorzqufpntslmlpceymdwgyxbub
|
||||
sk-ehaxrzfpfzzjgonerqpnbldhrgfhpbfrokasmalduycvkxuy
|
||||
sk-ckjrxmiwpmcehxnxqkyycicsstxqmibayymivijokaygafcy
|
||||
sk-dayrbzxeeiuemrgzytqcijdfeuapstvzzsjifhfuizdghawn
|
||||
sk-ojllzcltirvfxcfpuzjhxihsbyhdqpbrbxchlnkcvswldhpi
|
||||
sk-cofofgsmlsputithtgbkeuwpeiijiprwulslchadgnxhgszh
|
||||
sk-xitvnuyghqbdwvmgwjcfpsgrvfissovdornhynhmiqfuvxyl
|
||||
sk-ynmlgshhqbfuzgcnrkmdvepdmnicskhvdzvyibdfctruwplc
|
||||
sk-dgijfputvuywypkciylwtlqmsqolosvcmapfiwqjzsiodxmw
|
||||
sk-fkhreothoprxhfgymwdyfujmgqpvqbuozmbihvyhfggysdbb
|
||||
sk-rbixbbhxgqqzgziprzavfkcoawjeyhvowanbiyiaguqxdztf
|
||||
sk-nbknzrprpqkhyevsxthqoyeuehjuuubptrcxnlzoqviuqnck
|
||||
sk-scsexswqgeykaikligbjdjljqywcwlckeqkklawjoyrjalkq
|
||||
sk-kbaapbyjpsylehbgyieenaglcjwbjyihokeocshlgypkjkzu
|
||||
sk-mblgxsdlkdtzeckqmkcspiomeakgstdfduqvzjcbprhyqmka
|
||||
sk-dcmlgeidlpigamazrcrbdenugyhrmiunrnjpuxdjuduqdckh
|
||||
sk-dianaiylqnlcoljmxnfbsqowwcyctsuahfvztcawwwmxesus
|
||||
sk-mvhlbzbfkrtwgriasuzbvvnfcfkirslhchlbknjndrkwyufx
|
||||
sk-xgkzyzptybrnxanvugkynczygyfavddwwzhbgoyxkhgaykci
|
||||
sk-nfpftnxyiovqlzkqsleadsfnbpfxdxbdpghdwfyucrnzaimh
|
||||
sk-uiuhaihdvthpmwqeehmwufagpxcjunoqnfgpnaitaaqgrjoi
|
||||
@@ -128,49 +101,6 @@ sk-ufmqbuplpjvzzlzohvsxertwgnguhipsbajxnxecvvccozly
|
||||
sk-rypfoscrczeelowmrsixiuyunyqmqvknaprsnzmdguwzrkzx
|
||||
sk-lucemnosmcxuwedvzilpefuxjnyvaxldpbgaqwnwalxmntul
|
||||
sk-niymkyuzpyovndvvqvpaniiqfgoofnxczhdmjjessiocbeul
|
||||
sk-ymrktzmmacferrdkbohbnxzqnhzqnayfopmeaallsjnehqcw
|
||||
sk-zgrnzafrveugvkqthacmjvzmrhrasmjstpldkenmtapyvmvh
|
||||
sk-lyutswiifkfuzyzfuzhrtydmwiblytrmavzwkhwzafrkpkvt
|
||||
sk-sfvenswvedgsmuqfqwlphxmawdsnfruuxrksqsumclmkcepo
|
||||
sk-tqmpqwdczmvdymylfzsaymwpxopthsquwxbrgajbfqzoquuh
|
||||
sk-azajlrrajctqzeqlkdiifogcigkqikzvtlmxicftrqbtfiwj
|
||||
sk-poqooaejvsycpwveujigfbbqukuiftbhfnaamdcnzbeofbgz
|
||||
sk-ackysjgqtlxikiuhfjspflaspjyvndrydeoyaflfkrjedqzw
|
||||
sk-taddhesluxmdzaaqhkakemyputzuxkhvziregheczzvbicbe
|
||||
sk-wwnbxazrsjzojiwddwktxytxcrnclhtqnxreanzvvsulmeqa
|
||||
sk-yokhwoynevvdimlcfhgjkczwnmlvpegiinpydyomukjvcpoa
|
||||
sk-wwzquvlaxrxskxvystkrzqppiufjreaibofudrtxhquepktg
|
||||
sk-akvqccqgqeqltfqotmpglxlhqnkxthefwrfrannjiqgfexnd
|
||||
sk-bpergewkadzmoyipuwvvkmcenljxvvxocaapswmqwwzhctwu
|
||||
sk-xygqvydpnpzrrtqjmarayjzlmwkvertuxviejezgbhqjsono
|
||||
sk-yneqvefpfddedlhbmzixrjayhyafysiwuoaiccwpcnwlihvn
|
||||
sk-rbfzsdhnpsfwevtuaytcaerqvjjkyjdrksyfsizzcfslivdy
|
||||
sk-lnwngttbwhlshqimlnlzcogpdewuexoloxtsjgabsnrxiwum
|
||||
sk-zlgdfdxsotldyhgvapvwogxurjayztvsbnokzsoxepiqngwx
|
||||
sk-jqqmmvivlqqahobcznjpfsjuhmrypjwzfzzdfjeuanmgrjvj
|
||||
sk-kwgkxwwjpodvdfzcqcsildhwftfwqdufbfuqwzthsxnvuawb
|
||||
sk-pkhanejghbxnyskcrhheogvtlkibspvlnxseanvywwwphyzx
|
||||
sk-qhmotavtivehqghttsziemwywwkzffodysdrzedpmcplgjse
|
||||
sk-jspkuwzvusqozxthglfhuzhjurqgqmxgfjvfuatkwboutoow
|
||||
sk-mdakwcubtvvtomakhdtepaoxejcgbypxadnyccrehjztmrvz
|
||||
sk-dpawjyvlmfjbamtkqvdqfavwmwfilwonfcnhpcmudydwdroh
|
||||
sk-givozfwetofaxqyocrhzsvfoqfrkuhjocorffgkyrquaxhql
|
||||
sk-agqbvfcicmxobifpmpqiivkddiicolyhxsxelknkoiqsxczy
|
||||
sk-hywltpywcdhtcuxgrsydtnnpbihglytgaffqifubtanslgkb
|
||||
sk-enlmvvueqloxhickqhnyzyhchjaxolccxeihjgfegubxqdau
|
||||
sk-njinpeycwccspxyvwxjjurgccolpjwkyrvshruqrgqeyuket
|
||||
sk-kaofxorgblnyhyikderbhojucapccgiaabmrjvtprpjdidyr
|
||||
sk-lhlnnrjmdyadbpotbzjhexvpqdsakkawzamgvuvvvwrjxfuw
|
||||
sk-kbdxyxsqrjzxabmlnudbfmkoueetvozavetvynzebxeywtiz
|
||||
sk-pmkpljgfuobflxamthdiciwxxrohttudjacvnelxucfepsds
|
||||
sk-buouosufejuluopcastaumdwdkxtojrwtenvuolrnwgnogrz
|
||||
sk-xuptfizbgnytnfylewbdyqzktoeosinvjqpvmilfjjoenjcf
|
||||
sk-lzqwcqiwgxowodrhyewykbeazqemdiynvanwgxjilhgabqtp
|
||||
sk-ogasgwkcgvdtuforliczletkpteoininxtzwxiwgqyjxynuf
|
||||
sk-ijmecfxhdsnzrolannjmpedwxwckdjbxrofygqzpkpwfhcar
|
||||
sk-uuqiaramzroyhjtywxpfiulieesatwdbugpomxqxgfvuhryz
|
||||
sk-uouyvhsnnnjrxpjhacuehosbnksgnvtvhfdryoscwicuvcyl
|
||||
sk-exsjriliucrifygufvjhycgycdinafmdegnorlprigpugask
|
||||
sk-gkdrgcbrlddkgflolrqkpqyygmuszueozeoiwdqpeqghlths
|
||||
sk-gjqsqttmnydxavthjsjnirqxprogmvidmocfeoixuwutiqmd
|
||||
sk-pihgxlhhdsyeowjsbhbioxwqjbonwunteaubfisduzyibkzw
|
||||
@@ -191,126 +121,6 @@ sk-mtdxvrivtguhurniwlkkpiiqvsldzupdxeikktjjcecokprb
|
||||
sk-mlsnfruyphvjexbbxmpfzmqrfolmnmvagtartricqtvcfdcm
|
||||
sk-ofwsljbaccfrlecrwelbzwaervcdhayakfccltttqklfvjbq
|
||||
sk-hrocudrwzukputieyvwyqmxexladghassxhdpyofitevmlwa
|
||||
sk-odakxyiimzxrgeqxbwqdstrsebubrfwpyjlylnmvsrscnkha
|
||||
sk-cdgfctificqxybfnnqplpqiwjakocobsqssudngsvxyshbuu
|
||||
sk-etdalqllnekhfihnybtvbwbdvwgmjfqgbersdgzyznqlmouw
|
||||
sk-oonkwqqvblmoktvfhvzaxpinbgvwzqpwdhqhckessqnfgzhd
|
||||
sk-nhosrcqzlmajdecooxnarjuzsnymzmvgsyckjftkokjzahzd
|
||||
sk-nrbczpgnvfpeunrxxueavbbnnfqfqwxjsnvjbuzxeyuqqcpl
|
||||
sk-cobqudjpmsvjyazzdfirhhvhxggjnupqpbhyniavpdovmtte
|
||||
sk-fwqomjfvbjdwvnsansrkmvxriflylzbvtwosybuthopxnmlz
|
||||
sk-mlfemdejrncmwrtdqvjxwseeevoltaapteawikshisjnqasb
|
||||
sk-jpwmjlzlswykpjotranggkwljmujweizsgunsswwzvzjoppo
|
||||
sk-gqtxgkgzyubkggfyzddsuukamnreqnwsbhfllymflptuuhwe
|
||||
sk-nnbkwmaychnfrvwnuqfwvhabrdnpjzkftdkzgcygnryjgwjg
|
||||
sk-odxaszrrcpldjdbpaufpbwisulwsicekcmzbydvgyuxwwptb
|
||||
sk-gasievqwwlecetmorcekmpwvkjseykkyjvudxpvvmilxklak
|
||||
sk-cmrpqbrshozujdasrdmzaatiuhvdweswxfimoplaztultitn
|
||||
sk-elbtjdkqzpdvabhuxdfwozekfxmnoojazdkjgiocgpcxlnxv
|
||||
sk-tjicvjxhtceicrriirsdqcmciobuobbtcarthlypruxurirs
|
||||
sk-hstobggtwmsjewhixwkmaqgpqxwafcibehmdrondxuimqnpq
|
||||
sk-urlllffydhsqnexmlguxlvuyhbjzovzvkmzhhmuyirqwmryx
|
||||
sk-opdnnbzwnmguzsrsyhwcwzfqtzcjeteenutpqeaapibnpfaq
|
||||
sk-ckierrmqzhahjwxtizavwwwgiiehijbhjctgvlugzytylrug
|
||||
sk-ewcebklekmctyzfstecfsxzvjegvotijtltlgeequzcpagqv
|
||||
sk-mgzhwhxkmarcycfbzvrlgxlqikozpsfcxnhcrnkdvcqwqksy
|
||||
sk-rmyjpdhytpioibzqffijzshstpolxvcjghxhgmzzxomayiyw
|
||||
sk-yyhyekrhcnljpynqdoxsqovsauxgyqqiwgxqavykjwpjusqd
|
||||
sk-gxapgxldxpjhdznladnwtqkdjqmdgznjmezsksczifbjkbpk
|
||||
sk-kbrjttdnrfbuvzjakhohavoxorlynofradwwxdttlrqtlezx
|
||||
sk-zknrvhlmqgknlwiphvmnwlulhrtwicrkvkkfpnkpzxgbnthv
|
||||
sk-xbddhilpckwvzxacqjiocmaymbtqlahzwoyzrbdwqidmejvz
|
||||
sk-ueseyqbtfikezynjsenkehpwqqzooeocoabkithtmudmihsr
|
||||
sk-zrbtpufhxbeixieaykkcadcntipflbwrhfiicuaxopujvdvf
|
||||
sk-dnlydvatnswufwkchxdqiuaroxnmozfwvhhotxinrwzjytct
|
||||
sk-jhjebzqwkngjjgnpelwzcqlzmbczprkcjzzyumfbhsfvsqwg
|
||||
sk-xgbwqgrfznxkebewuqmzohfnhpbuxprmccfjemucyeofnbge
|
||||
sk-jmngmbdxbwagjofvsagutritvwqogyancfavtvlraoukibyz
|
||||
sk-qhiuffgnpabnamfyzffdnmgoevmtnhmsjyglqcjhvwublvgn
|
||||
sk-edzbujylbkvsvfadtcfzitavymeshrmesjsdsllcpubwrwto
|
||||
sk-rqpsvficelzysisksmsosmapjrrbqaaubiwluhcihmkgnezf
|
||||
sk-hkkcehvotmyurxbtzqzxkesyfnulqxnjycfxrfcjsqjsjqlm
|
||||
sk-lqdqjzumoqtcnvdhicpzfgmbmyjocyuhmfjzmbstijtjjyux
|
||||
sk-hwqptxptjgzdlppdffhinqjpxgiycidjmllbzzhslqraedwn
|
||||
sk-enjmyliemusvcrydmljjauvzwdplnwkfxfucwbhexcylxcgq
|
||||
sk-nurshorcqojkepxiefwyjgkbxeohdadabmcneenkkvodgset
|
||||
sk-pnvtsjmmcmgeetafqfskumilkxiflwcgqmqqpwurrkqdddew
|
||||
sk-lgygzwevhgnqlvvjzrwtwgbjsxfhucbzszydawpjjmkyiaof
|
||||
sk-cczpctnalpbifauarekxbijlpostgydhhnyyljggoipdgncx
|
||||
sk-evoyieozyhttwnnuwefzafclgasnsxgaslfqabiefaljqnhr
|
||||
sk-tqzgtdgrkglowsehrzqrfcarhadtjwypyxghktcjgucyppfb
|
||||
sk-iubygvdrbzqchsmlcpzrwnlghwlnjsyxqhoknegkdlanzkwh
|
||||
sk-vxkjotcvrbggozqhscrsmghxtsrnjjgbektdbvuxodjjkyju
|
||||
sk-nrkgnrmgpuygzstkodmiqpgvdzocokxkweijfjptwyypgdyy
|
||||
sk-bjogsxmvdpiwoqkrglouyadwkkfgrwjmzfwrzbscvyxjjwyb
|
||||
sk-gmtoauotodrgqdhoruxkyyjdkaemfcfveyklkrsyzdteqbmq
|
||||
sk-clxaqlxyhhvqarjvqncidzgldnpguvwkqfqdqwytsouvayii
|
||||
sk-fvvjzijhlhykvxobuqxzlphrtuslzirpjcxieohelcykiltv
|
||||
sk-ritcploocwoprekoxbkolrlyilbengvvkbncifgkqcrtspsw
|
||||
sk-iqvrozujapnnosyztxnodwgnlcoverpagxnohnpoadphmitc
|
||||
sk-byleqeovamwprlaykzgucctkksqcreblnafmeprbbvvbuihz
|
||||
sk-zjnfveqeyryargtgysszbecduuieektgfmttvrpvexxvkakg
|
||||
sk-iueoowqajryvqfcouyesvotdafzftccfwsvawqlabclhyyqv
|
||||
sk-pwhnwlmhttwxruafxquvnqeckpvwgpdvyppybobynuwzghjt
|
||||
sk-gbdtryfivlfmrbzxkoymjkujjtuicvjlnynmswyxbokoogjz
|
||||
sk-aguuqmyoxztbytcqefsazfzpcqmpyaikvuqsuvbrfmkbixld
|
||||
sk-yajbtnqdskoyvtnlqtvjbiovttjgihpyrdvzxygbsmlstviu
|
||||
sk-uzukjlqctiqjqumkagaarttqhcgmxcqansqueswefiiomwfe
|
||||
sk-ivgrwasidrharxlsnjajwnbjmqddmiztlqiikmowocqtsgds
|
||||
sk-ibggmzmjsuufgnmgtlrbbwmxiuvpwdclqhisqlxfokuxdydh
|
||||
sk-lprfpppdrmvnbvdhghpywgqhcgnldzegxhyfhwvpidkmqqrh
|
||||
sk-lqvrxzqesqchefxvbraczpiszykuyflckfhtyubxlfqkwttm
|
||||
sk-loezhefsmqhzwlstqkeynfnpgjclpbizxrjcgzutvaxzeawg
|
||||
sk-ydmfxaqtwhqcvsgoxxxvvfhmpepeknzbypgdgzugjuqbyduh
|
||||
sk-hzsvoziyxfmdbqloguctveimdxfiwclvrymkyohxaympdqsg
|
||||
sk-gkxyvdiadzfcwcsosyctvcajoxomlrhqnnojevvmyuhwchjy
|
||||
sk-pppdgqssjhqistbrreapfrdbqoncebjmqvwclbcnkikfyaig
|
||||
sk-ufixudywbeffcsuknffxowczzdbykwoovkjlgjjpcirgdjjd
|
||||
sk-plhbmyzqfnxvmcqlhuhywbkfgjqbevpdzuumvqrqiaudcldy
|
||||
sk-yjoitaywenwwrlllegmphporcwxpgpslzzmqararohdkgxfa
|
||||
sk-mssalskekreuyztesydcbmzwvubfxxxxfjndcuzhudulgaaz
|
||||
sk-vtkfdwzvxnhfvncqfmrklgpxpmwkpxdgrryradskbprpbgln
|
||||
sk-ixutedkcbcuwamiynidhtioqvxwzaypbqrahqgnzukslljyf
|
||||
sk-sjwasvpkfvmoruypknbeudbsxebmgzsyocnznvuruhtbxkdp
|
||||
sk-btgppgymypnzginwwfzwtvpcsjjsmyveofqopcpegceroxei
|
||||
sk-ufgenhaozdkykuoczjfywvsthsoxnnphxjaroeejouwffheb
|
||||
sk-umhxbxavxtlrizrcvwzzqggykpruojebdelwqpxoegurzizk
|
||||
sk-infbwvcscwfeovpnziofepodzfvcynpplpgagfouqssgnnmc
|
||||
sk-sfahqxtknyovnecotshqcojrtadadxvmgjbkzpmfwcmapnhr
|
||||
sk-szagfyiyjftymfswkoplmblxaofbnjrllalxzsekyduqbmrm
|
||||
sk-uzallmwkhjnswjhhyiubfrkskzkcpbkwpgppsklnotmawfut
|
||||
sk-hnefjgjgiolrmsdpbeqvysvaypfwtlsnewpudjzyieqeewze
|
||||
sk-slhumtzsjszyrodyufufmuqyjrqzglrfkvjxdacvquyrctsk
|
||||
sk-uqzbkzzzpmpjeliwgxpugsrmhypxtbuwnmbudtjbsksznffw
|
||||
sk-mimztgfgojmjhxglhuhqkvdritcalbuckihkrwdazatslmli
|
||||
sk-fgsrnkabadokxmcbbmvbnfmrrcrcajjuorwsvquvbxluswbr
|
||||
sk-gcjubjefdwczuhmvtnwvigjtpiyzhcnaulqeudinexwewjrc
|
||||
sk-vxecdntxuawtfwtlikfhwxkfuxvckrljsmpdxcnczknuoivw
|
||||
sk-qhsfufnajkwqmahcpnjhqvaxrwakgdrkoqiyjjhzxhfqmvjw
|
||||
sk-xnivgfdglvrpysavwfosxufodfzpwmskegomzrerlrieroan
|
||||
sk-xciczaweyisbeiyqrjojzkyfqzlcrrekoooxvvlyuhialqwq
|
||||
sk-kmamcuorjnvtuokpupsbtotqzovnbquhxsnrllreignwifvx
|
||||
sk-sgeplkiycdvxjawkhhupbxyzkrqqdvrnmwwgubfjzwidnfac
|
||||
sk-ukkrovjjvaumkokogtndhnocjlcbhmxqieruhrhdrxjeqsrw
|
||||
sk-ewdedkictqtabasbsgkfjxnymxdgsmuqhgjnvvngrcmscrei
|
||||
sk-uuhamqmppytkbisrebsusyyedpoufillrjqhnbauvnyliyrm
|
||||
sk-ncxdshinjahwzogzarpqvaahimlftylkookeokbsmgatlhkp
|
||||
sk-kmkthtkyqxhicwpbvkavdsvixnhkhwnloqvdkbeegiaorfac
|
||||
sk-sgqkdykufwntcakjsulyjttsorpgevnbqemdidvnnokuxlur
|
||||
sk-aahqxkeidoebohilhrkbkbnhwhkkwjdvjulmyjumdxqpcmlt
|
||||
sk-cjgosmmljbtalvifxbxxrdtnngikrqxbnmmyczvwdhvaqrho
|
||||
sk-kjyfvkzfdlufstqwuolwmpiixvnygeyxcplkhtidcnkyuhof
|
||||
sk-ufmttqluqynwofdusyxyiliuclbqjavclepodqpietnxcpdz
|
||||
sk-dtfxfgxtvggrmcuahrbmgakotgabtdjfzfleybeyzzqgtkxe
|
||||
sk-risdpcbojturoktecteyfvuzsisznqvhpwkbmmjiipnebcnx
|
||||
sk-bnkaavqkyfwopsoandsecmehytikhvexiafnzvotiqvaadou
|
||||
sk-sydccxkatazajzoqazlhyhzifuifdfwhyobaoozemeaitcqr
|
||||
sk-mxkhmkttbiedcuvgnbuoruuuohmtvmtcdhxsytcksmtrambr
|
||||
sk-ngsttjqsxwrytzttxlcedemidchicwicmluuopbjnadggjzx
|
||||
sk-nulzzgtpaubyfhnuyaiktwhkznkzjxpppbymvncsfkrgmeif
|
||||
sk-ripjkrltyugoczhaeuxgkqmhtcznyawepvlvyjebezhorahn
|
||||
sk-hjciuefcqhoycgggyewlqmugsqafiefyowwfbzqprzupucpo
|
||||
sk-vtufrtvajondbtrkdhbuxlwvbqgszqazozxzqxotaydhbtvp
|
||||
sk-kfdlgrtqkyprcetrnrychpfevpjjgywwivxabzfqtasgyiix
|
||||
sk-feeqqafnomkhecmdmynthzwzrxtnycmqukruftwqskdkwtnf
|
||||
sk-npkvsjgcybufvdunwkphuaonyldjbxzjsafqqedahbtzeoiz
|
||||
@@ -321,144 +131,9 @@ sk-aglzgphsokubceekkeuzoyvgkcbqfycnxaqcgaedgjdvhcxg
|
||||
sk-hardkitrxfbychdakzysippwpyjmedmgiagqkgeqvmlafmjf
|
||||
sk-gsnxjocrbnaaxhzfeibrmdcbsnwfqbofhzndazjkrrvltxzv
|
||||
sk-ithxlhonxaorsmqdvckipubyntltddtgwptzckokngcpncfm
|
||||
sk-rydajrzjgdvygsnzsofzghsmpdeypytcupfzansxqjlxamhe
|
||||
sk-bwdbkrjdrrkvgwrlpcaawdumuvoltvfozuhcjkgtnsnrwlsk
|
||||
sk-pxsdjjeoihoycvqzeqcwxzttakxjjrzwkgbdphwxpwxofsbw
|
||||
sk-qrgxddgzmkgfdgjrxqsbxmtmpeyatwhzvxvmuvvbhfumpvuh
|
||||
sk-niqkplmnplnqxrbeeunicxarawnsoesfdwhubjicyecwnrek
|
||||
sk-kblxwcdacilfvwgpnyjuzyaxijvpxrzxgeeqzlkyvgznnhoa
|
||||
sk-hrcqaueghbzbhafmooboiwuaxqjmgkzhquswmyuxfougnako
|
||||
sk-sylauojlfotekasxmqpkuozbduvpbzjctyepkcbcgkipixse
|
||||
sk-irmklysclnupijhiijsnteyntgdurabgqeqnxghhibticnzb
|
||||
sk-uvofwcdjgoohkhuhevxpjncumzvtlkbllzlsumxhpssqsbsp
|
||||
sk-ltvriahhavqdfbckdcgzjivdicojvekeeoqwwooruokvvpug
|
||||
sk-zamhufoytbudycjydcwbsaktzgbbtpjnyejxeytgvmwcpwlu
|
||||
sk-fqapklszmsagbgpqrnznhvqnwzaoydnqvltjemypsqskotfy
|
||||
sk-pqcrbgbtolzodtramsexmsmfmrhbfjlbjtknizvcbkelbtmo
|
||||
sk-cpgsgbccgchuegzxuqqdmtcvleywvckexawpdwmqacwtkzls
|
||||
sk-hazswunedwvussdqsrgzwihixemitblhdniunrxuejcbrlko
|
||||
sk-piwkgmjimijvecwotaxdeuwavrxkplqijvfxmrhlaovmpgla
|
||||
sk-qemigcpicqqpaqaqvldghhpjdzaplpgvvqlzfwynyliyfrdk
|
||||
sk-mfcufhmoaegeklawzqklpimieknkgpywmilfofcxibjfhqtb
|
||||
sk-yvsaxmfkkeumfghphamfduxyjoskyqnchnsdmgrnharyfvcg
|
||||
sk-vwcauhdrefygiorzvgbgxppzwuechwujqhmalemscexebtvh
|
||||
sk-omajbnulidbutyqfbzrylzpcsotwxmyvkuklbikephfqcnaf
|
||||
sk-ivedlbwcdbeibugizbqgmlqlymjtejcoeoepdwrbxaleydad
|
||||
sk-yagbbopbkxikizbamlnysdsrdvachkzatmhtgremvnzepzrc
|
||||
sk-yislighgbsvmqocbevunpopcnapjxtnfzwhelycodkquyoxr
|
||||
sk-lufzzjathagzjdxzvywxozsgjahqbjptykbxdrvwrucqevtq
|
||||
sk-hphgvkzxifkekadopqzcaonwdaryfrgmtpwfqhbbdflbhsvj
|
||||
sk-itghsfwznrvvqmimujewgocmasllvebczhfvaepbdhrareol
|
||||
sk-mbdlewsjoqyvwfeugqvuwlkhoysdvhhkpyiphhrwxxhsxqja
|
||||
sk-rjnidikzzmvzlysihmwjzyzztgrfvqyohdplxbwpuvolctal
|
||||
sk-czxrlphpexsvxttfhgfpnmpdwnjdlecbuuyqikjnrlrocosj
|
||||
sk-eaaurjpunuzvolyxjosnjgudeqfikjjrlzdfcaeagniinmfz
|
||||
sk-srvewvzghpgjjjrssykgdhibtrfyqakyozmhwoypsbgkjmiv
|
||||
sk-dbqkfrbdwcnpphwbmlbatutqoknemwrcpkdahwtpgrybhdjb
|
||||
sk-mfwgnrhjlxqboxocbwgdemejjkrpizzompauozwbkynsimvd
|
||||
sk-vefkbebsxxiytlvgkqzzxfsdfdsxyoucfwyxyjvawpshjafs
|
||||
sk-rmdrphiuddwlyycyyxciuvvfypjgiqkaffxgnbblpjoramox
|
||||
sk-pvtelvkokqxsoqdmcmcybczobbyffqyiabfzlporvstrfbnl
|
||||
sk-wxjczvduvkezwamrmiqiqrdgsowjwmdrweihptstbydnkiyk
|
||||
sk-clnokwxbloklzhftymuoktkqcqwtbukteiswchfwygaemzxd
|
||||
sk-upvrfbkhjmeplwniscrsuhinvyhfjqdkxoxlwsnitjwiewzx
|
||||
sk-gmdwcjemnhtjefbtmpnljtsdepfgmrpdlgedamuxzcwewroa
|
||||
sk-kullfthpcemysaxmkrkktdosvxtzjkhhbfkpksvnqiviymkm
|
||||
sk-uantaculkjoueqmobedmjxnpufeppypxpqaazuzmloolyslb
|
||||
sk-uzqnmcllemnuwmioqcadooklwohsxuoijspmnzayuabdxgpo
|
||||
sk-velznojgjarqkmxuagajuizzxgshqvwysofujvnznwfkfmxo
|
||||
sk-pejyqvncscswbcjjmwbyekunlovppjselrkxwtwxlnrqoeph
|
||||
sk-kdoxtxvhjdwcqelwebbaoqscifaycxfytjojukqxqrbiiinx
|
||||
sk-mlrrttakzumknywzgxeyrrizfhqtlbbuaojiirsjsbdbvpqw
|
||||
sk-jidotcylhkuesnlmedoojupbcprqokvvjsbelkmpcrapeblo
|
||||
sk-bghjoljwoqowwaytxdygrmokjwbucpodwexfbqogfdobepjp
|
||||
sk-izfufzgvdvhffjwrgswfsnvwjehgswnuqndzpsdtqaukjqik
|
||||
sk-givwcjohnwowctfspxfzthnolplyqsncqpdndhnopceizncb
|
||||
sk-enooithbjoweqkloaoedhyuqmbehfthyehjqrudnijyxpwif
|
||||
sk-dxfksbhlibzzmvbjmhaohtmuhspgpnlfasauhmsybgzmvrqx
|
||||
sk-egsiqsepcweqgvcwexpcdrpvwrxmobvwixqccglcpesvmxed
|
||||
sk-spnqvuyeyqabuwilklpfhswgxlamzqlluwohhwcedrdcjoos
|
||||
sk-myzjstorsfjlwxmgskzkkyvubhhziukyavdfbkocvgljmlyn
|
||||
sk-gteiidrbqmgdxzvpjuiyeqyvipmxzqtoiambwldwgjsfmtdd
|
||||
sk-wiewtlkdlipdbjjrnsurhzmckxmlhmmqacjmpsvfwehlrzjc
|
||||
sk-crfrhbolnaesyidqqhugecipyxaiurydhhxpirnbtdpuqyje
|
||||
sk-xfwommmcrvvaicqxickvqzjpsslbgwltgkgcnjpywmbgkfux
|
||||
sk-vfceyfpfpefmdpwiwxnembwkxezfjvrulxiyckhunfszddcr
|
||||
sk-hragaaswdncjkxjamcsyetdzydvdpmvqfnlkncctrdcgqyts
|
||||
sk-rteqaliitwaplfjrjnfzissfopogiwpkxbllorgkllsflbyi
|
||||
sk-jrhvkbyrjinghhlrbvpuzlliujmwlytowvvbehyqzpgeqgpd
|
||||
sk-povvbewiqabsltipzbrsmogciuymrrcjdrsjajfonzugxlmb
|
||||
sk-rcgpvynyysinbnwreoqtnhzoyjajgkfpdkbfalfxycluzmyv
|
||||
sk-edsfyylfdstryliesizynqbsghoctythcxhtqycltnovdqyf
|
||||
sk-ruqwiacoyoaovpbgxgzjugpucdqweehmlxqfmzwubyhimuhk
|
||||
sk-xiluiivjbsuelibcbdjdkdcexhveukdonvwdanlkruifhksn
|
||||
sk-nusnonfvpndlozqewrbmaafzijdzzhwevbbeejqsmimktqga
|
||||
sk-zcjwylafnvwccbcsofvwfwnzwekivdsdcjkhxztezrebgfjg
|
||||
sk-nafijjlguxcresetwwkeblvhtzhzvdtlzdcmtgnznrsbaysp
|
||||
sk-gjzccgpgmtoxoyykdtytmiccyridjqavtmttbpkhuhxchcuz
|
||||
sk-aelhfuriagxjzdkhvraymqmzrskcgymmqzieszbbkhggxzmb
|
||||
sk-bwnaaxggxrcczsjahmpburjrogxvuslftpbiebukbdoojkav
|
||||
sk-hnhmwszlgyfsveokvjxxyqrmudwkjlynaqgroyvnfkvtfoen
|
||||
sk-gvhdltdldwlgtafcvkudjuaxccwquwprdmgdxoyhzxqfrnlt
|
||||
sk-vceekcyzeiehthcwrcsojxgjeicjlmaklwurfhsmindmaoze
|
||||
sk-gafvawwsdlidlcdstobbzwbkcmyzftlknimpfsksdobyhfdd
|
||||
sk-mlxznzvvpjyuzutevojnqhufvmviixpxxnoozgawqpmigipf
|
||||
sk-yowbggizjclintdzdvrupvukchktdscwptckwzqfjktyhsqu
|
||||
sk-itjkzhwbcijipsbfemcomdxrfjjlsawmafsokvqehftebeax
|
||||
sk-ftgafevejonwkzhsowshcajtzhcrzkgbygqeywpjdypwxcee
|
||||
sk-ydyesipjfeeslxychvdvhyhjnsjwrrxwzcyvxorgecgobdxt
|
||||
sk-ajlveigvqbcpqhhiegwoeufwzfmvhxgezvhoxbvzqfptexhk
|
||||
sk-vumrmneszkiykdqdmktemlyfawpwabodqbaqgwwhjwlemrus
|
||||
sk-cndsxoajozwuchkkjkestsykkkifnygaeqznktcaazzvyexk
|
||||
sk-swzvvwztfmbbwkobsrjozbncrkyozyjppysryskehrmslmqr
|
||||
sk-ofzoljaggdhqdehmllhspluecqlqwvpckynvgrcucnxgnetz
|
||||
sk-ilxpmzzdqvehuqengiflehjbjksjuzxxdkfblvbbotqnxcvg
|
||||
sk-vffajjwdonfgyobqlzsjvsmfgppmtozitlyuiklzddmwjasp
|
||||
sk-hlnlundabnwhlxnevvedrdgwgkixnwxxfcszkkmiyiogqtyh
|
||||
sk-zzinlifzyvwlbrcqolobsvpiuafofujmdempirjbblyiobrq
|
||||
sk-mhatttcuajcdubbhjszzuiugwahihvkhujqqdhuipypimppx
|
||||
sk-qcdnlpjjxsttqvfgrtdeaaemuxxiweodfpjaidaocxdjfbfi
|
||||
sk-jwhjcsjhpjqyynrlffbzchqynooegtwxoyweivpehomcjrpz
|
||||
sk-jwczimaoevkonzfczjjflpwvdckejejmxcpqztzwfzpvtgvp
|
||||
sk-jrrfbllfheymstqibwllzxmyvednewzhfemrpvskqvpxygob
|
||||
sk-istkpwkpczjbeoyeziajiefqvqhlxcececbskgyvrmomnmmk
|
||||
sk-zhpndzqwwngxmfqmyablhxfkpfzmriaiavjlqvjeggysgcvm
|
||||
sk-bizlrntmuhimynjyojsmyvsehgluwjqoaayvdrvtpeoiioul
|
||||
sk-nxoemksjhekrnbpcufgldxwtyzwnzlbqpuixleygnyrqzwfg
|
||||
sk-hydktomuqlcmgqtrojeifpifavsbczuqdtcejwfnjvocsaqw
|
||||
sk-ueimfegidcpvitpkbmioqqjrsdlzcckpdhiowjxruvcfoftr
|
||||
sk-isebnaohrsneefwpmaakltyekhsdbvuyoicyjrgmhhviyctp
|
||||
sk-lozddkkffmdttutlewxfsibvboszsvizcsvxxitpannvyhjp
|
||||
sk-dypsaykmiojssgflecaysuoffafksgjiiopkrsapvngtzcpq
|
||||
sk-vphqbyxqhmddhpgutalrismdloiclevinqettinmamivadrl
|
||||
sk-xueiltcvrqdczoehpylgppajnrxnfdwbdomddezgytyqgybc
|
||||
sk-kujnotimzgtjohdvihicyxrowoujlyydiyekssnzbeatoeam
|
||||
sk-rnmlxgqtvugiurgbmleubbywaldtdjpnmujlujuwruhnrscj
|
||||
sk-htudvczugpxxwumrixzmcsuuljpulyrycnvrbgyypgbhzlom
|
||||
sk-rmoauqmsvseihtjnmonzmhtjxqvehsoxosetgtzaeekfphsq
|
||||
sk-jwdjyicxrwarllnbxelmlpxyhlngxlwyblausbqzjybhfsgg
|
||||
sk-zjqaxewvbtmzicyudgwtouwpmaumqjxgfwvnixbebphzvkfw
|
||||
sk-btopemwluceyrvqrexdduzgiracedkipkjpvyqcekibvveld
|
||||
sk-mlmsyxbhulksdphymbunrtyvlfaulloafpdotspgujspvzhs
|
||||
sk-eoflcedwmujvkglnusptdvpejijmzqkgmzkugjfcowybyros
|
||||
sk-duudnurisfjbbiaijzvgjqxvmeqsaobyjonimqqubasffgtc
|
||||
sk-mrnqwyygtwfzdxwfxvdykciltrwremhhxkdwasvvyuyqlohe
|
||||
sk-nsojpxqdiiiwhudcbhieqcynwlqeqjcevdilpllnzexswdlf
|
||||
sk-fnaqnadhaxdioyvxarzessxcyonfusgjbvadmgofbbwlbfro
|
||||
sk-azihdytaexnxxhgklpeagvqrspjpuoirxkcpnmgwdpkhjocf
|
||||
sk-nsgabwkipznwkuhsipwcjmfrwzqrnsqsqbegmnsaderncbum
|
||||
sk-fnxklmkrxxchctgugherfgutqonyrhnbhmuywiwjmsgzripx
|
||||
sk-zdcppgntgaongebtgxumkjwlshnhmuwxbqeuysajxisonyze
|
||||
sk-iwixpdpuuovovlttpsxifhqbkgokexhaucwcoswtgrtkhsdj
|
||||
sk-jqlgopzfvpduihyqfxsbkbphfvyxtafhddyovshoumdhqtyx
|
||||
sk-gpvqdlyhppiezulgjeefurcszcgbkozkxzcismhzncjwiwws
|
||||
sk-owttiioxyexktqejevajanmjcyfozbyedqmxpgzpwapkuqit
|
||||
sk-gepnrwtefcnzbpbtsvsxwvmadrfcdcxxdcymhkrvjwkagjps
|
||||
sk-xybhogvmrvlkmandeeetmxllxprxbvumqchpyhrcsspejkge
|
||||
sk-wnljauedihizucllmpouarpkmdedwrhammttzbmsibeqwtlb
|
||||
sk-sincovmmvdslodfqdnwnlnzqowpdrmsjljnloqlobmqibkqm
|
||||
sk-zgfrqvtwapwbvmrbcmzfzesjzoiexmiuewqkbsxfrzvsarpr
|
||||
sk-veydusuesejnqgupmvnamwdoulqpquuzyihqysmxyzbgseom
|
||||
sk-mtpgzluddgftxbujteqpuviywfqgyncpfqjgjqgjpcicgyck
|
||||
sk-ftjkknqxfmkkkmpfdsgfqesjeeqmmmbhivdirifbhvfcepwt
|
||||
sk-ovckcrghcmczguqdrerkxkkkscyouizksvmpsjumgiewapfd
|
||||
@@ -571,59 +246,3 @@ sk-qcufjgduzuxahwuyuxhohdlxlwzspgjervcqavejjjsqnxuk
|
||||
sk-gcrooquaphfpgkpqzulmknyelauhntgpkahdndfmghwspfzl
|
||||
sk-ikpyfcknhbygvzzufzyltgproktmjlwrkcpaulzdbwitcnsd
|
||||
sk-ojglzjacacbqyxlepculoodgwawycatixfjygdghvwoedxqn
|
||||
sk-hzwilvkmjeonxyqkefvmyntslodwvzshbjtqcnyzyugamlwt
|
||||
sk-ykqumucxvezvtklxrzcdxtrcjxezdvorccjxeetpvwjxasig
|
||||
sk-ymvixkeoxxmslmvyjtttaszsqurhggtrljmnjpypjpvlxgie
|
||||
sk-gdxyanoxmzuzoqwrjuzqltwcfrkcfjffjysnobzwsxknzbas
|
||||
sk-uksvsffcuwuqycpcgsxoagytwisvyizvqissvzxfbwepmoxt
|
||||
sk-ecjdubyxsxvorouvsluugguckqyzagjoubzlpfgitdlpxnjr
|
||||
sk-vznvfvcccqzpazawlhqenjcyyhcgjmewwqlcumpoouftbdkd
|
||||
sk-aocgyriobhfzmbdsjeycjovlabdyoobuzvsicoqvqgjufeyq
|
||||
sk-fftwoemnmkaigzvtsdvcovxnsrqaivpghbqlhhzwfupgqixd
|
||||
sk-pdimahhuaciuytjcdrrqepnvekcwcajjaccfnkbtywksgrzj
|
||||
sk-gpubyipyxilqncwsbjtifyvihytttndwnnzwfpacezogorkt
|
||||
sk-vyohlulojrccftrlttbgkhtssrsevjwvbyprkxxozztcagfl
|
||||
sk-evkreookljfrkivshdtukmvdmigfsrkbxqlvpplmmxrqdeeq
|
||||
sk-mfhqlomnnimfyalspfgrifvljcyqibqecaogvcmtpfdinvww
|
||||
sk-mgrgagopmvspegvtlxcczewuyacnclzkmidfhjjrlftigygb
|
||||
sk-taokfbrmnwpuzfvwmjkqkxmbtcsrhiosbvxhlzjvtkfyyzew
|
||||
sk-arllcykeoxnaayvrnlwdgefptzbwdrrpfswzuellaxfpwiyv
|
||||
sk-jswurhdubigubwaywtmjwesgdmeublcfgyxkqhjfqolhjkxw
|
||||
sk-owufbwamtvnyfcmnokoletwfiqcolbisusgjxaqsucjddgaj
|
||||
sk-oavuxluuhgarupuonjdajtiyqzfaappsnepvrwsaecoqytps
|
||||
sk-abjzukurnfvycmobsrlupngiuoihydxfolguwdkiuousjrog
|
||||
sk-areauhdxiwjqvtvkjzghvhzlkctawgxaqykbzrtcpzkeemzh
|
||||
sk-hvirdspguzuxeeyqtzozteuqpnlgdmvkhdbcseeiwvvvjwpc
|
||||
sk-bxgtnigshpgbzrxnpirnknmuwrvmmrdsdxgrsxqxhklkbiag
|
||||
sk-qsrvvcyyvetacumkffpifzvpprvueiuyxcymjhuoywulyckx
|
||||
sk-chtrfzhvlwvkcxhmlwgxylhhbzoghmlhfqnrkrjjbxougtlm
|
||||
sk-kaorsolyqbqktpjyxmvsothfkpklauhwvjoewdnxvjcdtmiu
|
||||
sk-yyhihitvelctqipvcygnqgzadifvssmowwauroyydvzxqwxz
|
||||
sk-zuwnyslmzstjwcimfdfdhxhqxdyoyaspfzlqieqklptyrqod
|
||||
sk-mrdqhmnwzubfiegupgjcdcrxwscwyrkqburcovprpcsyuikd
|
||||
sk-wvggprumleznebcbaycjpmpxvnxrhkcibfohwavxxdjtbkah
|
||||
sk-bzknzzwnftgzgfhddwtbrethrykvdcprldmzkimgwvactluv
|
||||
sk-kayozpvefrtnyjgzlxvfupxcvpygqpvrwzfjofexkoohcyfj
|
||||
sk-telmszepewpsgvtgwmtjuhevatqbigjazjpcqowtxctjcfqt
|
||||
sk-jzicpgrljkzzvtpnonkdtlesztuzxdblflpmvazghnawbrbp
|
||||
sk-idsyjybbhuwtkznjqwehfkirotujqlecqmxvouthbgblkopq
|
||||
sk-guriavtreoqrodvtqtoxxfgibrpvucnvifbviizkmjgfnojs
|
||||
sk-oeakuguoobfgyiajtmnuucnhvdplmwmleoskybmwaynxxjjc
|
||||
sk-jlrvbzmamxafavnaqgslolvqqaroamzsczezzckacxfufuut
|
||||
sk-belofvolykizlwcheykdwlrvxtmkzrvxmwunqfepstbzsano
|
||||
sk-ngkqukytxzjmgxctwxzahtkdrtiokkrrhjzjptcmedvygijw
|
||||
sk-lbsbwiwdfxfdomoorrepdswizrxicjohitaxgnwmckpidgqp
|
||||
sk-slugbeutaykvjvzbdeuxzjtxdmhtbgqlmpxyxlagomcrmfex
|
||||
sk-zzstwfsdicghbutiruvfqqmssicedkwpynkftlpdnbknkeac
|
||||
sk-wxcefyxwviuxsdrwvbojvahchrhyzvplbqvckhwigneqweal
|
||||
sk-petosxdqphybeaexrmavbenovqshbgyqzigvbdpsmlgwamrf
|
||||
sk-akzqrlcbkvyoycpsypipswwxdkyyyojpdqaaauomugpeqmme
|
||||
sk-cggdeyuwhrmmwyvrxyxgjqkgflpfzxuscizyorpntqyfeekm
|
||||
sk-akjolruzfygkrozyqsxzmwbxspbihdxqffbxrvlxmmnugxid
|
||||
sk-tfhblzxfibbjkroxcyfjmvjtttnkrvcrqdhvgqafmbiwfrmp
|
||||
sk-kyqkmusnnecseiddehvpaicymdppwtnztwsllsjgbeubtngx
|
||||
sk-itzdkmmqvrkqsczfellwdsmqusvncdnhfupnyplcexzgjqys
|
||||
sk-smlwnqivoryuqbgduvcawnlznwakpparjlcjpbbartgljtip
|
||||
sk-rrrdvcmvyqmkfwavimkdzsrycshwwikxklfcbsqsjthbduod
|
||||
sk-jwiuszauztewftwmcxtfjmzonvjujvemulekyxtdkjwdgmah
|
||||
sk-cwjrcfssqcakkpwypldjocdvxhrjdbtmsstuuviixcsfmkoy
|
||||
+61
-25631
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@@ -1,187 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
File: deduplicate_json.py
|
||||
Description: 对指定JSON文件进行去重并重新保存
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from tqdm import tqdm
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dotenv import load_dotenv
|
||||
from rag2_0.tool.ModelTool import OpenAiLLM
|
||||
from langchain.output_parsers import PydanticOutputParser
|
||||
from pydantic import BaseModel, Field
|
||||
from rag2_0.intent_recognition.DataModels import Term
|
||||
|
||||
# 加载环境变量
|
||||
load_dotenv()
|
||||
|
||||
|
||||
class JsonDeduplicator:
|
||||
"""JSON文件去重类"""
|
||||
|
||||
def __init__(self, input_path=None, output_path=None, key_field="name", max_workers=3):
|
||||
"""初始化JSON去重器
|
||||
|
||||
Args:
|
||||
input_path: 输入JSON文件路径
|
||||
output_path: 去重后的输出文件路径
|
||||
key_field: 用于去重的键字段名
|
||||
max_workers: 线程池最大工作线程数
|
||||
"""
|
||||
self.INPUT_PATH = input_path
|
||||
self.OUTPUT_PATH = output_path or input_path.replace('.json', '_deduplicated.json')
|
||||
self.KEY_FIELD = key_field
|
||||
self.MAX_WORKERS = max_workers
|
||||
self.item_parser = PydanticOutputParser(pydantic_object=Term)
|
||||
self.MERGE_PROMPT = '''
|
||||
请将以下多个描述相同名词"{name}"的条目合并为一个，合并时请：
|
||||
- 同义词（synonymous）去重合并
|
||||
- 描述（description）合并为更完整、简明的描述
|
||||
- 保持输出格式为：
|
||||
{output_format}
|
||||
原始条目：
|
||||
{items}
|
||||
'''
|
||||
# 配置LLM
|
||||
model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
|
||||
api_key = os.getenv("OPENAI_API_KEY")
|
||||
base_url = os.getenv("OPENAI_API_BASE")
|
||||
llm_params = {"temperature": 0.3, "model": model_name}
|
||||
if api_key:
|
||||
llm_params["api_key"] = api_key
|
||||
if base_url:
|
||||
llm_params["base_url"] = base_url
|
||||
self.llm = OpenAiLLM(**llm_params)
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
def load_json_data(self):
|
||||
"""读取JSON文件"""
|
||||
try:
|
||||
with open(self.INPUT_PATH, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
logging.info(f"从{self.INPUT_PATH}加载了{len(data)}条记录")
|
||||
return data
|
||||
except Exception as e:
|
||||
logging.error(f"读取{self.INPUT_PATH}失败: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
def group_items_by_key(self, items):
|
||||
"""按指定键字段聚合项目"""
|
||||
key_to_items = defaultdict(list)
|
||||
for item in items:
|
||||
key = item.get(self.KEY_FIELD, '').strip()
|
||||
if key:
|
||||
key_to_items[key].append(item)
|
||||
return key_to_items
|
||||
|
||||
def merge_items_with_llm(self, key, item_list):
|
||||
"""调用LLM合并具有相同键的项目，失败最多重试三次"""
|
||||
items = json.dumps(item_list, ensure_ascii=False)
|
||||
prompt = self.MERGE_PROMPT.format(
|
||||
name=key,
|
||||
items=items,
|
||||
output_format=self.item_parser.get_format_instructions()
|
||||
)
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
response = self.llm.invoke(prompt, False)
|
||||
parsed_output = self.item_parser.parse(response.content)
|
||||
return {"name": parsed_output.name, "synonymous": parsed_output.synonymous, "description": parsed_output.description}
|
||||
except Exception as e:
|
||||
if attempt == max_retries:
|
||||
logging.warning(f"解析LLM合并结果失败: {e}")
|
||||
return None
|
||||
else:
|
||||
import time
|
||||
time.sleep(5*attempt)
|
||||
|
||||
def process_item(self, key_items_tuple):
|
||||
"""处理单个键值对应的项目，用于线程池并行处理"""
|
||||
key, item_list = key_items_tuple
|
||||
try:
|
||||
if len(item_list) == 1:
|
||||
return item_list[0]
|
||||
|
||||
merged = self.merge_items_with_llm(key, item_list)
|
||||
if merged:
|
||||
return merged
|
||||
else:
|
||||
# 如果合并失败，返回第一个项目
|
||||
return item_list[0]
|
||||
except Exception as e:
|
||||
logging.error(f"处理键 {key} 时出错: {e}", exc_info=True)
|
||||
return item_list[0]
|
||||
|
||||
def deduplicate(self):
|
||||
"""去重所有项目的入口方法"""
|
||||
# 1. 读取JSON数据
|
||||
all_items = self.load_json_data()
|
||||
if not all_items:
|
||||
return []
|
||||
|
||||
# 2. 按键字段聚合
|
||||
key_to_items = self.group_items_by_key(all_items)
|
||||
logging.info(f"共{len(key_to_items)}个唯一键")
|
||||
|
||||
# 3. 使用线程池并行处理
|
||||
deduplicated_items = []
|
||||
items_to_process = []
|
||||
|
||||
# 先处理只有一个项目的键（不需要合并）
|
||||
for key, item_list in key_to_items.items():
|
||||
if len(item_list) == 1:
|
||||
deduplicated_items.append(item_list[0])
|
||||
else:
|
||||
items_to_process.append((key, item_list))
|
||||
|
||||
logging.info(f"共{len(deduplicated_items)}个单一项目，{len(items_to_process)}个需要合并的项目")
|
||||
|
||||
# 只对需要合并的项目使用线程池处理
|
||||
if items_to_process:
|
||||
with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
|
||||
# 使用tqdm显示进度
|
||||
for result in tqdm(executor.map(self.process_item, items_to_process), total=len(items_to_process)):
|
||||
deduplicated_items.append(result)
|
||||
|
||||
# 4. 保存去重结果
|
||||
os.makedirs(os.path.dirname(self.OUTPUT_PATH), exist_ok=True)
|
||||
with open(self.OUTPUT_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(deduplicated_items, f, ensure_ascii=False, indent=2)
|
||||
logging.info(f"去重后结果已保存到: {self.OUTPUT_PATH}")
|
||||
|
||||
return deduplicated_items
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数，解析命令行参数并执行去重"""
|
||||
parser = argparse.ArgumentParser(description='对JSON文件进行去重')
|
||||
input_path = 'data/wiki_extracted_nouns/技改检修计价通_nouns.json'
|
||||
|
||||
parser.add_argument('-i', '--input',default=input_path, help='输入JSON文件路径')
|
||||
parser.add_argument('-o', '--output', help='输出JSON文件路径')
|
||||
parser.add_argument('-k', '--key', default='name', help='用于去重的键字段名，默认为"name"')
|
||||
parser.add_argument('-w', '--workers', type=int, default=30, help='线程池最大工作线程数，默认为2')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
deduplicator = JsonDeduplicator(
|
||||
input_path=args.input,
|
||||
output_path=args.output,
|
||||
key_field=args.key,
|
||||
max_workers=args.workers
|
||||
)
|
||||
deduplicator.deduplicate()
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.getLogger('httpx').setLevel(logging.WARNING)
|
||||
logging.getLogger('openai').setLevel(logging.WARNING)
|
||||
main()
|
||||
@@ -15,6 +15,8 @@ import traceback
|
||||
import re
|
||||
import logging
|
||||
from tqdm import tqdm
|
||||
import glob
|
||||
import shutil
|
||||
|
||||
# 将项目根目录添加到Python路径
|
||||
sys.path.append(os.getcwd())
|
||||
@@ -84,13 +86,6 @@ class IsComplaint(BaseModel):
|
||||
dissatisfaction_reasoning: str = Field(description="抱怨原因")
|
||||
is_complaint: bool = Field(description="是否明确/暗示将进行投诉")
|
||||
|
||||
class ProductNameAndModuleName(BaseModel):
|
||||
product_name: str = Field(description="产品名称")
|
||||
module_name: str = Field(description="模块名称")
|
||||
|
||||
class ProductLine(BaseModel):
|
||||
product_line: str = Field(description="产品线")
|
||||
|
||||
# ================ 工具函数 ================
|
||||
def retry_llm_call(max_retries=3, delay=2):
|
||||
"""
|
||||
@@ -138,8 +133,6 @@ class DialogueToWorkorder:
|
||||
self.user_question_and_solution_list_parser = PydanticOutputParser(pydantic_object=UserQuestionAndSolutionList)
|
||||
self.question_type_parser = PydanticOutputParser(pydantic_object=QuestionType)
|
||||
self.is_complaint_parser = PydanticOutputParser(pydantic_object=IsComplaint)
|
||||
self.product_name_and_module_name_parser = PydanticOutputParser(pydantic_object=ProductNameAndModuleName)
|
||||
self.product_line_parser = PydanticOutputParser(pydantic_object=ProductLine)
|
||||
# 初始化LLM模型
|
||||
self.llm_params = llm_params or {
|
||||
"temperature": 0.2,
|
||||
@@ -159,6 +152,10 @@ class DialogueToWorkorder:
|
||||
# }
|
||||
self.llm = self._get_llm_instance()
|
||||
|
||||
# 创建工单JSON文件目录
|
||||
self.workorder_json_dir = "data/temp_workorder_json"
|
||||
os.makedirs(self.workorder_json_dir, exist_ok=True)
|
||||
|
||||
def _get_llm_instance(self):
|
||||
"""获取LLM实例"""
|
||||
return OpenAiLLM(**self.llm_params)
|
||||
@@ -483,6 +480,66 @@ class DialogueToWorkorder:
|
||||
is_complaint.dissatisfaction_reasoning,
|
||||
is_complaint.is_complaint)
|
||||
|
||||
def save_conversation_to_json(self, conversation_id, workorder_list):
|
||||
"""
|
||||
将会话处理结果保存为JSON文件
|
||||
|
||||
参数:
|
||||
conversation_id: 会话ID
|
||||
workorder_list: 工单列表
|
||||
"""
|
||||
# 确保目录存在
|
||||
os.makedirs(self.workorder_json_dir, exist_ok=True)
|
||||
|
||||
# 构建文件路径
|
||||
file_path = os.path.join(self.workorder_json_dir, f"{conversation_id}.json")
|
||||
|
||||
# 将工单列表转换为可序列化的字典列表
|
||||
serializable_workorder_list = []
|
||||
for workorder in workorder_list:
|
||||
# 处理datetime对象
|
||||
serializable_workorder = {}
|
||||
for key, value in workorder.items():
|
||||
if isinstance(value, datetime):
|
||||
serializable_workorder[key] = value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
else:
|
||||
serializable_workorder[key] = value
|
||||
serializable_workorder_list.append(serializable_workorder)
|
||||
|
||||
# 保存为JSON文件
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(serializable_workorder_list, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"会话ID: {conversation_id} 的处理结果已保存到 {file_path}")
|
||||
|
||||
def load_conversation_from_json(self, conversation_id):
|
||||
"""
|
||||
从JSON文件加载会话处理结果
|
||||
|
||||
参数:
|
||||
conversation_id: 会话ID
|
||||
|
||||
返回:
|
||||
工单列表，如果文件不存在则返回None
|
||||
"""
|
||||
# 构建文件路径
|
||||
file_path = os.path.join(self.workorder_json_dir, f"{conversation_id}.json")
|
||||
|
||||
# 检查文件是否存在
|
||||
if not os.path.exists(file_path):
|
||||
return None
|
||||
|
||||
# 从JSON文件加载工单列表
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
workorder_list = json.load(f)
|
||||
|
||||
logger.info(f"已从 {file_path} 加载会话ID: {conversation_id} 的处理结果")
|
||||
return workorder_list
|
||||
except Exception as e:
|
||||
logger.error(f"加载会话ID: {conversation_id} 的处理结果时发生错误: {e}")
|
||||
return None
|
||||
|
||||
def process_conversation(self, conversation_id, conversation_rows):
|
||||
"""处理单个会话的函数，用于多线程并发"""
|
||||
# if conversation_id!="b157aa91-3acb-11f0-a191-4fb224ef4b40":
|
||||
@@ -534,13 +591,16 @@ class DialogueToWorkorder:
|
||||
# 将工单添加到列表中
|
||||
workorder_list.append(workorder_dict)
|
||||
|
||||
# 将处理结果保存为JSON文件
|
||||
self.save_conversation_to_json(conversation_id, workorder_list)
|
||||
|
||||
return workorder_list
|
||||
except Exception as e:
|
||||
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {e}")
|
||||
return []
|
||||
|
||||
def analyze_conversation_data(self, conversation_excel_path, max_workers=10, start_date=None, end_date=None):
|
||||
"""分析会话数据主流程，使用多线程并发处理"""
|
||||
"""分析会话数据主流程，使用多线程并发处理，支持失败重试和JSON合并"""
|
||||
# 读取Excel文件
|
||||
df = pd.read_excel(conversation_excel_path)
|
||||
|
||||
@@ -576,36 +636,122 @@ class DialogueToWorkorder:
|
||||
|
||||
logger.info(f"会话总数为 {len(conversation_dict)}，处理全部会话")
|
||||
|
||||
# 使用线程池处理每个会话
|
||||
workorder_dict_list = []
|
||||
# ========== 新增：扫描已存在的JSON文件 ==========
|
||||
existing_json_files = set()
|
||||
workorder_json_dir = self.workorder_json_dir
|
||||
if not os.path.exists(workorder_json_dir):
|
||||
os.makedirs(workorder_json_dir, exist_ok=True)
|
||||
for fname in os.listdir(workorder_json_dir):
|
||||
if fname.endswith('.json'):
|
||||
conversation_id = fname[:-5]
|
||||
existing_json_files.add(conversation_id)
|
||||
|
||||
# 本次新生成的JSON文件
|
||||
newly_generated_json_files = set()
|
||||
# 本次未重新生成但已存在的JSON文件
|
||||
reused_json_files = set()
|
||||
|
||||
# ========== 线程池处理会话 ==========
|
||||
successful_conversations = set()
|
||||
failed_conversations = set()
|
||||
import threading
|
||||
lock = threading.Lock()
|
||||
|
||||
def process_wrapper(conversation_id, conversation_rows):
|
||||
json_file_path = os.path.join(workorder_json_dir, f"{conversation_id}.json")
|
||||
if conversation_id in existing_json_files and os.path.exists(json_file_path):
|
||||
# 已存在，直接复用
|
||||
with lock:
|
||||
reused_json_files.add(conversation_id)
|
||||
return None # 不处理
|
||||
# 否则正常处理
|
||||
result = self.process_conversation(conversation_id, conversation_rows)
|
||||
if result:
|
||||
with lock:
|
||||
newly_generated_json_files.add(conversation_id)
|
||||
return result
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
# 创建任务
|
||||
future_to_conversation = {
|
||||
executor.submit(self.process_conversation, conversation_id, conversation_rows): conversation_id
|
||||
executor.submit(process_wrapper, conversation_id, conversation_rows): conversation_id
|
||||
for conversation_id, conversation_rows in conversation_dict.items()
|
||||
}
|
||||
|
||||
# 获取结果
|
||||
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="处理会话进度"):
|
||||
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="第一轮处理会话进度"):
|
||||
conversation_id = future_to_conversation[future]
|
||||
try:
|
||||
result_workorders = future.result()
|
||||
if result_workorders:
|
||||
# 将每个会话的所有工单添加到总列表中
|
||||
workorder_dict_list.extend(result_workorders)
|
||||
successful_conversations.add(conversation_id)
|
||||
logger.info(f"完成处理会话ID: {conversation_id}，生成工单数量: {len(result_workorders)}")
|
||||
elif conversation_id in reused_json_files:
|
||||
successful_conversations.add(conversation_id)
|
||||
logger.info(f"跳过已存在JSON，会话ID: {conversation_id}")
|
||||
else:
|
||||
failed_conversations.add(conversation_id)
|
||||
logger.warning(f"会话ID: {conversation_id} 处理可能失败，将在第二轮重试")
|
||||
except Exception as exc:
|
||||
failed_conversations.add(conversation_id)
|
||||
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {exc}")
|
||||
|
||||
# 检查哪些会话没有成功生成JSON文件
|
||||
all_conversation_ids = set(conversation_dict.keys())
|
||||
for conversation_id in all_conversation_ids:
|
||||
json_file_path = os.path.join(workorder_json_dir, f"{conversation_id}.json")
|
||||
if not os.path.exists(json_file_path):
|
||||
failed_conversations.add(conversation_id)
|
||||
if conversation_id in successful_conversations:
|
||||
successful_conversations.remove(conversation_id)
|
||||
|
||||
# ========== 第二轮重试 ==========
|
||||
if failed_conversations:
|
||||
logger.info(f"第一轮处理后有 {len(failed_conversations)} 个会话需要重试")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, max_workers // 2)) as executor:
|
||||
future_to_conversation = {
|
||||
executor.submit(process_wrapper, conversation_id, conversation_dict[conversation_id]): conversation_id
|
||||
for conversation_id in failed_conversations
|
||||
}
|
||||
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="第二轮重试处理进度"):
|
||||
conversation_id = future_to_conversation[future]
|
||||
try:
|
||||
result_workorders = future.result()
|
||||
if result_workorders:
|
||||
successful_conversations.add(conversation_id)
|
||||
newly_generated_json_files.add(conversation_id)
|
||||
failed_conversations.remove(conversation_id)
|
||||
logger.info(f"重试成功: 会话ID: {conversation_id}，生成工单数量: {len(result_workorders)}")
|
||||
elif conversation_id in reused_json_files:
|
||||
successful_conversations.add(conversation_id)
|
||||
failed_conversations.remove(conversation_id)
|
||||
logger.info(f"重试跳过已存在JSON，会话ID: {conversation_id}")
|
||||
except Exception as exc:
|
||||
logger.error(f"重试处理会话ID: {conversation_id} 时仍然发生错误: {exc}")
|
||||
|
||||
# ========== 合并本次所有成功的JSON文件 ==========
|
||||
logger.info(f"开始合并JSON文件结果，成功处理会话数: {len(successful_conversations)}，失败会话数: {len(failed_conversations)}")
|
||||
workorder_dict_list = []
|
||||
# 只合并本次新生成和本次未重新生成但已存在的JSON
|
||||
all_json_ids_to_merge = newly_generated_json_files.union(reused_json_files)
|
||||
json_files = [os.path.join(workorder_json_dir, f"{cid}.json") for cid in all_json_ids_to_merge if os.path.exists(os.path.join(workorder_json_dir, f"{cid}.json"))]
|
||||
for json_file in tqdm(json_files, desc="合并JSON文件"):
|
||||
conversation_id = os.path.basename(json_file).replace(".json", "")
|
||||
try:
|
||||
with open(json_file, 'r', encoding='utf-8') as f:
|
||||
workorders = json.load(f)
|
||||
workorder_dict_list.extend(workorders)
|
||||
except Exception as e:
|
||||
logger.error(f"加载JSON文件 {json_file} 时发生错误: {e}")
|
||||
logger.info(f"处理完成，成功处理会话数: {len(successful_conversations)}，失败会话数: {len(failed_conversations)}")
|
||||
if failed_conversations:
|
||||
logger.warning(f"以下会话处理失败: {failed_conversations}")
|
||||
return workorder_dict_list
|
||||
|
||||
def save_results_to_excel(self, workorder_dict_list, output_file=None):
|
||||
"""将结果保存到Excel文件"""
|
||||
"""将结果保存到Excel文件，并清理JSON文件"""
|
||||
result_df = pd.DataFrame(workorder_dict_list)
|
||||
|
||||
# 按照指定的列顺序重新排列DataFrame的列
|
||||
columns_order = [
|
||||
'工单编号', '产品线', '产品名称', '模块名称', '问题类型',
|
||||
'工单编号', '产品线', '产品名称', '问题类型',
|
||||
'客户问题', '解决方案', '是否抱怨', "抱怨内容", '是否投诉', '抱怨级别',
|
||||
'会话id', '访客昵称', '处理坐席', "处理人", "处理技能组",'创建时间'
|
||||
]
|
||||
@@ -645,7 +791,6 @@ class DialogueToWorkorder:
|
||||
'工单编号': 15,
|
||||
'产品线': 24,
|
||||
'产品名称': 40,
|
||||
'模块名称': 40,
|
||||
'问题类型': 9,
|
||||
'客户问题': 20,
|
||||
'解决方案': 30,
|
||||
@@ -669,7 +814,6 @@ class DialogueToWorkorder:
|
||||
worksheet.column_dimensions[col_letter].width = column_widths[column]
|
||||
|
||||
logger.info(f"结果已保存到 {output_file}")
|
||||
|
||||
return output_file
|
||||
|
||||
# ================ 参数解析 ================
|
||||
|
||||
@@ -23,7 +23,7 @@ from tqdm import tqdm
|
||||
import concurrent.futures
|
||||
import sys
|
||||
|
||||
os.makedirs('./data/log', exist_ok=True)
|
||||
os.makedirs('./data/logs', exist_ok=True)
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
|
||||
@@ -214,20 +214,7 @@ class QueryRewriteProcessor:
|
||||
|
||||
# 根据enable_retrieval参数决定是否进行文档检索
|
||||
retrieved_doc = None
|
||||
if enable_retrieval:
|
||||
retrieved_doc = self.dify_query_retrieval.retrieve(original_query, query_list, classification_obj, current_softname)
|
||||
|
||||
# 判断检索文档是否相关
|
||||
relevance_result = {}
|
||||
if retrieved_doc:
|
||||
# 判断文档相关性
|
||||
relevance_result = self.is_retrieved_doc_relevant(query, retrieved_doc)
|
||||
else:
|
||||
relevance_result = {
|
||||
"is_relevant": False,
|
||||
"explanation": "没有检索到文档" if enable_retrieval else "文档检索功能未启用",
|
||||
"relevance_score": 0.0
|
||||
}
|
||||
|
||||
retrieved_doc_titles=[]
|
||||
if retrieved_doc:
|
||||
@@ -251,8 +238,6 @@ class QueryRewriteProcessor:
|
||||
"槽位信息": slot_filling_str,
|
||||
"检索的文档": "\n".join(retrieved_doc_titles),
|
||||
"检索的内容": json.dumps(retrieved_doc, ensure_ascii=False, indent=2) if retrieved_doc else "",
|
||||
"文档能否解决问题": "能" if relevance_result["is_relevant"] else "不能",
|
||||
"文档相关性解释": relevance_result["explanation"]
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"处理问题 '{query}' 时出错: ",exc_info=True)
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
File: merge_nouns_with_llm.py
|
||||
Description: 合并多个nouns.json中的同名专业名词，利用LLM生成唯一合并结果
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import glob
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections import defaultdict
|
||||
from dotenv import load_dotenv
|
||||
from rag2_0.tool.ModelTool import OpenAiLLM
|
||||
from rag2_0.intent_recognition.DataModels import Term
|
||||
import logging
|
||||
from langchain.output_parsers import PydanticOutputParser
|
||||
from tqdm import tqdm
|
||||
import time
|
||||
# 加载环境变量
|
||||
load_dotenv()
|
||||
|
||||
class TermMerger:
|
||||
"""专业名词合并类，用于合并多个数据源中的同名专业名词"""
|
||||
|
||||
def __init__(self, input_dir=None, output_path=None, max_workers=3):
|
||||
"""初始化名词合并器
|
||||
|
||||
Args:
|
||||
input_dir: 包含nouns.json文件的目录路径
|
||||
output_path: 合并结果的输出文件路径
|
||||
max_workers: 线程池最大工作线程数
|
||||
"""
|
||||
self.EXTRACTED_NOUNS_DIR = input_dir
|
||||
self.OUTPUT_PATH = output_path
|
||||
self.MAX_WORKERS = max_workers
|
||||
self.terms_parser = PydanticOutputParser(pydantic_object=Term)
|
||||
self.MERGE_PROMPT = '''
|
||||
请将以下多个描述相同名词"{name}"的条目合并为一个，合并时请：
|
||||
- 同义词（synonymous）去重合并
|
||||
- 描述（description）合并为更完整、简明的描述
|
||||
- 保持输出格式为：
|
||||
{output_format}
|
||||
原始条目：
|
||||
{items}
|
||||
'''
|
||||
# 配置LLM
|
||||
model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
|
||||
api_key = os.getenv("OPENAI_API_KEY")
|
||||
base_url = os.getenv("OPENAI_API_BASE")
|
||||
llm_params = {"temperature": 0.3, "model": model_name}
|
||||
if api_key:
|
||||
llm_params["api_key"] = api_key
|
||||
if base_url:
|
||||
llm_params["base_url"] = base_url
|
||||
self.llm = OpenAiLLM(**llm_params)
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
def load_all_terms(self):
|
||||
"""读取目录下所有nouns.json，返回所有Term列表"""
|
||||
all_terms = []
|
||||
for file in glob.glob(os.path.join(self.EXTRACTED_NOUNS_DIR, '*_nouns.json')):
|
||||
with open(file, 'r', encoding='utf-8') as f:
|
||||
try:
|
||||
file_terms = json.load(f)
|
||||
new_terms = [{"name": term["name"].upper(), "synonymous": term["synonymous"], "description": term["description"]} for term in file_terms]
|
||||
all_terms.extend(new_terms)
|
||||
logging.info(f"加载{file}，共{len(new_terms)}条")
|
||||
except Exception as e:
|
||||
logging.warning(f"读取{file}失败: {e}")
|
||||
|
||||
# 加载suffix_keywords.json文件
|
||||
# suffix_keywords_path = os.path.join(os.path.dirname(os.path.dirname(self.EXTRACTED_NOUNS_DIR)), 'data', 'nouns', 'suffix_keywords.json')
|
||||
# if os.path.exists(suffix_keywords_path):
|
||||
# try:
|
||||
# with open(suffix_keywords_path, 'r', encoding='utf-8') as f:
|
||||
# suffix_terms = json.load(f)
|
||||
# suffix_terms = [{"name": term["name"].upper(), "synonymous": "", "description": ""} for term in suffix_terms]
|
||||
# all_terms.extend(suffix_terms)
|
||||
# logging.info(f"加载{suffix_keywords_path}，共{len(suffix_terms)}条")
|
||||
# except Exception as e:
|
||||
# logging.warning(f"读取{suffix_keywords_path}失败: {e}")
|
||||
|
||||
return all_terms
|
||||
|
||||
def group_terms_by_name(self, terms):
|
||||
"""按name聚合Term"""
|
||||
name2terms = defaultdict(list)
|
||||
for term in terms:
|
||||
name = term.get('name', '').strip()
|
||||
if name:
|
||||
name2terms[name].append(term)
|
||||
return name2terms
|
||||
|
||||
def merge_terms_with_llm(self, name, term_list):
|
||||
"""调用LLM合并同名Term，失败最多重试三次"""
|
||||
items = json.dumps(term_list, ensure_ascii=False)
|
||||
prompt = self.MERGE_PROMPT.format(name=name, items=items, output_format=self.terms_parser.get_format_instructions())
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
response = self.llm.invoke(prompt, False)
|
||||
parsed_output = self.terms_parser.parse(response.content)
|
||||
return {"name": parsed_output.name, "synonymous": parsed_output.synonymous, "description": parsed_output.description}
|
||||
except Exception as e:
|
||||
if attempt == max_retries:
|
||||
logging.warning(f"解析LLM合并结果失败: {e}")
|
||||
return None
|
||||
else:
|
||||
time.sleep(10*attempt)
|
||||
|
||||
def process_term(self, name_terms_tuple):
|
||||
"""处理单个词条，用于线程池并行处理"""
|
||||
name, term_list = name_terms_tuple
|
||||
try:
|
||||
merged = self.merge_terms_with_llm(name, term_list)
|
||||
if merged:
|
||||
return merged
|
||||
else:
|
||||
return term_list[0]
|
||||
except Exception as e:
|
||||
logging.error(f"处理词条 {name} 时出错: {e}", exc_info=True)
|
||||
return term_list[0]
|
||||
|
||||
def merge(self):
|
||||
"""合并所有词条的入口方法"""
|
||||
# 1. 读取所有术语
|
||||
all_terms = self.load_all_terms()
|
||||
logging.info(f"共加载{len(all_terms)}条术语")
|
||||
|
||||
# 2. 按名称聚合
|
||||
name2terms = self.group_terms_by_name(all_terms)
|
||||
logging.info(f"共{len(name2terms)}个唯一名词")
|
||||
|
||||
# 3. 使用线程池并行处理
|
||||
merged_terms = []
|
||||
items_to_process = []
|
||||
|
||||
# 先处理只有一个条目的词条（不需要合并）
|
||||
for name, term_list in name2terms.items():
|
||||
if len(term_list) == 1:
|
||||
merged_terms.append(term_list[0])
|
||||
else:
|
||||
items_to_process.append((name, term_list))
|
||||
|
||||
logging.info(f"共{len(merged_terms)}个单一条目，{len(items_to_process)}个需要合并的条目")
|
||||
|
||||
# 只对需要合并的词条使用线程池处理
|
||||
if items_to_process:
|
||||
with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
|
||||
# 使用tqdm显示进度
|
||||
for result in tqdm(executor.map(self.process_term, items_to_process), total=len(items_to_process)):
|
||||
merged_terms.append(result)
|
||||
|
||||
# 4. 去重
|
||||
merged_terms = self.deduplicate_synonymous_name(merged_terms)
|
||||
|
||||
# 4. 保存合并结果
|
||||
os.makedirs(os.path.dirname(self.OUTPUT_PATH), exist_ok=True)
|
||||
with open(self.OUTPUT_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(merged_terms, f, ensure_ascii=False, indent=2)
|
||||
logging.info(f"合并后结果已保存到: {self.OUTPUT_PATH}")
|
||||
|
||||
return merged_terms
|
||||
|
||||
def deduplicate_synonymous_name(self, terms):
|
||||
# 1. 删除name字段重复的条目
|
||||
unique_names = set()
|
||||
unique_data = []
|
||||
|
||||
for item in terms:
|
||||
if item["name"] not in unique_names:
|
||||
unique_names.add(item["name"])
|
||||
unique_data.append(item)
|
||||
# 如果重复，则跳过该条目
|
||||
|
||||
# 2. 如果A条目的某一个synonymou字段是B条目的name，则删除A条目中的对应的synonymou
|
||||
name_set = {item["name"] for item in unique_data}
|
||||
|
||||
for item in unique_data:
|
||||
# 过滤掉synonymous中与其他条目name重复的部分
|
||||
filtered_synonymous = [syn for syn in item["synonymous"] if syn not in name_set]
|
||||
item["synonymous"] = filtered_synonymous
|
||||
|
||||
return unique_data
|
||||
|
||||
def main():
|
||||
"""主函数，创建TermMerger实例并执行合并"""
|
||||
|
||||
cur_path = os.path.dirname(__file__)
|
||||
input_dir = os.path.abspath(os.path.join(cur_path, '../../data/wiki_extracted_nouns'))
|
||||
output_path = os.path.join(cur_path, "..", "..", "data", "nouns", 'merged_nouns.json')
|
||||
merger = TermMerger(input_dir=input_dir, output_path=output_path, max_workers=20)
|
||||
merger.merge()
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.getLogger('httpx').setLevel(logging.WARNING)
|
||||
logging.getLogger('openai').setLevel(logging.WARNING)
|
||||
main()
|
||||
@@ -42,8 +42,6 @@ logging.basicConfig(
|
||||
|
||||
class DifyCompareTest:
|
||||
def __init__(self):
|
||||
# 先词条后工单检索工作流
|
||||
self.first_wiki_client = ChatClient(api_key="app-gocvuqduBnJptYNPpnW9V9R6", base_url=os.getenv("DIFY_BSAE_URL"))
|
||||
# 词条与工单同时检索
|
||||
self.both_wiki_worker_client = ChatClient(api_key="app-CPoOMaGDsLRPAe9TW7Xjhszy", base_url=os.getenv("DIFY_BSAE_URL"))
|
||||
self.llm = OpenAiLLM(base_url=os.getenv("OPENAI_API_BASE"), model=os.getenv("MODEL_NAME"))
|
||||
@@ -88,8 +86,8 @@ class DifyCompareTest:
|
||||
time.sleep(1) # 等待1秒后重试
|
||||
|
||||
|
||||
def process_workflow(self, workflow_name, client, inputs, query, old_answer):
|
||||
"""处理单个工作流调用"""
|
||||
def process_workflow(self, client, inputs, query, old_answer):
|
||||
"""处理工作流调用"""
|
||||
max_retries = 3
|
||||
retry_count = 0
|
||||
|
||||
@@ -107,14 +105,14 @@ class DifyCompareTest:
|
||||
except Exception as e:
|
||||
retry_count += 1
|
||||
if retry_count >= max_retries:
|
||||
logging.error(f"{workflow_name}调用失败 (尝试 {max_retries} 次后): {e}")
|
||||
logging.error(f"词条与工单同时检索调用失败 (尝试 {max_retries} 次后): {e}")
|
||||
return '', ''
|
||||
else:
|
||||
import time
|
||||
time.sleep(1) # 等待1秒后重试
|
||||
|
||||
def process_single_row(self, index, row):
|
||||
"""处理单行数据的方法，用于多线程执行"""
|
||||
"""处理单行数据的方法"""
|
||||
try:
|
||||
query = row["提问"]
|
||||
old_answer = row["回答"]
|
||||
@@ -125,47 +123,18 @@ class DifyCompareTest:
|
||||
"user_name": "AutoCodeRun"
|
||||
}
|
||||
|
||||
# 并行调用两个工作流
|
||||
results = {'first_wiki': None, 'both_wiki_worker': None}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as workflow_executor:
|
||||
# 提交两个工作流任务
|
||||
futures = {
|
||||
workflow_executor.submit(
|
||||
self.process_workflow,
|
||||
"先词条后工单工作流",
|
||||
self.first_wiki_client,
|
||||
inputs,
|
||||
query,
|
||||
old_answer
|
||||
): 'first_wiki',
|
||||
|
||||
workflow_executor.submit(
|
||||
self.process_workflow,
|
||||
"词条与工单同时工作流",
|
||||
# 调用词条与工单同时检索工作流
|
||||
answer, judge_result = self.process_workflow(
|
||||
self.both_wiki_worker_client,
|
||||
inputs,
|
||||
query,
|
||||
old_answer
|
||||
): 'both_wiki_worker'
|
||||
}
|
||||
|
||||
# 收集结果
|
||||
for future in as_completed(futures):
|
||||
workflow_key = futures[future]
|
||||
try:
|
||||
answer, judge_result = future.result()
|
||||
results[workflow_key] = (answer, judge_result)
|
||||
except Exception as e:
|
||||
logging.error(f"工作流执行失败 (行{index}): {e}")
|
||||
results[workflow_key] = ('', '')
|
||||
)
|
||||
|
||||
# 构建结果
|
||||
result_row = row.copy()
|
||||
result_row["先词条后工单回答"] = results['first_wiki'][0]
|
||||
result_row["先词条后工单回答对比"] = results['first_wiki'][1]
|
||||
result_row["词条与工单同时回答"] = results['both_wiki_worker'][0]
|
||||
result_row["词条与工单同时回答对比"] = results['both_wiki_worker'][1]
|
||||
result_row["词条与工单同时回答"] = answer
|
||||
result_row["词条与工单同时回答对比"] = judge_result
|
||||
|
||||
logging.info(f"成功处理第 {index + 1} 行数据")
|
||||
return index, result_row
|
||||
@@ -173,8 +142,6 @@ class DifyCompareTest:
|
||||
except Exception as e:
|
||||
logging.error(f"处理第 {index + 1} 行数据时出错: {e}")
|
||||
result_row = row.copy()
|
||||
result_row["先词条后工单回答"] = ''
|
||||
result_row["先词条后工单回答对比"] = ''
|
||||
result_row["词条与工单同时回答"] = ''
|
||||
result_row["词条与工单同时回答对比"] = ''
|
||||
return index, result_row
|
||||
@@ -232,8 +199,6 @@ class DifyCompareTest:
|
||||
logging.error(f"线程执行失败 (行{original_index + 1}): {e}")
|
||||
# 添加失败的行
|
||||
result_row = df.iloc[original_index].copy()
|
||||
result_row["先词条后工单回答"] = '线程执行失败'
|
||||
result_row["先词条后工单回答对比"] = '线程执行失败'
|
||||
result_row["词条与工单同时回答"] = '线程执行失败'
|
||||
result_row["词条与工单同时回答对比"] = '线程执行失败'
|
||||
results[original_index] = result_row
|
||||
|
||||
@@ -1,32 +1,17 @@
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
from threading import Thread
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import List, Dict, Any
|
||||
import logging
|
||||
import time
|
||||
import asyncio
|
||||
import httpx
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from rag2_0.intent_recognition.DataModels import Classification
|
||||
from rag2_0.dify.dify_client.client import DifyClient, KnowledgeBaseClient
|
||||
from rag2_0.tool.ModelTool import XinferenceReRankerModel
|
||||
class DifyQueryRetrieval:
|
||||
|
||||
software_to_dataset_map = {"配网工程计价通D3":["下载安装注册(new)","配网造价知识(new)","配网造价软件知识(new)"],
|
||||
"新型储能电站建设计价通C1":["下载安装注册(new)","储能C1计价通软件知识(new)","新能源造价知识(new)"],
|
||||
"西藏电力工程计价通Z1":["下载安装注册(new)","西藏造价知识(new)","西藏造价软件知识(new)"],
|
||||
"技改检修工程计价通T1":["下载安装注册(new)","技改检修工程计价通T1软件知识(new)","技改造价知识(new)"],
|
||||
"技改检修清单计价通T1":["下载安装注册(new)","技改检修清单计价通T1软件知识(new)","技改造价知识(new)"],
|
||||
"电力建设计价通":["下载安装注册(new)","主网造价知识(new)","电力建设计价通（2018）软件知识(new)"],
|
||||
"其他":["下载安装注册(new)","技改检修清单计价通T1软件知识(new)",
|
||||
"主网造价知识(new)","西藏造价知识(new)","技改检修工程计价通T1软件知识(new)",
|
||||
"电力建设计价通（2018）软件知识(new)","储能C1计价通软件知识(new)",
|
||||
"西藏造价软件知识(new)","新能源造价知识(new)","配网造价知识(new)","技改造价知识(new)",
|
||||
"配网造价软件知识(new)"]}
|
||||
|
||||
def __init__(self, dify_dataset_key: str, dify_base_url: str):
|
||||
self._dify_dataset_key = dify_dataset_key
|
||||
self._dify_base_url = dify_base_url
|
||||
@@ -38,7 +23,7 @@ class DifyQueryRetrieval:
|
||||
datasets_json = datasets.json()
|
||||
return {dataset["name"]:dataset for dataset in datasets_json["data"]}
|
||||
|
||||
def retrieve_by_dataset(self, query: str, dataset_name: str) -> List[Dict[str, Any]]:
|
||||
def retrieve_by_dataset(self, query: str, dataset_name: str) -> Dict[str, Any]:
|
||||
try:
|
||||
dataset_id = self._datasets_list[dataset_name]["id"]
|
||||
retrieval_model = self._datasets_list[dataset_name]["retrieval_model_dict"]
|
||||
@@ -52,12 +37,21 @@ class DifyQueryRetrieval:
|
||||
retrieved_document["dataset_id"] = dataset_id
|
||||
retrieved_document["dataset_name"] = dataset_name
|
||||
|
||||
return retrieved_documents
|
||||
# 返回包含查询和文档的字典
|
||||
return {
|
||||
"query": query,
|
||||
"dataset_name": dataset_name,
|
||||
"documents": retrieved_documents
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
||||
return []
|
||||
return {
|
||||
"query": query,
|
||||
"dataset_name": dataset_name,
|
||||
"documents": []
|
||||
}
|
||||
|
||||
async def retrieve_by_dataset_async(self, query: str, dataset_name: str) -> List[Dict[str, Any]]:
|
||||
async def retrieve_by_dataset_async(self, query: str, dataset_name: str) -> Dict[str, Any]:
|
||||
"""
|
||||
异步版本的retrieve_by_dataset方法
|
||||
|
||||
@@ -66,7 +60,7 @@ class DifyQueryRetrieval:
|
||||
dataset_name: 数据集名称
|
||||
|
||||
Returns:
|
||||
检索到的文档列表
|
||||
包含查询、数据集名称和检索到的文档的字典
|
||||
"""
|
||||
try:
|
||||
# 使用asyncio.to_thread包装同步方法
|
||||
@@ -77,81 +71,13 @@ class DifyQueryRetrieval:
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"异步检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
||||
return []
|
||||
return {
|
||||
"query": query,
|
||||
"dataset_name": dataset_name,
|
||||
"documents": []
|
||||
}
|
||||
|
||||
def retrieve(self, original_query: str, query_list: List[str], classification: Classification, software_name: str) -> Optional[List[Dict[str, Any]]]:
|
||||
datasets = self.get_datasets_by_classification(classification, software_name)
|
||||
datasets=["电力建设计价通（2018）软件知识(new)", "主网造价知识(new)", "下载安装注册(new)"]
|
||||
if len(datasets) == 0:
|
||||
return None
|
||||
|
||||
return self.retrieve_api(original_query, query_list, datasets)
|
||||
|
||||
async def retrieve_async(self, original_query: str, query_list: List[str], classification: Classification, software_name: str) -> Optional[List[Dict[str, Any]]]:
|
||||
"""
|
||||
异步版本的retrieve方法
|
||||
|
||||
Args:
|
||||
original_query: 原始查询
|
||||
query_list: 查询列表
|
||||
classification: 分类信息
|
||||
software_name: 软件名称
|
||||
|
||||
Returns:
|
||||
检索到的文档列表
|
||||
"""
|
||||
datasets = self.get_datasets_by_classification(classification, software_name)
|
||||
if len(datasets) == 0:
|
||||
return None
|
||||
|
||||
return await self.retrieve_api_async(original_query, query_list, datasets)
|
||||
|
||||
def retrieve_api(self, original_query: str, query_list: List[str],data_set_list: List[str], top_k: int = 5)->List[Dict[str, Any]]:
|
||||
ssss = self.retrieve_by_dataset("怎么调整报表顺序", "电力建设计价通（2018）软件知识(new)")
|
||||
all_documents=[]
|
||||
# 使用线程池替代无限制创建线程
|
||||
# 设置合理的最大线程数，这里使用min(32, len(query_list) * len(datasets))来限制
|
||||
time_start = time.time()
|
||||
max_workers = min(os.cpu_count() * 2, len(query_list) * len(data_set_list))
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {}
|
||||
for query in query_list:
|
||||
for dataset in data_set_list:
|
||||
if dataset not in list(self._datasets_list.keys()):
|
||||
raise ValueError(f"dataset {dataset} not in datasets_list")
|
||||
|
||||
futures[executor.submit(self.retrieve_by_dataset, query, dataset)] = query
|
||||
|
||||
# 等待所有任务完成
|
||||
for future in as_completed(futures.keys()):
|
||||
# 处理可能的异常
|
||||
try:
|
||||
retrieved_documents = future.result()
|
||||
all_documents.extend(retrieved_documents)
|
||||
except Exception as e:
|
||||
logging.error(f"检索过程中发生错误: {str(e)}", exc_info=True)
|
||||
time_end = time.time()
|
||||
|
||||
logging.info(f"检索耗时: {time_end - time_start:.2f}秒")
|
||||
# 根据segment_id对文档进行去重
|
||||
unique_documents = {}
|
||||
for document in all_documents:
|
||||
segment_id = document['segment']['id']
|
||||
if segment_id not in unique_documents:
|
||||
unique_documents[segment_id] = document
|
||||
|
||||
# 将去重后的文档转换为列表
|
||||
deduplicated_documents = list(unique_documents.values())
|
||||
|
||||
# 对所有检索出来的文档进行重排序
|
||||
time_start = time.time()
|
||||
processed_documents = self.data_post_processor("怎么调整报表顺序", deduplicated_documents, top_k)
|
||||
time_end = time.time()
|
||||
logging.info(f"检索后重排序耗时: {time_end - time_start:.2f}秒")
|
||||
|
||||
return processed_documents
|
||||
|
||||
async def retrieve_api_async(self, original_query: str, query_list: List[str], data_set_list: List[str], top_k: int = 5)->List[Dict[str, Any]]:
|
||||
async def retrieve_api_async(self, original_query: str, query_list: List[str], data_set_list: List[str], query_expand_dict: dict, top_k: int = 5)->Dict[str, Any]:
|
||||
"""
|
||||
异步版本的retrieve_api方法，使用asyncio代替线程池
|
||||
|
||||
@@ -159,11 +85,14 @@ class DifyQueryRetrieval:
|
||||
original_query: 原始查询
|
||||
query_list: 查询列表
|
||||
data_set_list: 数据集列表
|
||||
query_expand_dict: 查询扩展字典，包含不同类型的查询
|
||||
top_k: 返回的文档数量
|
||||
|
||||
Returns:
|
||||
检索并重排序后的文档列表
|
||||
包含检索结果和查询命中统计的字典
|
||||
"""
|
||||
all_documents = []
|
||||
query_document_mapping = {} # 用于存储查询和文档的映射关系，键为查询，值为文档列表
|
||||
# 记录开始时间
|
||||
time_start = time.time()
|
||||
|
||||
@@ -187,7 +116,16 @@ class DifyQueryRetrieval:
|
||||
if isinstance(result, Exception):
|
||||
logging.error(f"异步检索过程中发生错误: {str(result)}", exc_info=True)
|
||||
else:
|
||||
all_documents.extend(result)
|
||||
# 添加查询和文档的映射关系
|
||||
query = result["query"]
|
||||
if query not in query_document_mapping:
|
||||
query_document_mapping[query] = []
|
||||
|
||||
# 将文档添加到对应查询的文档列表中
|
||||
query_document_mapping[query].extend([item['segment']['id'] for item in result["documents"]])
|
||||
|
||||
# 将文档添加到总文档列表
|
||||
all_documents.extend(result["documents"])
|
||||
|
||||
time_end = time.time()
|
||||
logging.info(f"异步检索耗时: {time_end - time_start:.2f}秒")
|
||||
@@ -203,50 +141,62 @@ class DifyQueryRetrieval:
|
||||
deduplicated_documents = list(unique_documents.values())
|
||||
|
||||
if len(deduplicated_documents) == 0:
|
||||
return []
|
||||
return {
|
||||
"documents": [],
|
||||
"query_hit_stats": {}
|
||||
}
|
||||
|
||||
# 对所有检索出来的文档进行重排序
|
||||
time_start = time.time()
|
||||
processed_documents = await self.data_post_processor_async(original_query, deduplicated_documents, top_k)
|
||||
time_end = time.time()
|
||||
logging.info(f"异步检索后重排序耗时: {time_end - time_start:.2f}秒")
|
||||
|
||||
return processed_documents
|
||||
# 统计不同类型查询命中的文档
|
||||
query_hit_stats = {}
|
||||
|
||||
def data_post_processor(self, query: str, all_documents: List[Dict[str, Any]], top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
reranker_model = XinferenceReRankerModel()
|
||||
documents = [document['segment']['content'] for document in all_documents]
|
||||
reranked_documents = reranker_model.rerank(query, documents, top_k=top_k)
|
||||
new_all_documents = []
|
||||
# 获取重排序后的文档ID列表
|
||||
reranked_doc_ids = [doc["metadata"]["segment_id"] for doc in processed_documents]
|
||||
reranked_doc_titles = [doc["title"].split("/")[-1] for doc in processed_documents]
|
||||
|
||||
# 解析query_expand_dict（如果是字符串）
|
||||
if isinstance(query_expand_dict, str):
|
||||
try:
|
||||
query_expand_dict = json.loads(query_expand_dict)
|
||||
except Exception as e:
|
||||
logging.error(f"解析query_expand_dict失败: {str(e)}")
|
||||
query_expand_dict = {}
|
||||
|
||||
# 统计各类型查询命中的文档
|
||||
for query_type, queries in query_expand_dict.items():
|
||||
if not isinstance(queries, list):
|
||||
queries = [queries]
|
||||
|
||||
# 初始化该查询类型命中的文档列表
|
||||
query_hit_stats[query_type] = []
|
||||
|
||||
# 合并所有该类型查询命中的文档
|
||||
hit_doc_ids_set = set()
|
||||
for query in queries:
|
||||
if query in query_document_mapping:
|
||||
hit_doc_ids_set.update(set(query_document_mapping[query]))
|
||||
|
||||
# 找出在重排序结果中的文档
|
||||
hit_doc_titles_set = set() # 用于去重
|
||||
for i, doc_id in enumerate(reranked_doc_ids):
|
||||
if doc_id in hit_doc_ids_set:
|
||||
doc_title = reranked_doc_titles[i]
|
||||
if doc_title not in hit_doc_titles_set: # 确保不添加重复的标题
|
||||
hit_doc_titles_set.add(doc_title)
|
||||
query_hit_stats[query_type].append(doc_title)
|
||||
|
||||
logging.info(f"查询命中统计: {json.dumps(query_hit_stats, ensure_ascii=False)}")
|
||||
|
||||
def to_dify_document_format(document: dict)->dict:
|
||||
return {
|
||||
"metadata": {
|
||||
"_source": "knowledge",
|
||||
"dataset_id": document["dataset_id"],
|
||||
"dataset_name": document["dataset_name"],
|
||||
"document_id": document['segment']['document_id'],
|
||||
"document_name": document["segment"]["document"]["name"],
|
||||
"data_source_type": document["segment"]["document"]["data_source_type"],
|
||||
"segment_id": document["segment"]["id"],
|
||||
"retriever_from": "api",
|
||||
"score": document.get("score", 0),
|
||||
"segment_hit_count": document.get("segment", {}).get("hit_count", 0),
|
||||
"segment_word_count": document.get("segment", {}).get("word_count", 0),
|
||||
"segment_position": document.get("segment", {}).get("position", 0),
|
||||
"segment_index_node_hash": document.get("segment", {}).get("index_node_hash", ""),
|
||||
"doc_metadata": document.get("segment", {}).get("document", {}).get("doc_metadata", None),
|
||||
"position": document["segment"].get("position", 0)
|
||||
},
|
||||
"title": document["segment"]["document"]["name"],
|
||||
"content": document["segment"]["content"]
|
||||
"documents": processed_documents,
|
||||
"query_hit_stats": query_hit_stats
|
||||
}
|
||||
|
||||
for reranked_document in reranked_documents:
|
||||
cur_doc_info = all_documents[reranked_document["index"]]
|
||||
cur_doc_info["score"] = reranked_document["score"]
|
||||
new_all_documents.append(to_dify_document_format(cur_doc_info))
|
||||
return new_all_documents
|
||||
|
||||
async def data_post_processor_async(self, query: str, all_documents: List[Dict[str, Any]], top_k: int = 5) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
异步版本的data_post_processor方法
|
||||
@@ -293,37 +243,9 @@ class DifyQueryRetrieval:
|
||||
new_all_documents.append(to_dify_document_format(cur_doc_info))
|
||||
return new_all_documents
|
||||
|
||||
def get_datasets_by_classification(self, classification: Classification, software_name: str) -> List[str]:
|
||||
if classification.vertical_classification=="软件问题" or classification.vertical_classification=="业务问题":
|
||||
software_name_list = self.software_to_dataset_map.keys()
|
||||
cur_software_name = ""
|
||||
for software_name_info in software_name_list:
|
||||
if software_name_info in software_name:
|
||||
cur_software_name = software_name_info
|
||||
break
|
||||
if cur_software_name == "":
|
||||
return self.software_to_dataset_map["其他"]
|
||||
else:
|
||||
return self.software_to_dataset_map[cur_software_name]
|
||||
|
||||
if classification.vertical_classification == "安装下载注册":
|
||||
if classification.sub_classification in ["后缀名咨询", "软件锁类"]:
|
||||
return ["下载安装注册(new)"]
|
||||
elif classification.sub_classification == "安装下载类":
|
||||
return []
|
||||
elif classification.sub_classification == "问题排查":
|
||||
return self.software_to_dataset_map["其他"]
|
||||
|
||||
return self.software_to_dataset_map["其他"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dify_query_retrieval = DifyQueryRetrieval(api_key="dataset-skLjmPVonjHo119OWNf3kAmY", base_url="http://10.1.16.39/v1")
|
||||
# datasets = dify_query_retrieval.retrieve("配网工程计价通D3软件如何新建工程？", Classification(vertical_classification="软件问题", sub_classification="软件功能"), "配网工程计价通D3")
|
||||
# datasets = dify_query_retrieval.retrieve_api("电力建设计价通软件如何批量修改设备价格？",
|
||||
# ["电力建设计价通软件如何批量修改设备价格？"],
|
||||
# ["电力建设计价通（2018）软件知识(new)"])
|
||||
# print(json.dumps(datasets, ensure_ascii=False, indent=2))
|
||||
|
||||
# 测试异步API
|
||||
async def test_async_api():
|
||||
|
||||
@@ -39,6 +39,7 @@ class RetrieveRequest(BaseModel):
|
||||
original_query: str
|
||||
query_list: str
|
||||
data_set_list: str
|
||||
query_expand_dict: dict | str = Field(default="{}")
|
||||
|
||||
# 创建FastAPI应用
|
||||
app = FastAPI(
|
||||
@@ -87,13 +88,17 @@ async def retrieve(request: RetrieveRequest):
|
||||
# 解析查询列表和数据集列表
|
||||
query_list = request.query_list.split("<sub_query>")
|
||||
data_set_list = request.data_set_list.split("<dataset>")
|
||||
|
||||
if isinstance(request.query_expand_dict, str):
|
||||
query_expand_dict = json.loads(request.query_expand_dict)
|
||||
else:
|
||||
query_expand_dict = request.query_expand_dict
|
||||
# 调用异步检索方法
|
||||
start_time = time.time()
|
||||
results = await dify_query_retrieval.retrieve_api_async(
|
||||
request.original_query,
|
||||
query_list,
|
||||
data_set_list,
|
||||
query_expand_dict=query_expand_dict,
|
||||
top_k=5
|
||||
)
|
||||
end_time = time.time()
|
||||
@@ -107,7 +112,7 @@ async def retrieve(request: RetrieveRequest):
|
||||
if __name__ == "__main__":
|
||||
# 使用Uvicorn运行FastAPI应用
|
||||
import uvicorn
|
||||
uvicorn.run("rag2_0.dify.DifyQueryRetrieval_api:app", host="0.0.0.0", port=8002, reload=False, workers=10, log_level="info")
|
||||
uvicorn.run("rag2_0.dify.DifyQueryRetrieval_api:app", host="0.0.0.0", port=8002, reload=False, workers=1, log_level="info")
|
||||
# # 使用uvicorn启动服务
|
||||
# import uvicorn
|
||||
# uvicorn.run(
|
||||
|
||||
Reference in New Issue
Block a user