更新.gitignore以忽略临时文件，修改api_key文件，重构合并名词的逻辑，删除不再使用的脚本，优化对话到工单的处理流程，添加会话结果保存为JSON的功能，调整API调用参数，修复部分代码中的错误。
This commit is contained in:
+2
-1
@@ -13,4 +13,5 @@ data/logs/*
|
|||||||
rag2_0/dify/Test.py
|
rag2_0/dify/Test.py
|
||||||
data/query_logs/*
|
data/query_logs/*
|
||||||
data/conversations/*
|
data/conversations/*
|
||||||
data/test*
|
data/test*
|
||||||
|
data/temp*
|
||||||
+1
-382
@@ -1,10 +1,3 @@
|
|||||||
sk-fqiorgwugzcrnwrkcnfsiaikkthgzzupzrvnwidbdmehpidn
|
|
||||||
sk-zdhbhipwbffoqmoxfxjsynfyjximahraxodzadiirnpbuzsg
|
|
||||||
sk-wsqrpkdpkszwdebckaxxbsjhovfwlosctzfpujxxpvhqkvyz
|
|
||||||
sk-krrffrphigolboeegpogoksdyeuxbaohiaxtflandfclmxsk
|
|
||||||
sk-hsbehxbxnpigxoootnjqlpckcezhlhypdgnabpbywnmquiqe
|
|
||||||
sk-fonqeeexzdgocliiwffopgdesvcmnlqtditsafklqgnqyxvm
|
|
||||||
sk-hjhftlgfeswvcsuowabthdkriaevhppvbgoriyiydgahfrgk
|
|
||||||
sk-cxlvgeuxavxfcajprxietuqyqjngtbrwrmrmrioxmgtbkpci
|
sk-cxlvgeuxavxfcajprxietuqyqjngtbrwrmrmrioxmgtbkpci
|
||||||
sk-vjjsuzntqbhcmelfsuquqyoxjivxcfwyxnrhpwzobgxlpmrv
|
sk-vjjsuzntqbhcmelfsuquqyoxjivxcfwyxnrhpwzobgxlpmrv
|
||||||
sk-hbgctnpvntsnelveaudpekyncfgstdfazezboxmcgjvudzyg
|
sk-hbgctnpvntsnelveaudpekyncfgstdfazezboxmcgjvudzyg
|
||||||
@@ -15,26 +8,6 @@ sk-kdfrkixwoeiuzdsdtystcxbhvwzeschacrnvhfiupnilnpoo
|
|||||||
sk-kelmbfsruwmjgrzzbjkqpqasabrnuskongtqxwzrfrpefdgz
|
sk-kelmbfsruwmjgrzzbjkqpqasabrnuskongtqxwzrfrpefdgz
|
||||||
sk-mtnlhcosddoztgmdgjmtlniqsrkfiskafmthtmnyzqmdfcib
|
sk-mtnlhcosddoztgmdgjmtlniqsrkfiskafmthtmnyzqmdfcib
|
||||||
sk-cnrleygygpuqogpqyvuqnhubkwschrtcpjlldsdhxrkhgttf
|
sk-cnrleygygpuqogpqyvuqnhubkwschrtcpjlldsdhxrkhgttf
|
||||||
sk-hycclhscssbpyorblvaalhugdacpsfhwppihzlxckiqvqqdp
|
|
||||||
sk-qpwzzrvywgblfzsvwachxpuawfuffyqichnkfxwnotgzxvic
|
|
||||||
sk-qhbearldhnfaiipdnysouxorzqufpntslmlpceymdwgyxbub
|
|
||||||
sk-ehaxrzfpfzzjgonerqpnbldhrgfhpbfrokasmalduycvkxuy
|
|
||||||
sk-ckjrxmiwpmcehxnxqkyycicsstxqmibayymivijokaygafcy
|
|
||||||
sk-dayrbzxeeiuemrgzytqcijdfeuapstvzzsjifhfuizdghawn
|
|
||||||
sk-ojllzcltirvfxcfpuzjhxihsbyhdqpbrbxchlnkcvswldhpi
|
|
||||||
sk-cofofgsmlsputithtgbkeuwpeiijiprwulslchadgnxhgszh
|
|
||||||
sk-xitvnuyghqbdwvmgwjcfpsgrvfissovdornhynhmiqfuvxyl
|
|
||||||
sk-ynmlgshhqbfuzgcnrkmdvepdmnicskhvdzvyibdfctruwplc
|
|
||||||
sk-dgijfputvuywypkciylwtlqmsqolosvcmapfiwqjzsiodxmw
|
|
||||||
sk-fkhreothoprxhfgymwdyfujmgqpvqbuozmbihvyhfggysdbb
|
|
||||||
sk-rbixbbhxgqqzgziprzavfkcoawjeyhvowanbiyiaguqxdztf
|
|
||||||
sk-nbknzrprpqkhyevsxthqoyeuehjuuubptrcxnlzoqviuqnck
|
|
||||||
sk-scsexswqgeykaikligbjdjljqywcwlckeqkklawjoyrjalkq
|
|
||||||
sk-kbaapbyjpsylehbgyieenaglcjwbjyihokeocshlgypkjkzu
|
|
||||||
sk-mblgxsdlkdtzeckqmkcspiomeakgstdfduqvzjcbprhyqmka
|
|
||||||
sk-dcmlgeidlpigamazrcrbdenugyhrmiunrnjpuxdjuduqdckh
|
|
||||||
sk-dianaiylqnlcoljmxnfbsqowwcyctsuahfvztcawwwmxesus
|
|
||||||
sk-mvhlbzbfkrtwgriasuzbvvnfcfkirslhchlbknjndrkwyufx
|
|
||||||
sk-xgkzyzptybrnxanvugkynczygyfavddwwzhbgoyxkhgaykci
|
sk-xgkzyzptybrnxanvugkynczygyfavddwwzhbgoyxkhgaykci
|
||||||
sk-nfpftnxyiovqlzkqsleadsfnbpfxdxbdpghdwfyucrnzaimh
|
sk-nfpftnxyiovqlzkqsleadsfnbpfxdxbdpghdwfyucrnzaimh
|
||||||
sk-uiuhaihdvthpmwqeehmwufagpxcjunoqnfgpnaitaaqgrjoi
|
sk-uiuhaihdvthpmwqeehmwufagpxcjunoqnfgpnaitaaqgrjoi
|
||||||
@@ -128,49 +101,6 @@ sk-ufmqbuplpjvzzlzohvsxertwgnguhipsbajxnxecvvccozly
|
|||||||
sk-rypfoscrczeelowmrsixiuyunyqmqvknaprsnzmdguwzrkzx
|
sk-rypfoscrczeelowmrsixiuyunyqmqvknaprsnzmdguwzrkzx
|
||||||
sk-lucemnosmcxuwedvzilpefuxjnyvaxldpbgaqwnwalxmntul
|
sk-lucemnosmcxuwedvzilpefuxjnyvaxldpbgaqwnwalxmntul
|
||||||
sk-niymkyuzpyovndvvqvpaniiqfgoofnxczhdmjjessiocbeul
|
sk-niymkyuzpyovndvvqvpaniiqfgoofnxczhdmjjessiocbeul
|
||||||
sk-ymrktzmmacferrdkbohbnxzqnhzqnayfopmeaallsjnehqcw
|
|
||||||
sk-zgrnzafrveugvkqthacmjvzmrhrasmjstpldkenmtapyvmvh
|
|
||||||
sk-lyutswiifkfuzyzfuzhrtydmwiblytrmavzwkhwzafrkpkvt
|
|
||||||
sk-sfvenswvedgsmuqfqwlphxmawdsnfruuxrksqsumclmkcepo
|
|
||||||
sk-tqmpqwdczmvdymylfzsaymwpxopthsquwxbrgajbfqzoquuh
|
|
||||||
sk-azajlrrajctqzeqlkdiifogcigkqikzvtlmxicftrqbtfiwj
|
|
||||||
sk-poqooaejvsycpwveujigfbbqukuiftbhfnaamdcnzbeofbgz
|
|
||||||
sk-ackysjgqtlxikiuhfjspflaspjyvndrydeoyaflfkrjedqzw
|
|
||||||
sk-taddhesluxmdzaaqhkakemyputzuxkhvziregheczzvbicbe
|
|
||||||
sk-wwnbxazrsjzojiwddwktxytxcrnclhtqnxreanzvvsulmeqa
|
|
||||||
sk-yokhwoynevvdimlcfhgjkczwnmlvpegiinpydyomukjvcpoa
|
|
||||||
sk-wwzquvlaxrxskxvystkrzqppiufjreaibofudrtxhquepktg
|
|
||||||
sk-akvqccqgqeqltfqotmpglxlhqnkxthefwrfrannjiqgfexnd
|
|
||||||
sk-bpergewkadzmoyipuwvvkmcenljxvvxocaapswmqwwzhctwu
|
|
||||||
sk-xygqvydpnpzrrtqjmarayjzlmwkvertuxviejezgbhqjsono
|
|
||||||
sk-yneqvefpfddedlhbmzixrjayhyafysiwuoaiccwpcnwlihvn
|
|
||||||
sk-rbfzsdhnpsfwevtuaytcaerqvjjkyjdrksyfsizzcfslivdy
|
|
||||||
sk-lnwngttbwhlshqimlnlzcogpdewuexoloxtsjgabsnrxiwum
|
|
||||||
sk-zlgdfdxsotldyhgvapvwogxurjayztvsbnokzsoxepiqngwx
|
|
||||||
sk-jqqmmvivlqqahobcznjpfsjuhmrypjwzfzzdfjeuanmgrjvj
|
|
||||||
sk-kwgkxwwjpodvdfzcqcsildhwftfwqdufbfuqwzthsxnvuawb
|
|
||||||
sk-pkhanejghbxnyskcrhheogvtlkibspvlnxseanvywwwphyzx
|
|
||||||
sk-qhmotavtivehqghttsziemwywwkzffodysdrzedpmcplgjse
|
|
||||||
sk-jspkuwzvusqozxthglfhuzhjurqgqmxgfjvfuatkwboutoow
|
|
||||||
sk-mdakwcubtvvtomakhdtepaoxejcgbypxadnyccrehjztmrvz
|
|
||||||
sk-dpawjyvlmfjbamtkqvdqfavwmwfilwonfcnhpcmudydwdroh
|
|
||||||
sk-givozfwetofaxqyocrhzsvfoqfrkuhjocorffgkyrquaxhql
|
|
||||||
sk-agqbvfcicmxobifpmpqiivkddiicolyhxsxelknkoiqsxczy
|
|
||||||
sk-hywltpywcdhtcuxgrsydtnnpbihglytgaffqifubtanslgkb
|
|
||||||
sk-enlmvvueqloxhickqhnyzyhchjaxolccxeihjgfegubxqdau
|
|
||||||
sk-njinpeycwccspxyvwxjjurgccolpjwkyrvshruqrgqeyuket
|
|
||||||
sk-kaofxorgblnyhyikderbhojucapccgiaabmrjvtprpjdidyr
|
|
||||||
sk-lhlnnrjmdyadbpotbzjhexvpqdsakkawzamgvuvvvwrjxfuw
|
|
||||||
sk-kbdxyxsqrjzxabmlnudbfmkoueetvozavetvynzebxeywtiz
|
|
||||||
sk-pmkpljgfuobflxamthdiciwxxrohttudjacvnelxucfepsds
|
|
||||||
sk-buouosufejuluopcastaumdwdkxtojrwtenvuolrnwgnogrz
|
|
||||||
sk-xuptfizbgnytnfylewbdyqzktoeosinvjqpvmilfjjoenjcf
|
|
||||||
sk-lzqwcqiwgxowodrhyewykbeazqemdiynvanwgxjilhgabqtp
|
|
||||||
sk-ogasgwkcgvdtuforliczletkpteoininxtzwxiwgqyjxynuf
|
|
||||||
sk-ijmecfxhdsnzrolannjmpedwxwckdjbxrofygqzpkpwfhcar
|
|
||||||
sk-uuqiaramzroyhjtywxpfiulieesatwdbugpomxqxgfvuhryz
|
|
||||||
sk-uouyvhsnnnjrxpjhacuehosbnksgnvtvhfdryoscwicuvcyl
|
|
||||||
sk-exsjriliucrifygufvjhycgycdinafmdegnorlprigpugask
|
|
||||||
sk-gkdrgcbrlddkgflolrqkpqyygmuszueozeoiwdqpeqghlths
|
sk-gkdrgcbrlddkgflolrqkpqyygmuszueozeoiwdqpeqghlths
|
||||||
sk-gjqsqttmnydxavthjsjnirqxprogmvidmocfeoixuwutiqmd
|
sk-gjqsqttmnydxavthjsjnirqxprogmvidmocfeoixuwutiqmd
|
||||||
sk-pihgxlhhdsyeowjsbhbioxwqjbonwunteaubfisduzyibkzw
|
sk-pihgxlhhdsyeowjsbhbioxwqjbonwunteaubfisduzyibkzw
|
||||||
@@ -191,126 +121,6 @@ sk-mtdxvrivtguhurniwlkkpiiqvsldzupdxeikktjjcecokprb
|
|||||||
sk-mlsnfruyphvjexbbxmpfzmqrfolmnmvagtartricqtvcfdcm
|
sk-mlsnfruyphvjexbbxmpfzmqrfolmnmvagtartricqtvcfdcm
|
||||||
sk-ofwsljbaccfrlecrwelbzwaervcdhayakfccltttqklfvjbq
|
sk-ofwsljbaccfrlecrwelbzwaervcdhayakfccltttqklfvjbq
|
||||||
sk-hrocudrwzukputieyvwyqmxexladghassxhdpyofitevmlwa
|
sk-hrocudrwzukputieyvwyqmxexladghassxhdpyofitevmlwa
|
||||||
sk-odakxyiimzxrgeqxbwqdstrsebubrfwpyjlylnmvsrscnkha
|
|
||||||
sk-cdgfctificqxybfnnqplpqiwjakocobsqssudngsvxyshbuu
|
|
||||||
sk-etdalqllnekhfihnybtvbwbdvwgmjfqgbersdgzyznqlmouw
|
|
||||||
sk-oonkwqqvblmoktvfhvzaxpinbgvwzqpwdhqhckessqnfgzhd
|
|
||||||
sk-nhosrcqzlmajdecooxnarjuzsnymzmvgsyckjftkokjzahzd
|
|
||||||
sk-nrbczpgnvfpeunrxxueavbbnnfqfqwxjsnvjbuzxeyuqqcpl
|
|
||||||
sk-cobqudjpmsvjyazzdfirhhvhxggjnupqpbhyniavpdovmtte
|
|
||||||
sk-fwqomjfvbjdwvnsansrkmvxriflylzbvtwosybuthopxnmlz
|
|
||||||
sk-mlfemdejrncmwrtdqvjxwseeevoltaapteawikshisjnqasb
|
|
||||||
sk-jpwmjlzlswykpjotranggkwljmujweizsgunsswwzvzjoppo
|
|
||||||
sk-gqtxgkgzyubkggfyzddsuukamnreqnwsbhfllymflptuuhwe
|
|
||||||
sk-nnbkwmaychnfrvwnuqfwvhabrdnpjzkftdkzgcygnryjgwjg
|
|
||||||
sk-odxaszrrcpldjdbpaufpbwisulwsicekcmzbydvgyuxwwptb
|
|
||||||
sk-gasievqwwlecetmorcekmpwvkjseykkyjvudxpvvmilxklak
|
|
||||||
sk-cmrpqbrshozujdasrdmzaatiuhvdweswxfimoplaztultitn
|
|
||||||
sk-elbtjdkqzpdvabhuxdfwozekfxmnoojazdkjgiocgpcxlnxv
|
|
||||||
sk-tjicvjxhtceicrriirsdqcmciobuobbtcarthlypruxurirs
|
|
||||||
sk-hstobggtwmsjewhixwkmaqgpqxwafcibehmdrondxuimqnpq
|
|
||||||
sk-urlllffydhsqnexmlguxlvuyhbjzovzvkmzhhmuyirqwmryx
|
|
||||||
sk-opdnnbzwnmguzsrsyhwcwzfqtzcjeteenutpqeaapibnpfaq
|
|
||||||
sk-ckierrmqzhahjwxtizavwwwgiiehijbhjctgvlugzytylrug
|
|
||||||
sk-ewcebklekmctyzfstecfsxzvjegvotijtltlgeequzcpagqv
|
|
||||||
sk-mgzhwhxkmarcycfbzvrlgxlqikozpsfcxnhcrnkdvcqwqksy
|
|
||||||
sk-rmyjpdhytpioibzqffijzshstpolxvcjghxhgmzzxomayiyw
|
|
||||||
sk-yyhyekrhcnljpynqdoxsqovsauxgyqqiwgxqavykjwpjusqd
|
|
||||||
sk-gxapgxldxpjhdznladnwtqkdjqmdgznjmezsksczifbjkbpk
|
|
||||||
sk-kbrjttdnrfbuvzjakhohavoxorlynofradwwxdttlrqtlezx
|
|
||||||
sk-zknrvhlmqgknlwiphvmnwlulhrtwicrkvkkfpnkpzxgbnthv
|
|
||||||
sk-xbddhilpckwvzxacqjiocmaymbtqlahzwoyzrbdwqidmejvz
|
|
||||||
sk-ueseyqbtfikezynjsenkehpwqqzooeocoabkithtmudmihsr
|
|
||||||
sk-zrbtpufhxbeixieaykkcadcntipflbwrhfiicuaxopujvdvf
|
|
||||||
sk-dnlydvatnswufwkchxdqiuaroxnmozfwvhhotxinrwzjytct
|
|
||||||
sk-jhjebzqwkngjjgnpelwzcqlzmbczprkcjzzyumfbhsfvsqwg
|
|
||||||
sk-xgbwqgrfznxkebewuqmzohfnhpbuxprmccfjemucyeofnbge
|
|
||||||
sk-jmngmbdxbwagjofvsagutritvwqogyancfavtvlraoukibyz
|
|
||||||
sk-qhiuffgnpabnamfyzffdnmgoevmtnhmsjyglqcjhvwublvgn
|
|
||||||
sk-edzbujylbkvsvfadtcfzitavymeshrmesjsdsllcpubwrwto
|
|
||||||
sk-rqpsvficelzysisksmsosmapjrrbqaaubiwluhcihmkgnezf
|
|
||||||
sk-hkkcehvotmyurxbtzqzxkesyfnulqxnjycfxrfcjsqjsjqlm
|
|
||||||
sk-lqdqjzumoqtcnvdhicpzfgmbmyjocyuhmfjzmbstijtjjyux
|
|
||||||
sk-hwqptxptjgzdlppdffhinqjpxgiycidjmllbzzhslqraedwn
|
|
||||||
sk-enjmyliemusvcrydmljjauvzwdplnwkfxfucwbhexcylxcgq
|
|
||||||
sk-nurshorcqojkepxiefwyjgkbxeohdadabmcneenkkvodgset
|
|
||||||
sk-pnvtsjmmcmgeetafqfskumilkxiflwcgqmqqpwurrkqdddew
|
|
||||||
sk-lgygzwevhgnqlvvjzrwtwgbjsxfhucbzszydawpjjmkyiaof
|
|
||||||
sk-cczpctnalpbifauarekxbijlpostgydhhnyyljggoipdgncx
|
|
||||||
sk-evoyieozyhttwnnuwefzafclgasnsxgaslfqabiefaljqnhr
|
|
||||||
sk-tqzgtdgrkglowsehrzqrfcarhadtjwypyxghktcjgucyppfb
|
|
||||||
sk-iubygvdrbzqchsmlcpzrwnlghwlnjsyxqhoknegkdlanzkwh
|
|
||||||
sk-vxkjotcvrbggozqhscrsmghxtsrnjjgbektdbvuxodjjkyju
|
|
||||||
sk-nrkgnrmgpuygzstkodmiqpgvdzocokxkweijfjptwyypgdyy
|
|
||||||
sk-bjogsxmvdpiwoqkrglouyadwkkfgrwjmzfwrzbscvyxjjwyb
|
|
||||||
sk-gmtoauotodrgqdhoruxkyyjdkaemfcfveyklkrsyzdteqbmq
|
|
||||||
sk-clxaqlxyhhvqarjvqncidzgldnpguvwkqfqdqwytsouvayii
|
|
||||||
sk-fvvjzijhlhykvxobuqxzlphrtuslzirpjcxieohelcykiltv
|
|
||||||
sk-ritcploocwoprekoxbkolrlyilbengvvkbncifgkqcrtspsw
|
|
||||||
sk-iqvrozujapnnosyztxnodwgnlcoverpagxnohnpoadphmitc
|
|
||||||
sk-byleqeovamwprlaykzgucctkksqcreblnafmeprbbvvbuihz
|
|
||||||
sk-zjnfveqeyryargtgysszbecduuieektgfmttvrpvexxvkakg
|
|
||||||
sk-iueoowqajryvqfcouyesvotdafzftccfwsvawqlabclhyyqv
|
|
||||||
sk-pwhnwlmhttwxruafxquvnqeckpvwgpdvyppybobynuwzghjt
|
|
||||||
sk-gbdtryfivlfmrbzxkoymjkujjtuicvjlnynmswyxbokoogjz
|
|
||||||
sk-aguuqmyoxztbytcqefsazfzpcqmpyaikvuqsuvbrfmkbixld
|
|
||||||
sk-yajbtnqdskoyvtnlqtvjbiovttjgihpyrdvzxygbsmlstviu
|
|
||||||
sk-uzukjlqctiqjqumkagaarttqhcgmxcqansqueswefiiomwfe
|
|
||||||
sk-ivgrwasidrharxlsnjajwnbjmqddmiztlqiikmowocqtsgds
|
|
||||||
sk-ibggmzmjsuufgnmgtlrbbwmxiuvpwdclqhisqlxfokuxdydh
|
|
||||||
sk-lprfpppdrmvnbvdhghpywgqhcgnldzegxhyfhwvpidkmqqrh
|
|
||||||
sk-lqvrxzqesqchefxvbraczpiszykuyflckfhtyubxlfqkwttm
|
|
||||||
sk-loezhefsmqhzwlstqkeynfnpgjclpbizxrjcgzutvaxzeawg
|
|
||||||
sk-ydmfxaqtwhqcvsgoxxxvvfhmpepeknzbypgdgzugjuqbyduh
|
|
||||||
sk-hzsvoziyxfmdbqloguctveimdxfiwclvrymkyohxaympdqsg
|
|
||||||
sk-gkxyvdiadzfcwcsosyctvcajoxomlrhqnnojevvmyuhwchjy
|
|
||||||
sk-pppdgqssjhqistbrreapfrdbqoncebjmqvwclbcnkikfyaig
|
|
||||||
sk-ufixudywbeffcsuknffxowczzdbykwoovkjlgjjpcirgdjjd
|
|
||||||
sk-plhbmyzqfnxvmcqlhuhywbkfgjqbevpdzuumvqrqiaudcldy
|
|
||||||
sk-yjoitaywenwwrlllegmphporcwxpgpslzzmqararohdkgxfa
|
|
||||||
sk-mssalskekreuyztesydcbmzwvubfxxxxfjndcuzhudulgaaz
|
|
||||||
sk-vtkfdwzvxnhfvncqfmrklgpxpmwkpxdgrryradskbprpbgln
|
|
||||||
sk-ixutedkcbcuwamiynidhtioqvxwzaypbqrahqgnzukslljyf
|
|
||||||
sk-sjwasvpkfvmoruypknbeudbsxebmgzsyocnznvuruhtbxkdp
|
|
||||||
sk-btgppgymypnzginwwfzwtvpcsjjsmyveofqopcpegceroxei
|
|
||||||
sk-ufgenhaozdkykuoczjfywvsthsoxnnphxjaroeejouwffheb
|
|
||||||
sk-umhxbxavxtlrizrcvwzzqggykpruojebdelwqpxoegurzizk
|
|
||||||
sk-infbwvcscwfeovpnziofepodzfvcynpplpgagfouqssgnnmc
|
|
||||||
sk-sfahqxtknyovnecotshqcojrtadadxvmgjbkzpmfwcmapnhr
|
|
||||||
sk-szagfyiyjftymfswkoplmblxaofbnjrllalxzsekyduqbmrm
|
|
||||||
sk-uzallmwkhjnswjhhyiubfrkskzkcpbkwpgppsklnotmawfut
|
|
||||||
sk-hnefjgjgiolrmsdpbeqvysvaypfwtlsnewpudjzyieqeewze
|
|
||||||
sk-slhumtzsjszyrodyufufmuqyjrqzglrfkvjxdacvquyrctsk
|
|
||||||
sk-uqzbkzzzpmpjeliwgxpugsrmhypxtbuwnmbudtjbsksznffw
|
|
||||||
sk-mimztgfgojmjhxglhuhqkvdritcalbuckihkrwdazatslmli
|
|
||||||
sk-fgsrnkabadokxmcbbmvbnfmrrcrcajjuorwsvquvbxluswbr
|
|
||||||
sk-gcjubjefdwczuhmvtnwvigjtpiyzhcnaulqeudinexwewjrc
|
|
||||||
sk-vxecdntxuawtfwtlikfhwxkfuxvckrljsmpdxcnczknuoivw
|
|
||||||
sk-qhsfufnajkwqmahcpnjhqvaxrwakgdrkoqiyjjhzxhfqmvjw
|
|
||||||
sk-xnivgfdglvrpysavwfosxufodfzpwmskegomzrerlrieroan
|
|
||||||
sk-xciczaweyisbeiyqrjojzkyfqzlcrrekoooxvvlyuhialqwq
|
|
||||||
sk-kmamcuorjnvtuokpupsbtotqzovnbquhxsnrllreignwifvx
|
|
||||||
sk-sgeplkiycdvxjawkhhupbxyzkrqqdvrnmwwgubfjzwidnfac
|
|
||||||
sk-ukkrovjjvaumkokogtndhnocjlcbhmxqieruhrhdrxjeqsrw
|
|
||||||
sk-ewdedkictqtabasbsgkfjxnymxdgsmuqhgjnvvngrcmscrei
|
|
||||||
sk-uuhamqmppytkbisrebsusyyedpoufillrjqhnbauvnyliyrm
|
|
||||||
sk-ncxdshinjahwzogzarpqvaahimlftylkookeokbsmgatlhkp
|
|
||||||
sk-kmkthtkyqxhicwpbvkavdsvixnhkhwnloqvdkbeegiaorfac
|
|
||||||
sk-sgqkdykufwntcakjsulyjttsorpgevnbqemdidvnnokuxlur
|
|
||||||
sk-aahqxkeidoebohilhrkbkbnhwhkkwjdvjulmyjumdxqpcmlt
|
|
||||||
sk-cjgosmmljbtalvifxbxxrdtnngikrqxbnmmyczvwdhvaqrho
|
|
||||||
sk-kjyfvkzfdlufstqwuolwmpiixvnygeyxcplkhtidcnkyuhof
|
|
||||||
sk-ufmttqluqynwofdusyxyiliuclbqjavclepodqpietnxcpdz
|
|
||||||
sk-dtfxfgxtvggrmcuahrbmgakotgabtdjfzfleybeyzzqgtkxe
|
|
||||||
sk-risdpcbojturoktecteyfvuzsisznqvhpwkbmmjiipnebcnx
|
|
||||||
sk-bnkaavqkyfwopsoandsecmehytikhvexiafnzvotiqvaadou
|
|
||||||
sk-sydccxkatazajzoqazlhyhzifuifdfwhyobaoozemeaitcqr
|
|
||||||
sk-mxkhmkttbiedcuvgnbuoruuuohmtvmtcdhxsytcksmtrambr
|
|
||||||
sk-ngsttjqsxwrytzttxlcedemidchicwicmluuopbjnadggjzx
|
|
||||||
sk-nulzzgtpaubyfhnuyaiktwhkznkzjxpppbymvncsfkrgmeif
|
|
||||||
sk-ripjkrltyugoczhaeuxgkqmhtcznyawepvlvyjebezhorahn
|
|
||||||
sk-hjciuefcqhoycgggyewlqmugsqafiefyowwfbzqprzupucpo
|
|
||||||
sk-vtufrtvajondbtrkdhbuxlwvbqgszqazozxzqxotaydhbtvp
|
|
||||||
sk-kfdlgrtqkyprcetrnrychpfevpjjgywwivxabzfqtasgyiix
|
sk-kfdlgrtqkyprcetrnrychpfevpjjgywwivxabzfqtasgyiix
|
||||||
sk-feeqqafnomkhecmdmynthzwzrxtnycmqukruftwqskdkwtnf
|
sk-feeqqafnomkhecmdmynthzwzrxtnycmqukruftwqskdkwtnf
|
||||||
sk-npkvsjgcybufvdunwkphuaonyldjbxzjsafqqedahbtzeoiz
|
sk-npkvsjgcybufvdunwkphuaonyldjbxzjsafqqedahbtzeoiz
|
||||||
@@ -321,144 +131,9 @@ sk-aglzgphsokubceekkeuzoyvgkcbqfycnxaqcgaedgjdvhcxg
|
|||||||
sk-hardkitrxfbychdakzysippwpyjmedmgiagqkgeqvmlafmjf
|
sk-hardkitrxfbychdakzysippwpyjmedmgiagqkgeqvmlafmjf
|
||||||
sk-gsnxjocrbnaaxhzfeibrmdcbsnwfqbofhzndazjkrrvltxzv
|
sk-gsnxjocrbnaaxhzfeibrmdcbsnwfqbofhzndazjkrrvltxzv
|
||||||
sk-ithxlhonxaorsmqdvckipubyntltddtgwptzckokngcpncfm
|
sk-ithxlhonxaorsmqdvckipubyntltddtgwptzckokngcpncfm
|
||||||
sk-rydajrzjgdvygsnzsofzghsmpdeypytcupfzansxqjlxamhe
|
|
||||||
sk-bwdbkrjdrrkvgwrlpcaawdumuvoltvfozuhcjkgtnsnrwlsk
|
|
||||||
sk-pxsdjjeoihoycvqzeqcwxzttakxjjrzwkgbdphwxpwxofsbw
|
|
||||||
sk-qrgxddgzmkgfdgjrxqsbxmtmpeyatwhzvxvmuvvbhfumpvuh
|
|
||||||
sk-niqkplmnplnqxrbeeunicxarawnsoesfdwhubjicyecwnrek
|
|
||||||
sk-kblxwcdacilfvwgpnyjuzyaxijvpxrzxgeeqzlkyvgznnhoa
|
|
||||||
sk-hrcqaueghbzbhafmooboiwuaxqjmgkzhquswmyuxfougnako
|
|
||||||
sk-sylauojlfotekasxmqpkuozbduvpbzjctyepkcbcgkipixse
|
|
||||||
sk-irmklysclnupijhiijsnteyntgdurabgqeqnxghhibticnzb
|
|
||||||
sk-uvofwcdjgoohkhuhevxpjncumzvtlkbllzlsumxhpssqsbsp
|
|
||||||
sk-ltvriahhavqdfbckdcgzjivdicojvekeeoqwwooruokvvpug
|
|
||||||
sk-zamhufoytbudycjydcwbsaktzgbbtpjnyejxeytgvmwcpwlu
|
|
||||||
sk-fqapklszmsagbgpqrnznhvqnwzaoydnqvltjemypsqskotfy
|
|
||||||
sk-pqcrbgbtolzodtramsexmsmfmrhbfjlbjtknizvcbkelbtmo
|
|
||||||
sk-cpgsgbccgchuegzxuqqdmtcvleywvckexawpdwmqacwtkzls
|
|
||||||
sk-hazswunedwvussdqsrgzwihixemitblhdniunrxuejcbrlko
|
|
||||||
sk-piwkgmjimijvecwotaxdeuwavrxkplqijvfxmrhlaovmpgla
|
|
||||||
sk-qemigcpicqqpaqaqvldghhpjdzaplpgvvqlzfwynyliyfrdk
|
|
||||||
sk-mfcufhmoaegeklawzqklpimieknkgpywmilfofcxibjfhqtb
|
|
||||||
sk-yvsaxmfkkeumfghphamfduxyjoskyqnchnsdmgrnharyfvcg
|
|
||||||
sk-vwcauhdrefygiorzvgbgxppzwuechwujqhmalemscexebtvh
|
|
||||||
sk-omajbnulidbutyqfbzrylzpcsotwxmyvkuklbikephfqcnaf
|
|
||||||
sk-ivedlbwcdbeibugizbqgmlqlymjtejcoeoepdwrbxaleydad
|
|
||||||
sk-yagbbopbkxikizbamlnysdsrdvachkzatmhtgremvnzepzrc
|
|
||||||
sk-yislighgbsvmqocbevunpopcnapjxtnfzwhelycodkquyoxr
|
|
||||||
sk-lufzzjathagzjdxzvywxozsgjahqbjptykbxdrvwrucqevtq
|
|
||||||
sk-hphgvkzxifkekadopqzcaonwdaryfrgmtpwfqhbbdflbhsvj
|
|
||||||
sk-itghsfwznrvvqmimujewgocmasllvebczhfvaepbdhrareol
|
|
||||||
sk-mbdlewsjoqyvwfeugqvuwlkhoysdvhhkpyiphhrwxxhsxqja
|
|
||||||
sk-rjnidikzzmvzlysihmwjzyzztgrfvqyohdplxbwpuvolctal
|
|
||||||
sk-czxrlphpexsvxttfhgfpnmpdwnjdlecbuuyqikjnrlrocosj
|
|
||||||
sk-eaaurjpunuzvolyxjosnjgudeqfikjjrlzdfcaeagniinmfz
|
|
||||||
sk-srvewvzghpgjjjrssykgdhibtrfyqakyozmhwoypsbgkjmiv
|
|
||||||
sk-dbqkfrbdwcnpphwbmlbatutqoknemwrcpkdahwtpgrybhdjb
|
|
||||||
sk-mfwgnrhjlxqboxocbwgdemejjkrpizzompauozwbkynsimvd
|
|
||||||
sk-vefkbebsxxiytlvgkqzzxfsdfdsxyoucfwyxyjvawpshjafs
|
|
||||||
sk-rmdrphiuddwlyycyyxciuvvfypjgiqkaffxgnbblpjoramox
|
|
||||||
sk-pvtelvkokqxsoqdmcmcybczobbyffqyiabfzlporvstrfbnl
|
|
||||||
sk-wxjczvduvkezwamrmiqiqrdgsowjwmdrweihptstbydnkiyk
|
|
||||||
sk-clnokwxbloklzhftymuoktkqcqwtbukteiswchfwygaemzxd
|
|
||||||
sk-upvrfbkhjmeplwniscrsuhinvyhfjqdkxoxlwsnitjwiewzx
|
|
||||||
sk-gmdwcjemnhtjefbtmpnljtsdepfgmrpdlgedamuxzcwewroa
|
|
||||||
sk-kullfthpcemysaxmkrkktdosvxtzjkhhbfkpksvnqiviymkm
|
|
||||||
sk-uantaculkjoueqmobedmjxnpufeppypxpqaazuzmloolyslb
|
|
||||||
sk-uzqnmcllemnuwmioqcadooklwohsxuoijspmnzayuabdxgpo
|
|
||||||
sk-velznojgjarqkmxuagajuizzxgshqvwysofujvnznwfkfmxo
|
|
||||||
sk-pejyqvncscswbcjjmwbyekunlovppjselrkxwtwxlnrqoeph
|
|
||||||
sk-kdoxtxvhjdwcqelwebbaoqscifaycxfytjojukqxqrbiiinx
|
|
||||||
sk-mlrrttakzumknywzgxeyrrizfhqtlbbuaojiirsjsbdbvpqw
|
|
||||||
sk-jidotcylhkuesnlmedoojupbcprqokvvjsbelkmpcrapeblo
|
|
||||||
sk-bghjoljwoqowwaytxdygrmokjwbucpodwexfbqogfdobepjp
|
|
||||||
sk-izfufzgvdvhffjwrgswfsnvwjehgswnuqndzpsdtqaukjqik
|
|
||||||
sk-givwcjohnwowctfspxfzthnolplyqsncqpdndhnopceizncb
|
|
||||||
sk-enooithbjoweqkloaoedhyuqmbehfthyehjqrudnijyxpwif
|
|
||||||
sk-dxfksbhlibzzmvbjmhaohtmuhspgpnlfasauhmsybgzmvrqx
|
|
||||||
sk-egsiqsepcweqgvcwexpcdrpvwrxmobvwixqccglcpesvmxed
|
sk-egsiqsepcweqgvcwexpcdrpvwrxmobvwixqccglcpesvmxed
|
||||||
sk-spnqvuyeyqabuwilklpfhswgxlamzqlluwohhwcedrdcjoos
|
|
||||||
sk-myzjstorsfjlwxmgskzkkyvubhhziukyavdfbkocvgljmlyn
|
sk-myzjstorsfjlwxmgskzkkyvubhhziukyavdfbkocvgljmlyn
|
||||||
sk-gteiidrbqmgdxzvpjuiyeqyvipmxzqtoiambwldwgjsfmtdd
|
sk-gteiidrbqmgdxzvpjuiyeqyvipmxzqtoiambwldwgjsfmtdd
|
||||||
sk-wiewtlkdlipdbjjrnsurhzmckxmlhmmqacjmpsvfwehlrzjc
|
|
||||||
sk-crfrhbolnaesyidqqhugecipyxaiurydhhxpirnbtdpuqyje
|
|
||||||
sk-xfwommmcrvvaicqxickvqzjpsslbgwltgkgcnjpywmbgkfux
|
|
||||||
sk-vfceyfpfpefmdpwiwxnembwkxezfjvrulxiyckhunfszddcr
|
|
||||||
sk-hragaaswdncjkxjamcsyetdzydvdpmvqfnlkncctrdcgqyts
|
|
||||||
sk-rteqaliitwaplfjrjnfzissfopogiwpkxbllorgkllsflbyi
|
|
||||||
sk-jrhvkbyrjinghhlrbvpuzlliujmwlytowvvbehyqzpgeqgpd
|
|
||||||
sk-povvbewiqabsltipzbrsmogciuymrrcjdrsjajfonzugxlmb
|
|
||||||
sk-rcgpvynyysinbnwreoqtnhzoyjajgkfpdkbfalfxycluzmyv
|
|
||||||
sk-edsfyylfdstryliesizynqbsghoctythcxhtqycltnovdqyf
|
|
||||||
sk-ruqwiacoyoaovpbgxgzjugpucdqweehmlxqfmzwubyhimuhk
|
|
||||||
sk-xiluiivjbsuelibcbdjdkdcexhveukdonvwdanlkruifhksn
|
|
||||||
sk-nusnonfvpndlozqewrbmaafzijdzzhwevbbeejqsmimktqga
|
|
||||||
sk-zcjwylafnvwccbcsofvwfwnzwekivdsdcjkhxztezrebgfjg
|
|
||||||
sk-nafijjlguxcresetwwkeblvhtzhzvdtlzdcmtgnznrsbaysp
|
|
||||||
sk-gjzccgpgmtoxoyykdtytmiccyridjqavtmttbpkhuhxchcuz
|
|
||||||
sk-aelhfuriagxjzdkhvraymqmzrskcgymmqzieszbbkhggxzmb
|
|
||||||
sk-bwnaaxggxrcczsjahmpburjrogxvuslftpbiebukbdoojkav
|
|
||||||
sk-hnhmwszlgyfsveokvjxxyqrmudwkjlynaqgroyvnfkvtfoen
|
|
||||||
sk-gvhdltdldwlgtafcvkudjuaxccwquwprdmgdxoyhzxqfrnlt
|
|
||||||
sk-vceekcyzeiehthcwrcsojxgjeicjlmaklwurfhsmindmaoze
|
|
||||||
sk-gafvawwsdlidlcdstobbzwbkcmyzftlknimpfsksdobyhfdd
|
|
||||||
sk-mlxznzvvpjyuzutevojnqhufvmviixpxxnoozgawqpmigipf
|
|
||||||
sk-yowbggizjclintdzdvrupvukchktdscwptckwzqfjktyhsqu
|
|
||||||
sk-itjkzhwbcijipsbfemcomdxrfjjlsawmafsokvqehftebeax
|
|
||||||
sk-ftgafevejonwkzhsowshcajtzhcrzkgbygqeywpjdypwxcee
|
|
||||||
sk-ydyesipjfeeslxychvdvhyhjnsjwrrxwzcyvxorgecgobdxt
|
|
||||||
sk-ajlveigvqbcpqhhiegwoeufwzfmvhxgezvhoxbvzqfptexhk
|
|
||||||
sk-vumrmneszkiykdqdmktemlyfawpwabodqbaqgwwhjwlemrus
|
|
||||||
sk-cndsxoajozwuchkkjkestsykkkifnygaeqznktcaazzvyexk
|
|
||||||
sk-swzvvwztfmbbwkobsrjozbncrkyozyjppysryskehrmslmqr
|
|
||||||
sk-ofzoljaggdhqdehmllhspluecqlqwvpckynvgrcucnxgnetz
|
|
||||||
sk-ilxpmzzdqvehuqengiflehjbjksjuzxxdkfblvbbotqnxcvg
|
|
||||||
sk-vffajjwdonfgyobqlzsjvsmfgppmtozitlyuiklzddmwjasp
|
|
||||||
sk-hlnlundabnwhlxnevvedrdgwgkixnwxxfcszkkmiyiogqtyh
|
|
||||||
sk-zzinlifzyvwlbrcqolobsvpiuafofujmdempirjbblyiobrq
|
|
||||||
sk-mhatttcuajcdubbhjszzuiugwahihvkhujqqdhuipypimppx
|
|
||||||
sk-qcdnlpjjxsttqvfgrtdeaaemuxxiweodfpjaidaocxdjfbfi
|
|
||||||
sk-jwhjcsjhpjqyynrlffbzchqynooegtwxoyweivpehomcjrpz
|
|
||||||
sk-jwczimaoevkonzfczjjflpwvdckejejmxcpqztzwfzpvtgvp
|
|
||||||
sk-jrrfbllfheymstqibwllzxmyvednewzhfemrpvskqvpxygob
|
|
||||||
sk-istkpwkpczjbeoyeziajiefqvqhlxcececbskgyvrmomnmmk
|
|
||||||
sk-zhpndzqwwngxmfqmyablhxfkpfzmriaiavjlqvjeggysgcvm
|
|
||||||
sk-bizlrntmuhimynjyojsmyvsehgluwjqoaayvdrvtpeoiioul
|
|
||||||
sk-nxoemksjhekrnbpcufgldxwtyzwnzlbqpuixleygnyrqzwfg
|
|
||||||
sk-hydktomuqlcmgqtrojeifpifavsbczuqdtcejwfnjvocsaqw
|
|
||||||
sk-ueimfegidcpvitpkbmioqqjrsdlzcckpdhiowjxruvcfoftr
|
|
||||||
sk-isebnaohrsneefwpmaakltyekhsdbvuyoicyjrgmhhviyctp
|
|
||||||
sk-lozddkkffmdttutlewxfsibvboszsvizcsvxxitpannvyhjp
|
|
||||||
sk-dypsaykmiojssgflecaysuoffafksgjiiopkrsapvngtzcpq
|
|
||||||
sk-vphqbyxqhmddhpgutalrismdloiclevinqettinmamivadrl
|
|
||||||
sk-xueiltcvrqdczoehpylgppajnrxnfdwbdomddezgytyqgybc
|
|
||||||
sk-kujnotimzgtjohdvihicyxrowoujlyydiyekssnzbeatoeam
|
|
||||||
sk-rnmlxgqtvugiurgbmleubbywaldtdjpnmujlujuwruhnrscj
|
|
||||||
sk-htudvczugpxxwumrixzmcsuuljpulyrycnvrbgyypgbhzlom
|
|
||||||
sk-rmoauqmsvseihtjnmonzmhtjxqvehsoxosetgtzaeekfphsq
|
|
||||||
sk-jwdjyicxrwarllnbxelmlpxyhlngxlwyblausbqzjybhfsgg
|
|
||||||
sk-zjqaxewvbtmzicyudgwtouwpmaumqjxgfwvnixbebphzvkfw
|
|
||||||
sk-btopemwluceyrvqrexdduzgiracedkipkjpvyqcekibvveld
|
|
||||||
sk-mlmsyxbhulksdphymbunrtyvlfaulloafpdotspgujspvzhs
|
|
||||||
sk-eoflcedwmujvkglnusptdvpejijmzqkgmzkugjfcowybyros
|
|
||||||
sk-duudnurisfjbbiaijzvgjqxvmeqsaobyjonimqqubasffgtc
|
|
||||||
sk-mrnqwyygtwfzdxwfxvdykciltrwremhhxkdwasvvyuyqlohe
|
|
||||||
sk-nsojpxqdiiiwhudcbhieqcynwlqeqjcevdilpllnzexswdlf
|
|
||||||
sk-fnaqnadhaxdioyvxarzessxcyonfusgjbvadmgofbbwlbfro
|
|
||||||
sk-azihdytaexnxxhgklpeagvqrspjpuoirxkcpnmgwdpkhjocf
|
|
||||||
sk-nsgabwkipznwkuhsipwcjmfrwzqrnsqsqbegmnsaderncbum
|
|
||||||
sk-fnxklmkrxxchctgugherfgutqonyrhnbhmuywiwjmsgzripx
|
|
||||||
sk-zdcppgntgaongebtgxumkjwlshnhmuwxbqeuysajxisonyze
|
|
||||||
sk-iwixpdpuuovovlttpsxifhqbkgokexhaucwcoswtgrtkhsdj
|
|
||||||
sk-jqlgopzfvpduihyqfxsbkbphfvyxtafhddyovshoumdhqtyx
|
|
||||||
sk-gpvqdlyhppiezulgjeefurcszcgbkozkxzcismhzncjwiwws
|
|
||||||
sk-owttiioxyexktqejevajanmjcyfozbyedqmxpgzpwapkuqit
|
|
||||||
sk-gepnrwtefcnzbpbtsvsxwvmadrfcdcxxdcymhkrvjwkagjps
|
|
||||||
sk-xybhogvmrvlkmandeeetmxllxprxbvumqchpyhrcsspejkge
|
|
||||||
sk-wnljauedihizucllmpouarpkmdedwrhammttzbmsibeqwtlb
|
|
||||||
sk-sincovmmvdslodfqdnwnlnzqowpdrmsjljnloqlobmqibkqm
|
|
||||||
sk-zgfrqvtwapwbvmrbcmzfzesjzoiexmiuewqkbsxfrzvsarpr
|
|
||||||
sk-veydusuesejnqgupmvnamwdoulqpquuzyihqysmxyzbgseom
|
|
||||||
sk-mtpgzluddgftxbujteqpuviywfqgyncpfqjgjqgjpcicgyck
|
sk-mtpgzluddgftxbujteqpuviywfqgyncpfqjgjqgjpcicgyck
|
||||||
sk-ftjkknqxfmkkkmpfdsgfqesjeeqmmmbhivdirifbhvfcepwt
|
sk-ftjkknqxfmkkkmpfdsgfqesjeeqmmmbhivdirifbhvfcepwt
|
||||||
sk-ovckcrghcmczguqdrerkxkkkscyouizksvmpsjumgiewapfd
|
sk-ovckcrghcmczguqdrerkxkkkscyouizksvmpsjumgiewapfd
|
||||||
@@ -570,60 +245,4 @@ sk-hnsuooinmmkptuycbhijyqeqqtdptipriituttztofvsyets
|
|||||||
sk-qcufjgduzuxahwuyuxhohdlxlwzspgjervcqavejjjsqnxuk
|
sk-qcufjgduzuxahwuyuxhohdlxlwzspgjervcqavejjjsqnxuk
|
||||||
sk-gcrooquaphfpgkpqzulmknyelauhntgpkahdndfmghwspfzl
|
sk-gcrooquaphfpgkpqzulmknyelauhntgpkahdndfmghwspfzl
|
||||||
sk-ikpyfcknhbygvzzufzyltgproktmjlwrkcpaulzdbwitcnsd
|
sk-ikpyfcknhbygvzzufzyltgproktmjlwrkcpaulzdbwitcnsd
|
||||||
sk-ojglzjacacbqyxlepculoodgwawycatixfjygdghvwoedxqn
|
sk-ojglzjacacbqyxlepculoodgwawycatixfjygdghvwoedxqn
|
||||||
sk-hzwilvkmjeonxyqkefvmyntslodwvzshbjtqcnyzyugamlwt
|
|
||||||
sk-ykqumucxvezvtklxrzcdxtrcjxezdvorccjxeetpvwjxasig
|
|
||||||
sk-ymvixkeoxxmslmvyjtttaszsqurhggtrljmnjpypjpvlxgie
|
|
||||||
sk-gdxyanoxmzuzoqwrjuzqltwcfrkcfjffjysnobzwsxknzbas
|
|
||||||
sk-uksvsffcuwuqycpcgsxoagytwisvyizvqissvzxfbwepmoxt
|
|
||||||
sk-ecjdubyxsxvorouvsluugguckqyzagjoubzlpfgitdlpxnjr
|
|
||||||
sk-vznvfvcccqzpazawlhqenjcyyhcgjmewwqlcumpoouftbdkd
|
|
||||||
sk-aocgyriobhfzmbdsjeycjovlabdyoobuzvsicoqvqgjufeyq
|
|
||||||
sk-fftwoemnmkaigzvtsdvcovxnsrqaivpghbqlhhzwfupgqixd
|
|
||||||
sk-pdimahhuaciuytjcdrrqepnvekcwcajjaccfnkbtywksgrzj
|
|
||||||
sk-gpubyipyxilqncwsbjtifyvihytttndwnnzwfpacezogorkt
|
|
||||||
sk-vyohlulojrccftrlttbgkhtssrsevjwvbyprkxxozztcagfl
|
|
||||||
sk-evkreookljfrkivshdtukmvdmigfsrkbxqlvpplmmxrqdeeq
|
|
||||||
sk-mfhqlomnnimfyalspfgrifvljcyqibqecaogvcmtpfdinvww
|
|
||||||
sk-mgrgagopmvspegvtlxcczewuyacnclzkmidfhjjrlftigygb
|
|
||||||
sk-taokfbrmnwpuzfvwmjkqkxmbtcsrhiosbvxhlzjvtkfyyzew
|
|
||||||
sk-arllcykeoxnaayvrnlwdgefptzbwdrrpfswzuellaxfpwiyv
|
|
||||||
sk-jswurhdubigubwaywtmjwesgdmeublcfgyxkqhjfqolhjkxw
|
|
||||||
sk-owufbwamtvnyfcmnokoletwfiqcolbisusgjxaqsucjddgaj
|
|
||||||
sk-oavuxluuhgarupuonjdajtiyqzfaappsnepvrwsaecoqytps
|
|
||||||
sk-abjzukurnfvycmobsrlupngiuoihydxfolguwdkiuousjrog
|
|
||||||
sk-areauhdxiwjqvtvkjzghvhzlkctawgxaqykbzrtcpzkeemzh
|
|
||||||
sk-hvirdspguzuxeeyqtzozteuqpnlgdmvkhdbcseeiwvvvjwpc
|
|
||||||
sk-bxgtnigshpgbzrxnpirnknmuwrvmmrdsdxgrsxqxhklkbiag
|
|
||||||
sk-qsrvvcyyvetacumkffpifzvpprvueiuyxcymjhuoywulyckx
|
|
||||||
sk-chtrfzhvlwvkcxhmlwgxylhhbzoghmlhfqnrkrjjbxougtlm
|
|
||||||
sk-kaorsolyqbqktpjyxmvsothfkpklauhwvjoewdnxvjcdtmiu
|
|
||||||
sk-yyhihitvelctqipvcygnqgzadifvssmowwauroyydvzxqwxz
|
|
||||||
sk-zuwnyslmzstjwcimfdfdhxhqxdyoyaspfzlqieqklptyrqod
|
|
||||||
sk-mrdqhmnwzubfiegupgjcdcrxwscwyrkqburcovprpcsyuikd
|
|
||||||
sk-wvggprumleznebcbaycjpmpxvnxrhkcibfohwavxxdjtbkah
|
|
||||||
sk-bzknzzwnftgzgfhddwtbrethrykvdcprldmzkimgwvactluv
|
|
||||||
sk-kayozpvefrtnyjgzlxvfupxcvpygqpvrwzfjofexkoohcyfj
|
|
||||||
sk-telmszepewpsgvtgwmtjuhevatqbigjazjpcqowtxctjcfqt
|
|
||||||
sk-jzicpgrljkzzvtpnonkdtlesztuzxdblflpmvazghnawbrbp
|
|
||||||
sk-idsyjybbhuwtkznjqwehfkirotujqlecqmxvouthbgblkopq
|
|
||||||
sk-guriavtreoqrodvtqtoxxfgibrpvucnvifbviizkmjgfnojs
|
|
||||||
sk-oeakuguoobfgyiajtmnuucnhvdplmwmleoskybmwaynxxjjc
|
|
||||||
sk-jlrvbzmamxafavnaqgslolvqqaroamzsczezzckacxfufuut
|
|
||||||
sk-belofvolykizlwcheykdwlrvxtmkzrvxmwunqfepstbzsano
|
|
||||||
sk-ngkqukytxzjmgxctwxzahtkdrtiokkrrhjzjptcmedvygijw
|
|
||||||
sk-lbsbwiwdfxfdomoorrepdswizrxicjohitaxgnwmckpidgqp
|
|
||||||
sk-slugbeutaykvjvzbdeuxzjtxdmhtbgqlmpxyxlagomcrmfex
|
|
||||||
sk-zzstwfsdicghbutiruvfqqmssicedkwpynkftlpdnbknkeac
|
|
||||||
sk-wxcefyxwviuxsdrwvbojvahchrhyzvplbqvckhwigneqweal
|
|
||||||
sk-petosxdqphybeaexrmavbenovqshbgyqzigvbdpsmlgwamrf
|
|
||||||
sk-akzqrlcbkvyoycpsypipswwxdkyyyojpdqaaauomugpeqmme
|
|
||||||
sk-cggdeyuwhrmmwyvrxyxgjqkgflpfzxuscizyorpntqyfeekm
|
|
||||||
sk-akjolruzfygkrozyqsxzmwbxspbihdxqffbxrvlxmmnugxid
|
|
||||||
sk-tfhblzxfibbjkroxcyfjmvjtttnkrvcrqdhvgqafmbiwfrmp
|
|
||||||
sk-kyqkmusnnecseiddehvpaicymdppwtnztwsllsjgbeubtngx
|
|
||||||
sk-itzdkmmqvrkqsczfellwdsmqusvncdnhfupnyplcexzgjqys
|
|
||||||
sk-smlwnqivoryuqbgduvcawnlznwakpparjlcjpbbartgljtip
|
|
||||||
sk-rrrdvcmvyqmkfwavimkdzsrycshwwikxklfcbsqsjthbduod
|
|
||||||
sk-jwiuszauztewftwmcxtfjmzonvjujvemulekyxtdkjwdgmah
|
|
||||||
sk-cwjrcfssqcakkpwypldjocdvxhrjdbtmsstuuviixcsfmkoy
|
|
||||||
+926
-26496
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@@ -1,187 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
File: deduplicate_json.py
|
|
||||||
Description: 对指定JSON文件进行去重并重新保存
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
from collections import defaultdict
|
|
||||||
from tqdm import tqdm
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from rag2_0.tool.ModelTool import OpenAiLLM
|
|
||||||
from langchain.output_parsers import PydanticOutputParser
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
from rag2_0.intent_recognition.DataModels import Term
|
|
||||||
|
|
||||||
# 加载环境变量
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
|
|
||||||
class JsonDeduplicator:
|
|
||||||
"""JSON文件去重类"""
|
|
||||||
|
|
||||||
def __init__(self, input_path=None, output_path=None, key_field="name", max_workers=3):
|
|
||||||
"""初始化JSON去重器
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_path: 输入JSON文件路径
|
|
||||||
output_path: 去重后的输出文件路径
|
|
||||||
key_field: 用于去重的键字段名
|
|
||||||
max_workers: 线程池最大工作线程数
|
|
||||||
"""
|
|
||||||
self.INPUT_PATH = input_path
|
|
||||||
self.OUTPUT_PATH = output_path or input_path.replace('.json', '_deduplicated.json')
|
|
||||||
self.KEY_FIELD = key_field
|
|
||||||
self.MAX_WORKERS = max_workers
|
|
||||||
self.item_parser = PydanticOutputParser(pydantic_object=Term)
|
|
||||||
self.MERGE_PROMPT = '''
|
|
||||||
请将以下多个描述相同名词"{name}"的条目合并为一个，合并时请：
|
|
||||||
- 同义词（synonymous）去重合并
|
|
||||||
- 描述（description）合并为更完整、简明的描述
|
|
||||||
- 保持输出格式为：
|
|
||||||
{output_format}
|
|
||||||
原始条目：
|
|
||||||
{items}
|
|
||||||
'''
|
|
||||||
# 配置LLM
|
|
||||||
model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
|
|
||||||
api_key = os.getenv("OPENAI_API_KEY")
|
|
||||||
base_url = os.getenv("OPENAI_API_BASE")
|
|
||||||
llm_params = {"temperature": 0.3, "model": model_name}
|
|
||||||
if api_key:
|
|
||||||
llm_params["api_key"] = api_key
|
|
||||||
if base_url:
|
|
||||||
llm_params["base_url"] = base_url
|
|
||||||
self.llm = OpenAiLLM(**llm_params)
|
|
||||||
|
|
||||||
# 配置日志
|
|
||||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
||||||
|
|
||||||
def load_json_data(self):
|
|
||||||
"""读取JSON文件"""
|
|
||||||
try:
|
|
||||||
with open(self.INPUT_PATH, 'r', encoding='utf-8') as f:
|
|
||||||
data = json.load(f)
|
|
||||||
logging.info(f"从{self.INPUT_PATH}加载了{len(data)}条记录")
|
|
||||||
return data
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"读取{self.INPUT_PATH}失败: {e}", exc_info=True)
|
|
||||||
return []
|
|
||||||
|
|
||||||
def group_items_by_key(self, items):
|
|
||||||
"""按指定键字段聚合项目"""
|
|
||||||
key_to_items = defaultdict(list)
|
|
||||||
for item in items:
|
|
||||||
key = item.get(self.KEY_FIELD, '').strip()
|
|
||||||
if key:
|
|
||||||
key_to_items[key].append(item)
|
|
||||||
return key_to_items
|
|
||||||
|
|
||||||
def merge_items_with_llm(self, key, item_list):
|
|
||||||
"""调用LLM合并具有相同键的项目，失败最多重试三次"""
|
|
||||||
items = json.dumps(item_list, ensure_ascii=False)
|
|
||||||
prompt = self.MERGE_PROMPT.format(
|
|
||||||
name=key,
|
|
||||||
items=items,
|
|
||||||
output_format=self.item_parser.get_format_instructions()
|
|
||||||
)
|
|
||||||
|
|
||||||
max_retries = 3
|
|
||||||
for attempt in range(1, max_retries + 1):
|
|
||||||
try:
|
|
||||||
response = self.llm.invoke(prompt, False)
|
|
||||||
parsed_output = self.item_parser.parse(response.content)
|
|
||||||
return {"name": parsed_output.name, "synonymous": parsed_output.synonymous, "description": parsed_output.description}
|
|
||||||
except Exception as e:
|
|
||||||
if attempt == max_retries:
|
|
||||||
logging.warning(f"解析LLM合并结果失败: {e}")
|
|
||||||
return None
|
|
||||||
else:
|
|
||||||
import time
|
|
||||||
time.sleep(5*attempt)
|
|
||||||
|
|
||||||
def process_item(self, key_items_tuple):
|
|
||||||
"""处理单个键值对应的项目，用于线程池并行处理"""
|
|
||||||
key, item_list = key_items_tuple
|
|
||||||
try:
|
|
||||||
if len(item_list) == 1:
|
|
||||||
return item_list[0]
|
|
||||||
|
|
||||||
merged = self.merge_items_with_llm(key, item_list)
|
|
||||||
if merged:
|
|
||||||
return merged
|
|
||||||
else:
|
|
||||||
# 如果合并失败，返回第一个项目
|
|
||||||
return item_list[0]
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"处理键 {key} 时出错: {e}", exc_info=True)
|
|
||||||
return item_list[0]
|
|
||||||
|
|
||||||
def deduplicate(self):
|
|
||||||
"""去重所有项目的入口方法"""
|
|
||||||
# 1. 读取JSON数据
|
|
||||||
all_items = self.load_json_data()
|
|
||||||
if not all_items:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# 2. 按键字段聚合
|
|
||||||
key_to_items = self.group_items_by_key(all_items)
|
|
||||||
logging.info(f"共{len(key_to_items)}个唯一键")
|
|
||||||
|
|
||||||
# 3. 使用线程池并行处理
|
|
||||||
deduplicated_items = []
|
|
||||||
items_to_process = []
|
|
||||||
|
|
||||||
# 先处理只有一个项目的键（不需要合并）
|
|
||||||
for key, item_list in key_to_items.items():
|
|
||||||
if len(item_list) == 1:
|
|
||||||
deduplicated_items.append(item_list[0])
|
|
||||||
else:
|
|
||||||
items_to_process.append((key, item_list))
|
|
||||||
|
|
||||||
logging.info(f"共{len(deduplicated_items)}个单一项目，{len(items_to_process)}个需要合并的项目")
|
|
||||||
|
|
||||||
# 只对需要合并的项目使用线程池处理
|
|
||||||
if items_to_process:
|
|
||||||
with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
|
|
||||||
# 使用tqdm显示进度
|
|
||||||
for result in tqdm(executor.map(self.process_item, items_to_process), total=len(items_to_process)):
|
|
||||||
deduplicated_items.append(result)
|
|
||||||
|
|
||||||
# 4. 保存去重结果
|
|
||||||
os.makedirs(os.path.dirname(self.OUTPUT_PATH), exist_ok=True)
|
|
||||||
with open(self.OUTPUT_PATH, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(deduplicated_items, f, ensure_ascii=False, indent=2)
|
|
||||||
logging.info(f"去重后结果已保存到: {self.OUTPUT_PATH}")
|
|
||||||
|
|
||||||
return deduplicated_items
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""主函数，解析命令行参数并执行去重"""
|
|
||||||
parser = argparse.ArgumentParser(description='对JSON文件进行去重')
|
|
||||||
input_path = 'data/wiki_extracted_nouns/技改检修计价通_nouns.json'
|
|
||||||
|
|
||||||
parser.add_argument('-i', '--input',default=input_path, help='输入JSON文件路径')
|
|
||||||
parser.add_argument('-o', '--output', help='输出JSON文件路径')
|
|
||||||
parser.add_argument('-k', '--key', default='name', help='用于去重的键字段名，默认为"name"')
|
|
||||||
parser.add_argument('-w', '--workers', type=int, default=30, help='线程池最大工作线程数，默认为2')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
deduplicator = JsonDeduplicator(
|
|
||||||
input_path=args.input,
|
|
||||||
output_path=args.output,
|
|
||||||
key_field=args.key,
|
|
||||||
max_workers=args.workers
|
|
||||||
)
|
|
||||||
deduplicator.deduplicate()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
logging.getLogger('httpx').setLevel(logging.WARNING)
|
|
||||||
logging.getLogger('openai').setLevel(logging.WARNING)
|
|
||||||
main()
|
|
||||||
@@ -15,6 +15,8 @@ import traceback
|
|||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
import glob
|
||||||
|
import shutil
|
||||||
|
|
||||||
# 将项目根目录添加到Python路径
|
# 将项目根目录添加到Python路径
|
||||||
sys.path.append(os.getcwd())
|
sys.path.append(os.getcwd())
|
||||||
@@ -84,13 +86,6 @@ class IsComplaint(BaseModel):
|
|||||||
dissatisfaction_reasoning: str = Field(description="抱怨原因")
|
dissatisfaction_reasoning: str = Field(description="抱怨原因")
|
||||||
is_complaint: bool = Field(description="是否明确/暗示将进行投诉")
|
is_complaint: bool = Field(description="是否明确/暗示将进行投诉")
|
||||||
|
|
||||||
class ProductNameAndModuleName(BaseModel):
|
|
||||||
product_name: str = Field(description="产品名称")
|
|
||||||
module_name: str = Field(description="模块名称")
|
|
||||||
|
|
||||||
class ProductLine(BaseModel):
|
|
||||||
product_line: str = Field(description="产品线")
|
|
||||||
|
|
||||||
# ================ 工具函数 ================
|
# ================ 工具函数 ================
|
||||||
def retry_llm_call(max_retries=3, delay=2):
|
def retry_llm_call(max_retries=3, delay=2):
|
||||||
"""
|
"""
|
||||||
@@ -138,8 +133,6 @@ class DialogueToWorkorder:
|
|||||||
self.user_question_and_solution_list_parser = PydanticOutputParser(pydantic_object=UserQuestionAndSolutionList)
|
self.user_question_and_solution_list_parser = PydanticOutputParser(pydantic_object=UserQuestionAndSolutionList)
|
||||||
self.question_type_parser = PydanticOutputParser(pydantic_object=QuestionType)
|
self.question_type_parser = PydanticOutputParser(pydantic_object=QuestionType)
|
||||||
self.is_complaint_parser = PydanticOutputParser(pydantic_object=IsComplaint)
|
self.is_complaint_parser = PydanticOutputParser(pydantic_object=IsComplaint)
|
||||||
self.product_name_and_module_name_parser = PydanticOutputParser(pydantic_object=ProductNameAndModuleName)
|
|
||||||
self.product_line_parser = PydanticOutputParser(pydantic_object=ProductLine)
|
|
||||||
# 初始化LLM模型
|
# 初始化LLM模型
|
||||||
self.llm_params = llm_params or {
|
self.llm_params = llm_params or {
|
||||||
"temperature": 0.2,
|
"temperature": 0.2,
|
||||||
@@ -158,6 +151,10 @@ class DialogueToWorkorder:
|
|||||||
# "timeout": httpx.Timeout(600.0)
|
# "timeout": httpx.Timeout(600.0)
|
||||||
# }
|
# }
|
||||||
self.llm = self._get_llm_instance()
|
self.llm = self._get_llm_instance()
|
||||||
|
|
||||||
|
# 创建工单JSON文件目录
|
||||||
|
self.workorder_json_dir = "data/temp_workorder_json"
|
||||||
|
os.makedirs(self.workorder_json_dir, exist_ok=True)
|
||||||
|
|
||||||
def _get_llm_instance(self):
|
def _get_llm_instance(self):
|
||||||
"""获取LLM实例"""
|
"""获取LLM实例"""
|
||||||
@@ -483,6 +480,66 @@ class DialogueToWorkorder:
|
|||||||
is_complaint.dissatisfaction_reasoning,
|
is_complaint.dissatisfaction_reasoning,
|
||||||
is_complaint.is_complaint)
|
is_complaint.is_complaint)
|
||||||
|
|
||||||
|
def save_conversation_to_json(self, conversation_id, workorder_list):
|
||||||
|
"""
|
||||||
|
将会话处理结果保存为JSON文件
|
||||||
|
|
||||||
|
参数:
|
||||||
|
conversation_id: 会话ID
|
||||||
|
workorder_list: 工单列表
|
||||||
|
"""
|
||||||
|
# 确保目录存在
|
||||||
|
os.makedirs(self.workorder_json_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 构建文件路径
|
||||||
|
file_path = os.path.join(self.workorder_json_dir, f"{conversation_id}.json")
|
||||||
|
|
||||||
|
# 将工单列表转换为可序列化的字典列表
|
||||||
|
serializable_workorder_list = []
|
||||||
|
for workorder in workorder_list:
|
||||||
|
# 处理datetime对象
|
||||||
|
serializable_workorder = {}
|
||||||
|
for key, value in workorder.items():
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
serializable_workorder[key] = value.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
else:
|
||||||
|
serializable_workorder[key] = value
|
||||||
|
serializable_workorder_list.append(serializable_workorder)
|
||||||
|
|
||||||
|
# 保存为JSON文件
|
||||||
|
with open(file_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(serializable_workorder_list, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
logger.info(f"会话ID: {conversation_id} 的处理结果已保存到 {file_path}")
|
||||||
|
|
||||||
|
def load_conversation_from_json(self, conversation_id):
|
||||||
|
"""
|
||||||
|
从JSON文件加载会话处理结果
|
||||||
|
|
||||||
|
参数:
|
||||||
|
conversation_id: 会话ID
|
||||||
|
|
||||||
|
返回:
|
||||||
|
工单列表，如果文件不存在则返回None
|
||||||
|
"""
|
||||||
|
# 构建文件路径
|
||||||
|
file_path = os.path.join(self.workorder_json_dir, f"{conversation_id}.json")
|
||||||
|
|
||||||
|
# 检查文件是否存在
|
||||||
|
if not os.path.exists(file_path):
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 从JSON文件加载工单列表
|
||||||
|
try:
|
||||||
|
with open(file_path, 'r', encoding='utf-8') as f:
|
||||||
|
workorder_list = json.load(f)
|
||||||
|
|
||||||
|
logger.info(f"已从 {file_path} 加载会话ID: {conversation_id} 的处理结果")
|
||||||
|
return workorder_list
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"加载会话ID: {conversation_id} 的处理结果时发生错误: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
def process_conversation(self, conversation_id, conversation_rows):
|
def process_conversation(self, conversation_id, conversation_rows):
|
||||||
"""处理单个会话的函数，用于多线程并发"""
|
"""处理单个会话的函数，用于多线程并发"""
|
||||||
# if conversation_id!="b157aa91-3acb-11f0-a191-4fb224ef4b40":
|
# if conversation_id!="b157aa91-3acb-11f0-a191-4fb224ef4b40":
|
||||||
@@ -534,13 +591,16 @@ class DialogueToWorkorder:
|
|||||||
# 将工单添加到列表中
|
# 将工单添加到列表中
|
||||||
workorder_list.append(workorder_dict)
|
workorder_list.append(workorder_dict)
|
||||||
|
|
||||||
|
# 将处理结果保存为JSON文件
|
||||||
|
self.save_conversation_to_json(conversation_id, workorder_list)
|
||||||
|
|
||||||
return workorder_list
|
return workorder_list
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {e}")
|
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def analyze_conversation_data(self, conversation_excel_path, max_workers=10, start_date=None, end_date=None):
|
def analyze_conversation_data(self, conversation_excel_path, max_workers=10, start_date=None, end_date=None):
|
||||||
"""分析会话数据主流程，使用多线程并发处理"""
|
"""分析会话数据主流程，使用多线程并发处理，支持失败重试和JSON合并"""
|
||||||
# 读取Excel文件
|
# 读取Excel文件
|
||||||
df = pd.read_excel(conversation_excel_path)
|
df = pd.read_excel(conversation_excel_path)
|
||||||
|
|
||||||
@@ -575,37 +635,123 @@ class DialogueToWorkorder:
|
|||||||
conversation_dict = new_conversation_dict
|
conversation_dict = new_conversation_dict
|
||||||
|
|
||||||
logger.info(f"会话总数为 {len(conversation_dict)}，处理全部会话")
|
logger.info(f"会话总数为 {len(conversation_dict)}，处理全部会话")
|
||||||
|
|
||||||
|
# ========== 新增：扫描已存在的JSON文件 ==========
|
||||||
|
existing_json_files = set()
|
||||||
|
workorder_json_dir = self.workorder_json_dir
|
||||||
|
if not os.path.exists(workorder_json_dir):
|
||||||
|
os.makedirs(workorder_json_dir, exist_ok=True)
|
||||||
|
for fname in os.listdir(workorder_json_dir):
|
||||||
|
if fname.endswith('.json'):
|
||||||
|
conversation_id = fname[:-5]
|
||||||
|
existing_json_files.add(conversation_id)
|
||||||
|
|
||||||
|
# 本次新生成的JSON文件
|
||||||
|
newly_generated_json_files = set()
|
||||||
|
# 本次未重新生成但已存在的JSON文件
|
||||||
|
reused_json_files = set()
|
||||||
|
|
||||||
|
# ========== 线程池处理会话 ==========
|
||||||
|
successful_conversations = set()
|
||||||
|
failed_conversations = set()
|
||||||
|
import threading
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def process_wrapper(conversation_id, conversation_rows):
|
||||||
|
json_file_path = os.path.join(workorder_json_dir, f"{conversation_id}.json")
|
||||||
|
if conversation_id in existing_json_files and os.path.exists(json_file_path):
|
||||||
|
# 已存在，直接复用
|
||||||
|
with lock:
|
||||||
|
reused_json_files.add(conversation_id)
|
||||||
|
return None # 不处理
|
||||||
|
# 否则正常处理
|
||||||
|
result = self.process_conversation(conversation_id, conversation_rows)
|
||||||
|
if result:
|
||||||
|
with lock:
|
||||||
|
newly_generated_json_files.add(conversation_id)
|
||||||
|
return result
|
||||||
|
|
||||||
# 使用线程池处理每个会话
|
|
||||||
workorder_dict_list = []
|
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||||
# 创建任务
|
|
||||||
future_to_conversation = {
|
future_to_conversation = {
|
||||||
executor.submit(self.process_conversation, conversation_id, conversation_rows): conversation_id
|
executor.submit(process_wrapper, conversation_id, conversation_rows): conversation_id
|
||||||
for conversation_id, conversation_rows in conversation_dict.items()
|
for conversation_id, conversation_rows in conversation_dict.items()
|
||||||
}
|
}
|
||||||
|
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="第一轮处理会话进度"):
|
||||||
# 获取结果
|
|
||||||
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="处理会话进度"):
|
|
||||||
conversation_id = future_to_conversation[future]
|
conversation_id = future_to_conversation[future]
|
||||||
try:
|
try:
|
||||||
result_workorders = future.result()
|
result_workorders = future.result()
|
||||||
if result_workorders:
|
if result_workorders:
|
||||||
# 将每个会话的所有工单添加到总列表中
|
successful_conversations.add(conversation_id)
|
||||||
workorder_dict_list.extend(result_workorders)
|
|
||||||
logger.info(f"完成处理会话ID: {conversation_id}，生成工单数量: {len(result_workorders)}")
|
logger.info(f"完成处理会话ID: {conversation_id}，生成工单数量: {len(result_workorders)}")
|
||||||
|
elif conversation_id in reused_json_files:
|
||||||
|
successful_conversations.add(conversation_id)
|
||||||
|
logger.info(f"跳过已存在JSON，会话ID: {conversation_id}")
|
||||||
|
else:
|
||||||
|
failed_conversations.add(conversation_id)
|
||||||
|
logger.warning(f"会话ID: {conversation_id} 处理可能失败，将在第二轮重试")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
failed_conversations.add(conversation_id)
|
||||||
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {exc}")
|
logger.error(f"处理会话ID: {conversation_id} 时发生错误: {exc}")
|
||||||
|
|
||||||
|
# 检查哪些会话没有成功生成JSON文件
|
||||||
|
all_conversation_ids = set(conversation_dict.keys())
|
||||||
|
for conversation_id in all_conversation_ids:
|
||||||
|
json_file_path = os.path.join(workorder_json_dir, f"{conversation_id}.json")
|
||||||
|
if not os.path.exists(json_file_path):
|
||||||
|
failed_conversations.add(conversation_id)
|
||||||
|
if conversation_id in successful_conversations:
|
||||||
|
successful_conversations.remove(conversation_id)
|
||||||
|
|
||||||
|
# ========== 第二轮重试 ==========
|
||||||
|
if failed_conversations:
|
||||||
|
logger.info(f"第一轮处理后有 {len(failed_conversations)} 个会话需要重试")
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, max_workers // 2)) as executor:
|
||||||
|
future_to_conversation = {
|
||||||
|
executor.submit(process_wrapper, conversation_id, conversation_dict[conversation_id]): conversation_id
|
||||||
|
for conversation_id in failed_conversations
|
||||||
|
}
|
||||||
|
for future in tqdm(concurrent.futures.as_completed(future_to_conversation), total=len(future_to_conversation), desc="第二轮重试处理进度"):
|
||||||
|
conversation_id = future_to_conversation[future]
|
||||||
|
try:
|
||||||
|
result_workorders = future.result()
|
||||||
|
if result_workorders:
|
||||||
|
successful_conversations.add(conversation_id)
|
||||||
|
newly_generated_json_files.add(conversation_id)
|
||||||
|
failed_conversations.remove(conversation_id)
|
||||||
|
logger.info(f"重试成功: 会话ID: {conversation_id}，生成工单数量: {len(result_workorders)}")
|
||||||
|
elif conversation_id in reused_json_files:
|
||||||
|
successful_conversations.add(conversation_id)
|
||||||
|
failed_conversations.remove(conversation_id)
|
||||||
|
logger.info(f"重试跳过已存在JSON，会话ID: {conversation_id}")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(f"重试处理会话ID: {conversation_id} 时仍然发生错误: {exc}")
|
||||||
|
|
||||||
|
# ========== 合并本次所有成功的JSON文件 ==========
|
||||||
|
logger.info(f"开始合并JSON文件结果，成功处理会话数: {len(successful_conversations)}，失败会话数: {len(failed_conversations)}")
|
||||||
|
workorder_dict_list = []
|
||||||
|
# 只合并本次新生成和本次未重新生成但已存在的JSON
|
||||||
|
all_json_ids_to_merge = newly_generated_json_files.union(reused_json_files)
|
||||||
|
json_files = [os.path.join(workorder_json_dir, f"{cid}.json") for cid in all_json_ids_to_merge if os.path.exists(os.path.join(workorder_json_dir, f"{cid}.json"))]
|
||||||
|
for json_file in tqdm(json_files, desc="合并JSON文件"):
|
||||||
|
conversation_id = os.path.basename(json_file).replace(".json", "")
|
||||||
|
try:
|
||||||
|
with open(json_file, 'r', encoding='utf-8') as f:
|
||||||
|
workorders = json.load(f)
|
||||||
|
workorder_dict_list.extend(workorders)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"加载JSON文件 {json_file} 时发生错误: {e}")
|
||||||
|
logger.info(f"处理完成，成功处理会话数: {len(successful_conversations)}，失败会话数: {len(failed_conversations)}")
|
||||||
|
if failed_conversations:
|
||||||
|
logger.warning(f"以下会话处理失败: {failed_conversations}")
|
||||||
return workorder_dict_list
|
return workorder_dict_list
|
||||||
|
|
||||||
def save_results_to_excel(self, workorder_dict_list, output_file=None):
|
def save_results_to_excel(self, workorder_dict_list, output_file=None):
|
||||||
"""将结果保存到Excel文件"""
|
"""将结果保存到Excel文件，并清理JSON文件"""
|
||||||
result_df = pd.DataFrame(workorder_dict_list)
|
result_df = pd.DataFrame(workorder_dict_list)
|
||||||
|
|
||||||
# 按照指定的列顺序重新排列DataFrame的列
|
# 按照指定的列顺序重新排列DataFrame的列
|
||||||
columns_order = [
|
columns_order = [
|
||||||
'工单编号', '产品线', '产品名称', '模块名称', '问题类型',
|
'工单编号', '产品线', '产品名称', '问题类型',
|
||||||
'客户问题', '解决方案', '是否抱怨', "抱怨内容", '是否投诉', '抱怨级别',
|
'客户问题', '解决方案', '是否抱怨', "抱怨内容", '是否投诉', '抱怨级别',
|
||||||
'会话id', '访客昵称', '处理坐席', "处理人", "处理技能组",'创建时间'
|
'会话id', '访客昵称', '处理坐席', "处理人", "处理技能组",'创建时间'
|
||||||
]
|
]
|
||||||
@@ -645,7 +791,6 @@ class DialogueToWorkorder:
|
|||||||
'工单编号': 15,
|
'工单编号': 15,
|
||||||
'产品线': 24,
|
'产品线': 24,
|
||||||
'产品名称': 40,
|
'产品名称': 40,
|
||||||
'模块名称': 40,
|
|
||||||
'问题类型': 9,
|
'问题类型': 9,
|
||||||
'客户问题': 20,
|
'客户问题': 20,
|
||||||
'解决方案': 30,
|
'解决方案': 30,
|
||||||
@@ -668,8 +813,7 @@ class DialogueToWorkorder:
|
|||||||
col_letter = chr(64 + i // 26) + chr(65 + i % 26)
|
col_letter = chr(64 + i // 26) + chr(65 + i % 26)
|
||||||
worksheet.column_dimensions[col_letter].width = column_widths[column]
|
worksheet.column_dimensions[col_letter].width = column_widths[column]
|
||||||
|
|
||||||
logger.info(f"结果已保存到 {output_file}")
|
logger.info(f"结果已保存到 {output_file}")
|
||||||
|
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
# ================ 参数解析 ================
|
# ================ 参数解析 ================
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ from tqdm import tqdm
|
|||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
os.makedirs('./data/log', exist_ok=True)
|
os.makedirs('./data/logs', exist_ok=True)
|
||||||
# 配置日志
|
# 配置日志
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
|
|||||||
@@ -214,20 +214,7 @@ class QueryRewriteProcessor:
|
|||||||
|
|
||||||
# 根据enable_retrieval参数决定是否进行文档检索
|
# 根据enable_retrieval参数决定是否进行文档检索
|
||||||
retrieved_doc = None
|
retrieved_doc = None
|
||||||
if enable_retrieval:
|
|
||||||
retrieved_doc = self.dify_query_retrieval.retrieve(original_query, query_list, classification_obj, current_softname)
|
|
||||||
|
|
||||||
# 判断检索文档是否相关
|
|
||||||
relevance_result = {}
|
|
||||||
if retrieved_doc:
|
|
||||||
# 判断文档相关性
|
|
||||||
relevance_result = self.is_retrieved_doc_relevant(query, retrieved_doc)
|
|
||||||
else:
|
|
||||||
relevance_result = {
|
|
||||||
"is_relevant": False,
|
|
||||||
"explanation": "没有检索到文档" if enable_retrieval else "文档检索功能未启用",
|
|
||||||
"relevance_score": 0.0
|
|
||||||
}
|
|
||||||
|
|
||||||
retrieved_doc_titles=[]
|
retrieved_doc_titles=[]
|
||||||
if retrieved_doc:
|
if retrieved_doc:
|
||||||
@@ -251,8 +238,6 @@ class QueryRewriteProcessor:
|
|||||||
"槽位信息": slot_filling_str,
|
"槽位信息": slot_filling_str,
|
||||||
"检索的文档": "\n".join(retrieved_doc_titles),
|
"检索的文档": "\n".join(retrieved_doc_titles),
|
||||||
"检索的内容": json.dumps(retrieved_doc, ensure_ascii=False, indent=2) if retrieved_doc else "",
|
"检索的内容": json.dumps(retrieved_doc, ensure_ascii=False, indent=2) if retrieved_doc else "",
|
||||||
"文档能否解决问题": "能" if relevance_result["is_relevant"] else "不能",
|
|
||||||
"文档相关性解释": relevance_result["explanation"]
|
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"处理问题 '{query}' 时出错: ",exc_info=True)
|
logging.error(f"处理问题 '{query}' 时出错: ",exc_info=True)
|
||||||
|
|||||||
@@ -1,201 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
File: merge_nouns_with_llm.py
|
|
||||||
Description: 合并多个nouns.json中的同名专业名词，利用LLM生成唯一合并结果
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import glob
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from collections import defaultdict
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from rag2_0.tool.ModelTool import OpenAiLLM
|
|
||||||
from rag2_0.intent_recognition.DataModels import Term
|
|
||||||
import logging
|
|
||||||
from langchain.output_parsers import PydanticOutputParser
|
|
||||||
from tqdm import tqdm
|
|
||||||
import time
|
|
||||||
# 加载环境变量
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
class TermMerger:
|
|
||||||
"""专业名词合并类，用于合并多个数据源中的同名专业名词"""
|
|
||||||
|
|
||||||
def __init__(self, input_dir=None, output_path=None, max_workers=3):
|
|
||||||
"""初始化名词合并器
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_dir: 包含nouns.json文件的目录路径
|
|
||||||
output_path: 合并结果的输出文件路径
|
|
||||||
max_workers: 线程池最大工作线程数
|
|
||||||
"""
|
|
||||||
self.EXTRACTED_NOUNS_DIR = input_dir
|
|
||||||
self.OUTPUT_PATH = output_path
|
|
||||||
self.MAX_WORKERS = max_workers
|
|
||||||
self.terms_parser = PydanticOutputParser(pydantic_object=Term)
|
|
||||||
self.MERGE_PROMPT = '''
|
|
||||||
请将以下多个描述相同名词"{name}"的条目合并为一个，合并时请：
|
|
||||||
- 同义词（synonymous）去重合并
|
|
||||||
- 描述（description）合并为更完整、简明的描述
|
|
||||||
- 保持输出格式为：
|
|
||||||
{output_format}
|
|
||||||
原始条目：
|
|
||||||
{items}
|
|
||||||
'''
|
|
||||||
# 配置LLM
|
|
||||||
model_name = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
|
|
||||||
api_key = os.getenv("OPENAI_API_KEY")
|
|
||||||
base_url = os.getenv("OPENAI_API_BASE")
|
|
||||||
llm_params = {"temperature": 0.3, "model": model_name}
|
|
||||||
if api_key:
|
|
||||||
llm_params["api_key"] = api_key
|
|
||||||
if base_url:
|
|
||||||
llm_params["base_url"] = base_url
|
|
||||||
self.llm = OpenAiLLM(**llm_params)
|
|
||||||
|
|
||||||
# 配置日志
|
|
||||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
||||||
|
|
||||||
def load_all_terms(self):
|
|
||||||
"""读取目录下所有nouns.json，返回所有Term列表"""
|
|
||||||
all_terms = []
|
|
||||||
for file in glob.glob(os.path.join(self.EXTRACTED_NOUNS_DIR, '*_nouns.json')):
|
|
||||||
with open(file, 'r', encoding='utf-8') as f:
|
|
||||||
try:
|
|
||||||
file_terms = json.load(f)
|
|
||||||
new_terms = [{"name": term["name"].upper(), "synonymous": term["synonymous"], "description": term["description"]} for term in file_terms]
|
|
||||||
all_terms.extend(new_terms)
|
|
||||||
logging.info(f"加载{file}，共{len(new_terms)}条")
|
|
||||||
except Exception as e:
|
|
||||||
logging.warning(f"读取{file}失败: {e}")
|
|
||||||
|
|
||||||
# 加载suffix_keywords.json文件
|
|
||||||
# suffix_keywords_path = os.path.join(os.path.dirname(os.path.dirname(self.EXTRACTED_NOUNS_DIR)), 'data', 'nouns', 'suffix_keywords.json')
|
|
||||||
# if os.path.exists(suffix_keywords_path):
|
|
||||||
# try:
|
|
||||||
# with open(suffix_keywords_path, 'r', encoding='utf-8') as f:
|
|
||||||
# suffix_terms = json.load(f)
|
|
||||||
# suffix_terms = [{"name": term["name"].upper(), "synonymous": "", "description": ""} for term in suffix_terms]
|
|
||||||
# all_terms.extend(suffix_terms)
|
|
||||||
# logging.info(f"加载{suffix_keywords_path}，共{len(suffix_terms)}条")
|
|
||||||
# except Exception as e:
|
|
||||||
# logging.warning(f"读取{suffix_keywords_path}失败: {e}")
|
|
||||||
|
|
||||||
return all_terms
|
|
||||||
|
|
||||||
def group_terms_by_name(self, terms):
|
|
||||||
"""按name聚合Term"""
|
|
||||||
name2terms = defaultdict(list)
|
|
||||||
for term in terms:
|
|
||||||
name = term.get('name', '').strip()
|
|
||||||
if name:
|
|
||||||
name2terms[name].append(term)
|
|
||||||
return name2terms
|
|
||||||
|
|
||||||
def merge_terms_with_llm(self, name, term_list):
|
|
||||||
"""调用LLM合并同名Term，失败最多重试三次"""
|
|
||||||
items = json.dumps(term_list, ensure_ascii=False)
|
|
||||||
prompt = self.MERGE_PROMPT.format(name=name, items=items, output_format=self.terms_parser.get_format_instructions())
|
|
||||||
|
|
||||||
max_retries = 3
|
|
||||||
for attempt in range(1, max_retries + 1):
|
|
||||||
try:
|
|
||||||
response = self.llm.invoke(prompt, False)
|
|
||||||
parsed_output = self.terms_parser.parse(response.content)
|
|
||||||
return {"name": parsed_output.name, "synonymous": parsed_output.synonymous, "description": parsed_output.description}
|
|
||||||
except Exception as e:
|
|
||||||
if attempt == max_retries:
|
|
||||||
logging.warning(f"解析LLM合并结果失败: {e}")
|
|
||||||
return None
|
|
||||||
else:
|
|
||||||
time.sleep(10*attempt)
|
|
||||||
|
|
||||||
def process_term(self, name_terms_tuple):
|
|
||||||
"""处理单个词条，用于线程池并行处理"""
|
|
||||||
name, term_list = name_terms_tuple
|
|
||||||
try:
|
|
||||||
merged = self.merge_terms_with_llm(name, term_list)
|
|
||||||
if merged:
|
|
||||||
return merged
|
|
||||||
else:
|
|
||||||
return term_list[0]
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"处理词条 {name} 时出错: {e}", exc_info=True)
|
|
||||||
return term_list[0]
|
|
||||||
|
|
||||||
def merge(self):
|
|
||||||
"""合并所有词条的入口方法"""
|
|
||||||
# 1. 读取所有术语
|
|
||||||
all_terms = self.load_all_terms()
|
|
||||||
logging.info(f"共加载{len(all_terms)}条术语")
|
|
||||||
|
|
||||||
# 2. 按名称聚合
|
|
||||||
name2terms = self.group_terms_by_name(all_terms)
|
|
||||||
logging.info(f"共{len(name2terms)}个唯一名词")
|
|
||||||
|
|
||||||
# 3. 使用线程池并行处理
|
|
||||||
merged_terms = []
|
|
||||||
items_to_process = []
|
|
||||||
|
|
||||||
# 先处理只有一个条目的词条（不需要合并）
|
|
||||||
for name, term_list in name2terms.items():
|
|
||||||
if len(term_list) == 1:
|
|
||||||
merged_terms.append(term_list[0])
|
|
||||||
else:
|
|
||||||
items_to_process.append((name, term_list))
|
|
||||||
|
|
||||||
logging.info(f"共{len(merged_terms)}个单一条目，{len(items_to_process)}个需要合并的条目")
|
|
||||||
|
|
||||||
# 只对需要合并的词条使用线程池处理
|
|
||||||
if items_to_process:
|
|
||||||
with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
|
|
||||||
# 使用tqdm显示进度
|
|
||||||
for result in tqdm(executor.map(self.process_term, items_to_process), total=len(items_to_process)):
|
|
||||||
merged_terms.append(result)
|
|
||||||
|
|
||||||
# 4. 去重
|
|
||||||
merged_terms = self.deduplicate_synonymous_name(merged_terms)
|
|
||||||
|
|
||||||
# 4. 保存合并结果
|
|
||||||
os.makedirs(os.path.dirname(self.OUTPUT_PATH), exist_ok=True)
|
|
||||||
with open(self.OUTPUT_PATH, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(merged_terms, f, ensure_ascii=False, indent=2)
|
|
||||||
logging.info(f"合并后结果已保存到: {self.OUTPUT_PATH}")
|
|
||||||
|
|
||||||
return merged_terms
|
|
||||||
|
|
||||||
def deduplicate_synonymous_name(self, terms):
|
|
||||||
# 1. 删除name字段重复的条目
|
|
||||||
unique_names = set()
|
|
||||||
unique_data = []
|
|
||||||
|
|
||||||
for item in terms:
|
|
||||||
if item["name"] not in unique_names:
|
|
||||||
unique_names.add(item["name"])
|
|
||||||
unique_data.append(item)
|
|
||||||
# 如果重复，则跳过该条目
|
|
||||||
|
|
||||||
# 2. 如果A条目的某一个synonymou字段是B条目的name，则删除A条目中的对应的synonymou
|
|
||||||
name_set = {item["name"] for item in unique_data}
|
|
||||||
|
|
||||||
for item in unique_data:
|
|
||||||
# 过滤掉synonymous中与其他条目name重复的部分
|
|
||||||
filtered_synonymous = [syn for syn in item["synonymous"] if syn not in name_set]
|
|
||||||
item["synonymous"] = filtered_synonymous
|
|
||||||
|
|
||||||
return unique_data
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""主函数，创建TermMerger实例并执行合并"""
|
|
||||||
|
|
||||||
cur_path = os.path.dirname(__file__)
|
|
||||||
input_dir = os.path.abspath(os.path.join(cur_path, '../../data/wiki_extracted_nouns'))
|
|
||||||
output_path = os.path.join(cur_path, "..", "..", "data", "nouns", 'merged_nouns.json')
|
|
||||||
merger = TermMerger(input_dir=input_dir, output_path=output_path, max_workers=20)
|
|
||||||
merger.merge()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
logging.getLogger('httpx').setLevel(logging.WARNING)
|
|
||||||
logging.getLogger('openai').setLevel(logging.WARNING)
|
|
||||||
main()
|
|
||||||
@@ -42,8 +42,6 @@ logging.basicConfig(
|
|||||||
|
|
||||||
class DifyCompareTest:
|
class DifyCompareTest:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
# 先词条后工单检索工作流
|
|
||||||
self.first_wiki_client = ChatClient(api_key="app-gocvuqduBnJptYNPpnW9V9R6", base_url=os.getenv("DIFY_BSAE_URL"))
|
|
||||||
# 词条与工单同时检索
|
# 词条与工单同时检索
|
||||||
self.both_wiki_worker_client = ChatClient(api_key="app-CPoOMaGDsLRPAe9TW7Xjhszy", base_url=os.getenv("DIFY_BSAE_URL"))
|
self.both_wiki_worker_client = ChatClient(api_key="app-CPoOMaGDsLRPAe9TW7Xjhszy", base_url=os.getenv("DIFY_BSAE_URL"))
|
||||||
self.llm = OpenAiLLM(base_url=os.getenv("OPENAI_API_BASE"), model=os.getenv("MODEL_NAME"))
|
self.llm = OpenAiLLM(base_url=os.getenv("OPENAI_API_BASE"), model=os.getenv("MODEL_NAME"))
|
||||||
@@ -88,8 +86,8 @@ class DifyCompareTest:
|
|||||||
time.sleep(1) # 等待1秒后重试
|
time.sleep(1) # 等待1秒后重试
|
||||||
|
|
||||||
|
|
||||||
def process_workflow(self, workflow_name, client, inputs, query, old_answer):
|
def process_workflow(self, client, inputs, query, old_answer):
|
||||||
"""处理单个工作流调用"""
|
"""处理工作流调用"""
|
||||||
max_retries = 3
|
max_retries = 3
|
||||||
retry_count = 0
|
retry_count = 0
|
||||||
|
|
||||||
@@ -107,14 +105,14 @@ class DifyCompareTest:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
retry_count += 1
|
retry_count += 1
|
||||||
if retry_count >= max_retries:
|
if retry_count >= max_retries:
|
||||||
logging.error(f"{workflow_name}调用失败 (尝试 {max_retries} 次后): {e}")
|
logging.error(f"词条与工单同时检索调用失败 (尝试 {max_retries} 次后): {e}")
|
||||||
return '', ''
|
return '', ''
|
||||||
else:
|
else:
|
||||||
import time
|
import time
|
||||||
time.sleep(1) # 等待1秒后重试
|
time.sleep(1) # 等待1秒后重试
|
||||||
|
|
||||||
def process_single_row(self, index, row):
|
def process_single_row(self, index, row):
|
||||||
"""处理单行数据的方法，用于多线程执行"""
|
"""处理单行数据的方法"""
|
||||||
try:
|
try:
|
||||||
query = row["提问"]
|
query = row["提问"]
|
||||||
old_answer = row["回答"]
|
old_answer = row["回答"]
|
||||||
@@ -125,47 +123,18 @@ class DifyCompareTest:
|
|||||||
"user_name": "AutoCodeRun"
|
"user_name": "AutoCodeRun"
|
||||||
}
|
}
|
||||||
|
|
||||||
# 并行调用两个工作流
|
# 调用词条与工单同时检索工作流
|
||||||
results = {'first_wiki': None, 'both_wiki_worker': None}
|
answer, judge_result = self.process_workflow(
|
||||||
|
self.both_wiki_worker_client,
|
||||||
with ThreadPoolExecutor(max_workers=2) as workflow_executor:
|
inputs,
|
||||||
# 提交两个工作流任务
|
query,
|
||||||
futures = {
|
old_answer
|
||||||
workflow_executor.submit(
|
)
|
||||||
self.process_workflow,
|
|
||||||
"先词条后工单工作流",
|
|
||||||
self.first_wiki_client,
|
|
||||||
inputs,
|
|
||||||
query,
|
|
||||||
old_answer
|
|
||||||
): 'first_wiki',
|
|
||||||
|
|
||||||
workflow_executor.submit(
|
|
||||||
self.process_workflow,
|
|
||||||
"词条与工单同时工作流",
|
|
||||||
self.both_wiki_worker_client,
|
|
||||||
inputs,
|
|
||||||
query,
|
|
||||||
old_answer
|
|
||||||
): 'both_wiki_worker'
|
|
||||||
}
|
|
||||||
|
|
||||||
# 收集结果
|
|
||||||
for future in as_completed(futures):
|
|
||||||
workflow_key = futures[future]
|
|
||||||
try:
|
|
||||||
answer, judge_result = future.result()
|
|
||||||
results[workflow_key] = (answer, judge_result)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"工作流执行失败 (行{index}): {e}")
|
|
||||||
results[workflow_key] = ('', '')
|
|
||||||
|
|
||||||
# 构建结果
|
# 构建结果
|
||||||
result_row = row.copy()
|
result_row = row.copy()
|
||||||
result_row["先词条后工单回答"] = results['first_wiki'][0]
|
result_row["词条与工单同时回答"] = answer
|
||||||
result_row["先词条后工单回答对比"] = results['first_wiki'][1]
|
result_row["词条与工单同时回答对比"] = judge_result
|
||||||
result_row["词条与工单同时回答"] = results['both_wiki_worker'][0]
|
|
||||||
result_row["词条与工单同时回答对比"] = results['both_wiki_worker'][1]
|
|
||||||
|
|
||||||
logging.info(f"成功处理第 {index + 1} 行数据")
|
logging.info(f"成功处理第 {index + 1} 行数据")
|
||||||
return index, result_row
|
return index, result_row
|
||||||
@@ -173,8 +142,6 @@ class DifyCompareTest:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"处理第 {index + 1} 行数据时出错: {e}")
|
logging.error(f"处理第 {index + 1} 行数据时出错: {e}")
|
||||||
result_row = row.copy()
|
result_row = row.copy()
|
||||||
result_row["先词条后工单回答"] = ''
|
|
||||||
result_row["先词条后工单回答对比"] = ''
|
|
||||||
result_row["词条与工单同时回答"] = ''
|
result_row["词条与工单同时回答"] = ''
|
||||||
result_row["词条与工单同时回答对比"] = ''
|
result_row["词条与工单同时回答对比"] = ''
|
||||||
return index, result_row
|
return index, result_row
|
||||||
@@ -232,8 +199,6 @@ class DifyCompareTest:
|
|||||||
logging.error(f"线程执行失败 (行{original_index + 1}): {e}")
|
logging.error(f"线程执行失败 (行{original_index + 1}): {e}")
|
||||||
# 添加失败的行
|
# 添加失败的行
|
||||||
result_row = df.iloc[original_index].copy()
|
result_row = df.iloc[original_index].copy()
|
||||||
result_row["先词条后工单回答"] = '线程执行失败'
|
|
||||||
result_row["先词条后工单回答对比"] = '线程执行失败'
|
|
||||||
result_row["词条与工单同时回答"] = '线程执行失败'
|
result_row["词条与工单同时回答"] = '线程执行失败'
|
||||||
result_row["词条与工单同时回答对比"] = '线程执行失败'
|
result_row["词条与工单同时回答对比"] = '线程执行失败'
|
||||||
results[original_index] = result_row
|
results[original_index] = result_row
|
||||||
|
|||||||
@@ -1,32 +1,17 @@
|
|||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
from threading import Thread
|
from typing import List, Dict, Any
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
||||||
from typing import List, Dict, Any, Optional
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
import httpx
|
import httpx
|
||||||
sys.path.append(os.getcwd())
|
sys.path.append(os.getcwd())
|
||||||
|
|
||||||
from rag2_0.intent_recognition.DataModels import Classification
|
|
||||||
from rag2_0.dify.dify_client.client import DifyClient, KnowledgeBaseClient
|
from rag2_0.dify.dify_client.client import DifyClient, KnowledgeBaseClient
|
||||||
from rag2_0.tool.ModelTool import XinferenceReRankerModel
|
from rag2_0.tool.ModelTool import XinferenceReRankerModel
|
||||||
class DifyQueryRetrieval:
|
class DifyQueryRetrieval:
|
||||||
|
|
||||||
software_to_dataset_map = {"配网工程计价通D3":["下载安装注册(new)","配网造价知识(new)","配网造价软件知识(new)"],
|
|
||||||
"新型储能电站建设计价通C1":["下载安装注册(new)","储能C1计价通软件知识(new)","新能源造价知识(new)"],
|
|
||||||
"西藏电力工程计价通Z1":["下载安装注册(new)","西藏造价知识(new)","西藏造价软件知识(new)"],
|
|
||||||
"技改检修工程计价通T1":["下载安装注册(new)","技改检修工程计价通T1软件知识(new)","技改造价知识(new)"],
|
|
||||||
"技改检修清单计价通T1":["下载安装注册(new)","技改检修清单计价通T1软件知识(new)","技改造价知识(new)"],
|
|
||||||
"电力建设计价通":["下载安装注册(new)","主网造价知识(new)","电力建设计价通（2018）软件知识(new)"],
|
|
||||||
"其他":["下载安装注册(new)","技改检修清单计价通T1软件知识(new)",
|
|
||||||
"主网造价知识(new)","西藏造价知识(new)","技改检修工程计价通T1软件知识(new)",
|
|
||||||
"电力建设计价通（2018）软件知识(new)","储能C1计价通软件知识(new)",
|
|
||||||
"西藏造价软件知识(new)","新能源造价知识(new)","配网造价知识(new)","技改造价知识(new)",
|
|
||||||
"配网造价软件知识(new)"]}
|
|
||||||
|
|
||||||
def __init__(self, dify_dataset_key: str, dify_base_url: str):
|
def __init__(self, dify_dataset_key: str, dify_base_url: str):
|
||||||
self._dify_dataset_key = dify_dataset_key
|
self._dify_dataset_key = dify_dataset_key
|
||||||
self._dify_base_url = dify_base_url
|
self._dify_base_url = dify_base_url
|
||||||
@@ -38,7 +23,7 @@ class DifyQueryRetrieval:
|
|||||||
datasets_json = datasets.json()
|
datasets_json = datasets.json()
|
||||||
return {dataset["name"]:dataset for dataset in datasets_json["data"]}
|
return {dataset["name"]:dataset for dataset in datasets_json["data"]}
|
||||||
|
|
||||||
def retrieve_by_dataset(self, query: str, dataset_name: str) -> List[Dict[str, Any]]:
|
def retrieve_by_dataset(self, query: str, dataset_name: str) -> Dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
dataset_id = self._datasets_list[dataset_name]["id"]
|
dataset_id = self._datasets_list[dataset_name]["id"]
|
||||||
retrieval_model = self._datasets_list[dataset_name]["retrieval_model_dict"]
|
retrieval_model = self._datasets_list[dataset_name]["retrieval_model_dict"]
|
||||||
@@ -52,12 +37,21 @@ class DifyQueryRetrieval:
|
|||||||
retrieved_document["dataset_id"] = dataset_id
|
retrieved_document["dataset_id"] = dataset_id
|
||||||
retrieved_document["dataset_name"] = dataset_name
|
retrieved_document["dataset_name"] = dataset_name
|
||||||
|
|
||||||
return retrieved_documents
|
# 返回包含查询和文档的字典
|
||||||
|
return {
|
||||||
|
"query": query,
|
||||||
|
"dataset_name": dataset_name,
|
||||||
|
"documents": retrieved_documents
|
||||||
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
logging.error(f"检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
||||||
return []
|
return {
|
||||||
|
"query": query,
|
||||||
|
"dataset_name": dataset_name,
|
||||||
|
"documents": []
|
||||||
|
}
|
||||||
|
|
||||||
async def retrieve_by_dataset_async(self, query: str, dataset_name: str) -> List[Dict[str, Any]]:
|
async def retrieve_by_dataset_async(self, query: str, dataset_name: str) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
异步版本的retrieve_by_dataset方法
|
异步版本的retrieve_by_dataset方法
|
||||||
|
|
||||||
@@ -66,7 +60,7 @@ class DifyQueryRetrieval:
|
|||||||
dataset_name: 数据集名称
|
dataset_name: 数据集名称
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
检索到的文档列表
|
包含查询、数据集名称和检索到的文档的字典
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# 使用asyncio.to_thread包装同步方法
|
# 使用asyncio.to_thread包装同步方法
|
||||||
@@ -77,81 +71,13 @@ class DifyQueryRetrieval:
|
|||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"异步检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
logging.error(f"异步检索数据集 {dataset_name} 时出错: {str(e)}", exc_info=True)
|
||||||
return []
|
return {
|
||||||
|
"query": query,
|
||||||
|
"dataset_name": dataset_name,
|
||||||
|
"documents": []
|
||||||
|
}
|
||||||
|
|
||||||
def retrieve(self, original_query: str, query_list: List[str], classification: Classification, software_name: str) -> Optional[List[Dict[str, Any]]]:
|
async def retrieve_api_async(self, original_query: str, query_list: List[str], data_set_list: List[str], query_expand_dict: dict, top_k: int = 5)->Dict[str, Any]:
|
||||||
datasets = self.get_datasets_by_classification(classification, software_name)
|
|
||||||
datasets=["电力建设计价通（2018）软件知识(new)", "主网造价知识(new)", "下载安装注册(new)"]
|
|
||||||
if len(datasets) == 0:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return self.retrieve_api(original_query, query_list, datasets)
|
|
||||||
|
|
||||||
async def retrieve_async(self, original_query: str, query_list: List[str], classification: Classification, software_name: str) -> Optional[List[Dict[str, Any]]]:
|
|
||||||
"""
|
|
||||||
异步版本的retrieve方法
|
|
||||||
|
|
||||||
Args:
|
|
||||||
original_query: 原始查询
|
|
||||||
query_list: 查询列表
|
|
||||||
classification: 分类信息
|
|
||||||
software_name: 软件名称
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
检索到的文档列表
|
|
||||||
"""
|
|
||||||
datasets = self.get_datasets_by_classification(classification, software_name)
|
|
||||||
if len(datasets) == 0:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return await self.retrieve_api_async(original_query, query_list, datasets)
|
|
||||||
|
|
||||||
def retrieve_api(self, original_query: str, query_list: List[str],data_set_list: List[str], top_k: int = 5)->List[Dict[str, Any]]:
|
|
||||||
ssss = self.retrieve_by_dataset("怎么调整报表顺序", "电力建设计价通（2018）软件知识(new)")
|
|
||||||
all_documents=[]
|
|
||||||
# 使用线程池替代无限制创建线程
|
|
||||||
# 设置合理的最大线程数，这里使用min(32, len(query_list) * len(datasets))来限制
|
|
||||||
time_start = time.time()
|
|
||||||
max_workers = min(os.cpu_count() * 2, len(query_list) * len(data_set_list))
|
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
||||||
futures = {}
|
|
||||||
for query in query_list:
|
|
||||||
for dataset in data_set_list:
|
|
||||||
if dataset not in list(self._datasets_list.keys()):
|
|
||||||
raise ValueError(f"dataset {dataset} not in datasets_list")
|
|
||||||
|
|
||||||
futures[executor.submit(self.retrieve_by_dataset, query, dataset)] = query
|
|
||||||
|
|
||||||
# 等待所有任务完成
|
|
||||||
for future in as_completed(futures.keys()):
|
|
||||||
# 处理可能的异常
|
|
||||||
try:
|
|
||||||
retrieved_documents = future.result()
|
|
||||||
all_documents.extend(retrieved_documents)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f"检索过程中发生错误: {str(e)}", exc_info=True)
|
|
||||||
time_end = time.time()
|
|
||||||
|
|
||||||
logging.info(f"检索耗时: {time_end - time_start:.2f}秒")
|
|
||||||
# 根据segment_id对文档进行去重
|
|
||||||
unique_documents = {}
|
|
||||||
for document in all_documents:
|
|
||||||
segment_id = document['segment']['id']
|
|
||||||
if segment_id not in unique_documents:
|
|
||||||
unique_documents[segment_id] = document
|
|
||||||
|
|
||||||
# 将去重后的文档转换为列表
|
|
||||||
deduplicated_documents = list(unique_documents.values())
|
|
||||||
|
|
||||||
# 对所有检索出来的文档进行重排序
|
|
||||||
time_start = time.time()
|
|
||||||
processed_documents = self.data_post_processor("怎么调整报表顺序", deduplicated_documents, top_k)
|
|
||||||
time_end = time.time()
|
|
||||||
logging.info(f"检索后重排序耗时: {time_end - time_start:.2f}秒")
|
|
||||||
|
|
||||||
return processed_documents
|
|
||||||
|
|
||||||
async def retrieve_api_async(self, original_query: str, query_list: List[str], data_set_list: List[str], top_k: int = 5)->List[Dict[str, Any]]:
|
|
||||||
"""
|
"""
|
||||||
异步版本的retrieve_api方法，使用asyncio代替线程池
|
异步版本的retrieve_api方法，使用asyncio代替线程池
|
||||||
|
|
||||||
@@ -159,11 +85,14 @@ class DifyQueryRetrieval:
|
|||||||
original_query: 原始查询
|
original_query: 原始查询
|
||||||
query_list: 查询列表
|
query_list: 查询列表
|
||||||
data_set_list: 数据集列表
|
data_set_list: 数据集列表
|
||||||
|
query_expand_dict: 查询扩展字典，包含不同类型的查询
|
||||||
|
top_k: 返回的文档数量
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
检索并重排序后的文档列表
|
包含检索结果和查询命中统计的字典
|
||||||
"""
|
"""
|
||||||
all_documents = []
|
all_documents = []
|
||||||
|
query_document_mapping = {} # 用于存储查询和文档的映射关系，键为查询，值为文档列表
|
||||||
# 记录开始时间
|
# 记录开始时间
|
||||||
time_start = time.time()
|
time_start = time.time()
|
||||||
|
|
||||||
@@ -187,7 +116,16 @@ class DifyQueryRetrieval:
|
|||||||
if isinstance(result, Exception):
|
if isinstance(result, Exception):
|
||||||
logging.error(f"异步检索过程中发生错误: {str(result)}", exc_info=True)
|
logging.error(f"异步检索过程中发生错误: {str(result)}", exc_info=True)
|
||||||
else:
|
else:
|
||||||
all_documents.extend(result)
|
# 添加查询和文档的映射关系
|
||||||
|
query = result["query"]
|
||||||
|
if query not in query_document_mapping:
|
||||||
|
query_document_mapping[query] = []
|
||||||
|
|
||||||
|
# 将文档添加到对应查询的文档列表中
|
||||||
|
query_document_mapping[query].extend([item['segment']['id'] for item in result["documents"]])
|
||||||
|
|
||||||
|
# 将文档添加到总文档列表
|
||||||
|
all_documents.extend(result["documents"])
|
||||||
|
|
||||||
time_end = time.time()
|
time_end = time.time()
|
||||||
logging.info(f"异步检索耗时: {time_end - time_start:.2f}秒")
|
logging.info(f"异步检索耗时: {time_end - time_start:.2f}秒")
|
||||||
@@ -203,49 +141,61 @@ class DifyQueryRetrieval:
|
|||||||
deduplicated_documents = list(unique_documents.values())
|
deduplicated_documents = list(unique_documents.values())
|
||||||
|
|
||||||
if len(deduplicated_documents) == 0:
|
if len(deduplicated_documents) == 0:
|
||||||
return []
|
return {
|
||||||
|
"documents": [],
|
||||||
|
"query_hit_stats": {}
|
||||||
|
}
|
||||||
|
|
||||||
# 对所有检索出来的文档进行重排序
|
# 对所有检索出来的文档进行重排序
|
||||||
time_start = time.time()
|
time_start = time.time()
|
||||||
processed_documents = await self.data_post_processor_async(original_query, deduplicated_documents, top_k)
|
processed_documents = await self.data_post_processor_async(original_query, deduplicated_documents, top_k)
|
||||||
time_end = time.time()
|
time_end = time.time()
|
||||||
logging.info(f"异步检索后重排序耗时: {time_end - time_start:.2f}秒")
|
logging.info(f"异步检索后重排序耗时: {time_end - time_start:.2f}秒")
|
||||||
|
|
||||||
return processed_documents
|
# 统计不同类型查询命中的文档
|
||||||
|
query_hit_stats = {}
|
||||||
def data_post_processor(self, query: str, all_documents: List[Dict[str, Any]], top_k: int = 5) -> List[Dict[str, Any]]:
|
|
||||||
reranker_model = XinferenceReRankerModel()
|
# 获取重排序后的文档ID列表
|
||||||
documents = [document['segment']['content'] for document in all_documents]
|
reranked_doc_ids = [doc["metadata"]["segment_id"] for doc in processed_documents]
|
||||||
reranked_documents = reranker_model.rerank(query, documents, top_k=top_k)
|
reranked_doc_titles = [doc["title"].split("/")[-1] for doc in processed_documents]
|
||||||
new_all_documents = []
|
|
||||||
|
# 解析query_expand_dict（如果是字符串）
|
||||||
def to_dify_document_format(document: dict)->dict:
|
if isinstance(query_expand_dict, str):
|
||||||
return {
|
try:
|
||||||
"metadata": {
|
query_expand_dict = json.loads(query_expand_dict)
|
||||||
"_source": "knowledge",
|
except Exception as e:
|
||||||
"dataset_id": document["dataset_id"],
|
logging.error(f"解析query_expand_dict失败: {str(e)}")
|
||||||
"dataset_name": document["dataset_name"],
|
query_expand_dict = {}
|
||||||
"document_id": document['segment']['document_id'],
|
|
||||||
"document_name": document["segment"]["document"]["name"],
|
# 统计各类型查询命中的文档
|
||||||
"data_source_type": document["segment"]["document"]["data_source_type"],
|
for query_type, queries in query_expand_dict.items():
|
||||||
"segment_id": document["segment"]["id"],
|
if not isinstance(queries, list):
|
||||||
"retriever_from": "api",
|
queries = [queries]
|
||||||
"score": document.get("score", 0),
|
|
||||||
"segment_hit_count": document.get("segment", {}).get("hit_count", 0),
|
# 初始化该查询类型命中的文档列表
|
||||||
"segment_word_count": document.get("segment", {}).get("word_count", 0),
|
query_hit_stats[query_type] = []
|
||||||
"segment_position": document.get("segment", {}).get("position", 0),
|
|
||||||
"segment_index_node_hash": document.get("segment", {}).get("index_node_hash", ""),
|
# 合并所有该类型查询命中的文档
|
||||||
"doc_metadata": document.get("segment", {}).get("document", {}).get("doc_metadata", None),
|
hit_doc_ids_set = set()
|
||||||
"position": document["segment"].get("position", 0)
|
for query in queries:
|
||||||
},
|
if query in query_document_mapping:
|
||||||
"title": document["segment"]["document"]["name"],
|
hit_doc_ids_set.update(set(query_document_mapping[query]))
|
||||||
"content": document["segment"]["content"]
|
|
||||||
}
|
# 找出在重排序结果中的文档
|
||||||
|
hit_doc_titles_set = set() # 用于去重
|
||||||
for reranked_document in reranked_documents:
|
for i, doc_id in enumerate(reranked_doc_ids):
|
||||||
cur_doc_info = all_documents[reranked_document["index"]]
|
if doc_id in hit_doc_ids_set:
|
||||||
cur_doc_info["score"] = reranked_document["score"]
|
doc_title = reranked_doc_titles[i]
|
||||||
new_all_documents.append(to_dify_document_format(cur_doc_info))
|
if doc_title not in hit_doc_titles_set: # 确保不添加重复的标题
|
||||||
return new_all_documents
|
hit_doc_titles_set.add(doc_title)
|
||||||
|
query_hit_stats[query_type].append(doc_title)
|
||||||
|
|
||||||
|
logging.info(f"查询命中统计: {json.dumps(query_hit_stats, ensure_ascii=False)}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"documents": processed_documents,
|
||||||
|
"query_hit_stats": query_hit_stats
|
||||||
|
}
|
||||||
|
|
||||||
async def data_post_processor_async(self, query: str, all_documents: List[Dict[str, Any]], top_k: int = 5) -> List[Dict[str, Any]]:
|
async def data_post_processor_async(self, query: str, all_documents: List[Dict[str, Any]], top_k: int = 5) -> List[Dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
@@ -293,37 +243,9 @@ class DifyQueryRetrieval:
|
|||||||
new_all_documents.append(to_dify_document_format(cur_doc_info))
|
new_all_documents.append(to_dify_document_format(cur_doc_info))
|
||||||
return new_all_documents
|
return new_all_documents
|
||||||
|
|
||||||
def get_datasets_by_classification(self, classification: Classification, software_name: str) -> List[str]:
|
|
||||||
if classification.vertical_classification=="软件问题" or classification.vertical_classification=="业务问题":
|
|
||||||
software_name_list = self.software_to_dataset_map.keys()
|
|
||||||
cur_software_name = ""
|
|
||||||
for software_name_info in software_name_list:
|
|
||||||
if software_name_info in software_name:
|
|
||||||
cur_software_name = software_name_info
|
|
||||||
break
|
|
||||||
if cur_software_name == "":
|
|
||||||
return self.software_to_dataset_map["其他"]
|
|
||||||
else:
|
|
||||||
return self.software_to_dataset_map[cur_software_name]
|
|
||||||
|
|
||||||
if classification.vertical_classification == "安装下载注册":
|
|
||||||
if classification.sub_classification in ["后缀名咨询", "软件锁类"]:
|
|
||||||
return ["下载安装注册(new)"]
|
|
||||||
elif classification.sub_classification == "安装下载类":
|
|
||||||
return []
|
|
||||||
elif classification.sub_classification == "问题排查":
|
|
||||||
return self.software_to_dataset_map["其他"]
|
|
||||||
|
|
||||||
return self.software_to_dataset_map["其他"]
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
dify_query_retrieval = DifyQueryRetrieval(api_key="dataset-skLjmPVonjHo119OWNf3kAmY", base_url="http://10.1.16.39/v1")
|
dify_query_retrieval = DifyQueryRetrieval(api_key="dataset-skLjmPVonjHo119OWNf3kAmY", base_url="http://10.1.16.39/v1")
|
||||||
# datasets = dify_query_retrieval.retrieve("配网工程计价通D3软件如何新建工程？", Classification(vertical_classification="软件问题", sub_classification="软件功能"), "配网工程计价通D3")
|
|
||||||
# datasets = dify_query_retrieval.retrieve_api("电力建设计价通软件如何批量修改设备价格？",
|
|
||||||
# ["电力建设计价通软件如何批量修改设备价格？"],
|
|
||||||
# ["电力建设计价通（2018）软件知识(new)"])
|
|
||||||
# print(json.dumps(datasets, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
# 测试异步API
|
# 测试异步API
|
||||||
async def test_async_api():
|
async def test_async_api():
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ class RetrieveRequest(BaseModel):
|
|||||||
original_query: str
|
original_query: str
|
||||||
query_list: str
|
query_list: str
|
||||||
data_set_list: str
|
data_set_list: str
|
||||||
|
query_expand_dict: dict | str = Field(default="{}")
|
||||||
|
|
||||||
# 创建FastAPI应用
|
# 创建FastAPI应用
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
@@ -87,13 +88,17 @@ async def retrieve(request: RetrieveRequest):
|
|||||||
# 解析查询列表和数据集列表
|
# 解析查询列表和数据集列表
|
||||||
query_list = request.query_list.split("<sub_query>")
|
query_list = request.query_list.split("<sub_query>")
|
||||||
data_set_list = request.data_set_list.split("<dataset>")
|
data_set_list = request.data_set_list.split("<dataset>")
|
||||||
|
if isinstance(request.query_expand_dict, str):
|
||||||
|
query_expand_dict = json.loads(request.query_expand_dict)
|
||||||
|
else:
|
||||||
|
query_expand_dict = request.query_expand_dict
|
||||||
# 调用异步检索方法
|
# 调用异步检索方法
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
results = await dify_query_retrieval.retrieve_api_async(
|
results = await dify_query_retrieval.retrieve_api_async(
|
||||||
request.original_query,
|
request.original_query,
|
||||||
query_list,
|
query_list,
|
||||||
data_set_list,
|
data_set_list,
|
||||||
|
query_expand_dict=query_expand_dict,
|
||||||
top_k=5
|
top_k=5
|
||||||
)
|
)
|
||||||
end_time = time.time()
|
end_time = time.time()
|
||||||
@@ -107,7 +112,7 @@ async def retrieve(request: RetrieveRequest):
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# 使用Uvicorn运行FastAPI应用
|
# 使用Uvicorn运行FastAPI应用
|
||||||
import uvicorn
|
import uvicorn
|
||||||
uvicorn.run("rag2_0.dify.DifyQueryRetrieval_api:app", host="0.0.0.0", port=8002, reload=False, workers=10, log_level="info")
|
uvicorn.run("rag2_0.dify.DifyQueryRetrieval_api:app", host="0.0.0.0", port=8002, reload=False, workers=1, log_level="info")
|
||||||
# # 使用uvicorn启动服务
|
# # 使用uvicorn启动服务
|
||||||
# import uvicorn
|
# import uvicorn
|
||||||
# uvicorn.run(
|
# uvicorn.run(
|
||||||
|
|||||||
Reference in New Issue
Block a user