6.13提交，语义处理测试

2025-06-13 10:11:42 +08:00
parent 026f9bd70c
commit 89b0154217
11 changed files with 2481 additions and 0 deletions
@@ -0,0 +1,39 @@
+import os
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+
+with open("./data/data.txt", 'r', encoding='utf-8') as file:
+    txt_list = [line.strip() for line in file]
+
+embedding_path = "/data/Z_LLM_data/Embed_data/bge-m3"
+embeddings = HuggingFaceEmbeddings(model_name=embedding_path)
+
+faiss_archived = "./data/faiss_data/data"
+vectorstore_txt_faiss = FAISS.from_texts(txt_list, embeddings)
+vectorstore_txt_faiss.save_local(faiss_archived)
+
+retriever_txt_faiss1 = vectorstore_txt_faiss.as_retriever(search_kwargs={"k":3})
+retriever_txt_faiss2 = vectorstore_txt_faiss.as_retriever(
+    search_type="mmr",
+    search_kwargs={"k": 3,  # 检索结果
+                   "fetch_k": 1,  # 候选结果数量
+                   "lambda_mult": 0.5}  # 平衡指数，1为相关性；0为多样性
+)
+retriever_txt_faiss3 = vectorstore_txt_faiss.as_retriever(
+    search_type="similarity_score_threshold",
+    search_kwargs={"score_threshold": 0.5}
+)
+
+def intersection_of_three_lists(input_str):
+    list1 = retriever_txt_faiss1.invoke(input_str)
+    list2 = retriever_txt_faiss2.invoke(input_str)
+    list3 = retriever_txt_faiss3.invoke(input_str)
+    
+    def _intersection_of_three_lists(retrieval_results):
+        return [doc.page_content for doc in retrieval_results]
+        
+    list11 = _intersection_of_three_lists(list1)
+    list22 = _intersection_of_three_lists(list2)
+    list33 = _intersection_of_three_lists(list3)
+        
+    return list(set(list11) & set(list22) & set(list33))