From 576a2ae7375f948e591ee40864a659a255f97650 Mon Sep 17 00:00:00 2001 From: chentianrui Date: Wed, 28 Aug 2024 18:12:28 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E8=AF=84?= =?UTF-8?q?=E4=BC=B0=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/test1/evaluator.py | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 backend/test1/evaluator.py diff --git a/backend/test1/evaluator.py b/backend/test1/evaluator.py new file mode 100644 index 0000000..c53084d --- /dev/null +++ b/backend/test1/evaluator.py @@ -0,0 +1,75 @@ +from app.observability import init_observability +from app.settings import init_settings +from dotenv import load_dotenv + +import nest_asyncio +nest_asyncio.apply() + +load_dotenv() + + +from llama_index.core.node_parser import SentenceSplitter +from llama_index.core import ( + VectorStoreIndex, + SimpleDirectoryReader, + Response, +) +from llama_index.core.evaluation import FaithfulnessEvaluator, DatasetGenerator + + + +init_settings() +init_observability() + +evaluator_qwen = FaithfulnessEvaluator() + +documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data() + +splitter = SentenceSplitter(chunk_size=512) + + +vector_index = VectorStoreIndex.from_documents( + documents, transformations=[splitter], +) + + +# # 运行评估 +# query_engine = vector_index.as_query_engine() +# response_vector = query_engine.query("工程监理费的金额是多少？") +# eval_result = evaluator_qwen.evaluate_response(response=response_vector) + +# print(response_vector) +# print(eval_result) + + +question_generator = DatasetGenerator.from_documents(documents) +eval_questions = question_generator.generate_questions_from_nodes(5) +print(eval_questions) + +import asyncio + +async def evaluate_query_engine_async(query_engine, questions): + c = [query_engine.aquery(q) for q in questions] + gathering_future = asyncio.gather(*c) + results = await gathering_future + print(results) + + total_correct = 0 + for r in results: + eval_result = ( + 1 if evaluator_qwen.evaluate_response(response=r).passing else 0 + ) + total_correct += eval_result + + return total_correct, len(results) + +def evaluate_query_engine(query_engine, questions): + loop = asyncio.get_event_loop() + correct, total = loop.run_until_complete(evaluate_query_engine_async(query_engine, questions)) + return correct, total + +# 使用 evaluate_query_engine 函数 +vector_query_engine = vector_index.as_query_engine() +correct, total = evaluate_query_engine(vector_query_engine, eval_questions[:5]) + +print(f"score: {correct}/{total}") \ No newline at end of file From 0db159ac895b8f4418c487ed0769e8edbecbe625 Mon Sep 17 00:00:00 2001 From: paituo <330435863@qq.com> Date: Thu, 29 Aug 2024 10:48:40 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E5=AD=90=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitmodules | 3 +++ webapp | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 webapp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..df2e940 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "webapp"] + path = webapp + url = https://git.97id.com/ly/webapp.git diff --git a/webapp b/webapp new file mode 160000 index 0000000..77dbc14 --- /dev/null +++ b/webapp @@ -0,0 +1 @@ +Subproject commit 77dbc14a64e6eb16409deda752d0aacef3862dd9 From aff1793c4ec7d0aa83cf0213299e3f089b279878 Mon Sep 17 00:00:00 2001 From: chentianrui Date: Thu, 29 Aug 2024 11:38:45 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=86=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E8=AF=84=E4=BC=B0=E8=84=9A=E6=9C=AC=E5=92=8C=E8=AF=84?= =?UTF-8?q?=E5=88=86=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/test1/evaluator.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/test1/evaluator.py b/backend/test1/evaluator.py index c53084d..fcefc4c 100644 --- a/backend/test1/evaluator.py +++ b/backend/test1/evaluator.py @@ -14,14 +14,20 @@ from llama_index.core import ( SimpleDirectoryReader, Response, ) -from llama_index.core.evaluation import FaithfulnessEvaluator, DatasetGenerator +from llama_index.core.evaluation import ( + FaithfulnessEvaluator, + DatasetGenerator, + CorrectnessEvaluator, + SemanticSimilarityEvaluator,) init_settings() init_observability() -evaluator_qwen = FaithfulnessEvaluator() +faith_evaluator_qwen = FaithfulnessEvaluator() #诚实度评测 +corr_evaluator_qwen = CorrectnessEvaluator() #准确率评测 +Seman_evaluator_qwen = SemanticSimilarityEvaluator()#嵌入相似度评估 documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data() @@ -52,12 +58,12 @@ async def evaluate_query_engine_async(query_engine, questions): c = [query_engine.aquery(q) for q in questions] gathering_future = asyncio.gather(*c) results = await gathering_future - print(results) + #print(results) total_correct = 0 for r in results: eval_result = ( - 1 if evaluator_qwen.evaluate_response(response=r).passing else 0 + 1 if faith_evaluator_qwen.evaluate_response(response=r).passing else 0 ) total_correct += eval_result From 2706cf9d5aafd43bd0eb42a5af523f538bff499c Mon Sep 17 00:00:00 2001 From: chentianrui Date: Thu, 29 Aug 2024 11:41:42 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/pyproject.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index de1fbbb..981083d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,7 +17,7 @@ aiostream = "^0.6.2" llama-index = "0.10.63" cachetools = "^5.3.3" protobuf = "4.25.4" -nltk = "^3.8.2" +nltk = "^3.9.1" jieba = "^0.42.1" #arize-phoenix = "^4.12.0" @@ -35,6 +35,7 @@ chroma="^0.2.0" llama-index-vector-stores-chroma = "^0.1.10" llama-index-readers-json = "^0.1.5" llama-index-retrievers-bm25 = "^0.2.2" +llama-index-experimental = "^0.2.0" duckduckgo_search = "^6.2.6" @@ -62,6 +63,12 @@ version = "^0.8" version = "0.0.7" + +[[tool.poetry.source]] +name = "mirrors" +url = "https://pypi.tuna.tsinghua.edu.cn/simple/" +priority = "default" + [build-system] requires = [ "poetry-core" ] build-backend = "poetry.core.masonry.api" \ No newline at end of file From de34c3938c21246b8caccac00f5b7af70d36a1fa Mon Sep 17 00:00:00 2001 From: chentianrui Date: Thu, 29 Aug 2024 12:02:53 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E8=AF=84=E4=BC=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/test1/ParamTuner.py | 107 ++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 backend/test1/ParamTuner.py diff --git a/backend/test1/ParamTuner.py b/backend/test1/ParamTuner.py new file mode 100644 index 0000000..a8bbb12 --- /dev/null +++ b/backend/test1/ParamTuner.py @@ -0,0 +1,107 @@ +import nest_asyncio +nest_asyncio.apply() +from llama_index.core import SimpleDirectoryReader +from llama_index.core.node_parser import SentenceSplitter +from llama_index.core import VectorStoreIndex +from llama_index.core.evaluation import ( + FaithfulnessEvaluator, + DatasetGenerator, + CorrectnessEvaluator, + SemanticSimilarityEvaluator, +) +from llama_index.experimental.param_tuner import ParamTuner +from llama_index.experimental.param_tuner.base import RunResult +from llama_index.llms.openai import OpenAI + +import asyncio + +# 初始化环境 +from app.observability import init_observability +from app.settings import init_settings +from dotenv import load_dotenv + +load_dotenv() +init_settings() +init_observability() + +# 读取文档 +documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data() + +# 辅助函数 +def _build_index(chunk_size, documents): + # 构建索引 + splitter = SentenceSplitter(chunk_size=chunk_size) + vector_index = VectorStoreIndex.from_documents( + documents, transformations=[splitter], + ) + return vector_index + +# 评估函数 +def evaluate_query_engine(query_engine, questions): + loop = asyncio.get_event_loop() + correct, total = loop.run_until_complete(_evaluate_query_engine_async(query_engine, questions)) + return correct, total + +async def _evaluate_query_engine_async(query_engine, questions): + c = [query_engine.aquery(q) for q in questions] + gathering_future = asyncio.gather(*c) + results = await gathering_future + + total_correct = 0 + for r in results: + eval_result = ( + 1 if FaithfulnessEvaluator().evaluate_response(response=r).passing else 0 + ) + total_correct += eval_result + + return total_correct, len(results) + +# 目标函数 +def objective_function(params_dict, documents, question_count): + chunk_size = params_dict["chunk_size"] + top_k = params_dict["top_k"] + temperature = params_dict["temperature"] + + # 构建索引 + vector_index = _build_index(chunk_size, documents) + + # 查询引擎 + query_engine = vector_index.as_query_engine( + similarity_top_k=top_k, temperature=temperature + ) + + # 生成问题 + question_generator = DatasetGenerator.from_documents(documents) + eval_questions = question_generator.generate_questions_from_nodes(question_count) + + # 评估查询引擎 + correct, total = evaluate_query_engine(query_engine, eval_questions) + + # 计算分数 + score = correct / total if total > 0 else 0 + return RunResult(score=score, params=params_dict) + +# 参数字典 +param_dict = { + "chunk_size": [512, 1024], + "top_k": [1, 5], + "temperature": [0.1, 1.0] +} + +# 创建 ParamTuner 实例 +param_tuner = ParamTuner( + param_fn=lambda params_dict: objective_function(params_dict, documents, 1), + param_dict=param_dict, + show_progress=True, +) + +# 调用 tune 方法 +results = param_tuner.tune() +best_result = results.best_run_result +best_top_k = best_result.params["top_k"] +best_chunk_size = best_result.params["chunk_size"] +best_temperature = best_result.params["temperature"] +print(f"Score: {best_result.score}") +print(f"Top-k: {best_top_k}") +print(f"Chunk size: {best_chunk_size}") +print(f"Temperature: {best_temperature}") \ No newline at end of file From 4a8c79e83dc12043b333654632447e3555880fa1 Mon Sep 17 00:00:00 2001 From: chentianrui Date: Thu, 29 Aug 2024 15:09:55 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=8F=82=E6=95=B0=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=92=88=E5=AF=B9=E9=97=AE=E9=A2=98=E5=81=9A=E5=87=BA=E4=BA=86?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/test1/ParamTuner.py | 67 +++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/backend/test1/ParamTuner.py b/backend/test1/ParamTuner.py index a8bbb12..5e31b57 100644 --- a/backend/test1/ParamTuner.py +++ b/backend/test1/ParamTuner.py @@ -27,6 +27,13 @@ init_observability() # 读取文档 documents = SimpleDirectoryReader("D:/LLM_model/text2sql/zjdataai-app-test/backend/data-test").load_data() +# 参数字典 +param_dict = { + "chunk_size": [512, 1024], + "top_k": [1, 5], + "temperature": [0.1, 1.0] +} + # 辅助函数 def _build_index(chunk_size, documents): # 构建索引 @@ -56,8 +63,18 @@ async def _evaluate_query_engine_async(query_engine, questions): return total_correct, len(results) + + +# 生成问题 +question_generator = DatasetGenerator.from_documents(documents) +eval_questions = question_generator.generate_questions_from_nodes(1) # 假设生成10个问题 + +# 打印生成的问题 +for i, q in enumerate(eval_questions, start=1): + print(f"问题 {i}: {q}") + # 目标函数 -def objective_function(params_dict, documents, question_count): +def objective_function(params_dict, documents, questions): chunk_size = params_dict["chunk_size"] top_k = params_dict["top_k"] temperature = params_dict["temperature"] @@ -70,27 +87,25 @@ def objective_function(params_dict, documents, question_count): similarity_top_k=top_k, temperature=temperature ) - # 生成问题 - question_generator = DatasetGenerator.from_documents(documents) - eval_questions = question_generator.generate_questions_from_nodes(question_count) - # 评估查询引擎 - correct, total = evaluate_query_engine(query_engine, eval_questions) + correct, total = 0, len(questions) + question_answers = [] # 添加列表来收集问题和答案 + for question in questions: + response = query_engine.query(question) + if response is not None: + question_answers.append((question, response.response)) + eval_result = FaithfulnessEvaluator().evaluate_response(response=response, query_str=question) + if eval_result.passing: + correct += 1 + # 计算分数 score = correct / total if total > 0 else 0 - return RunResult(score=score, params=params_dict) - -# 参数字典 -param_dict = { - "chunk_size": [512, 1024], - "top_k": [1, 5], - "temperature": [0.1, 1.0] -} + return RunResult(score=score, params=params_dict, question_answers=question_answers) # 创建 ParamTuner 实例 param_tuner = ParamTuner( - param_fn=lambda params_dict: objective_function(params_dict, documents, 1), + param_fn=lambda params_dict: objective_function(params_dict, documents, eval_questions), param_dict=param_dict, show_progress=True, ) @@ -101,7 +116,23 @@ best_result = results.best_run_result best_top_k = best_result.params["top_k"] best_chunk_size = best_result.params["chunk_size"] best_temperature = best_result.params["temperature"] -print(f"Score: {best_result.score}") +print(f"得分: {best_result.score}") print(f"Top-k: {best_top_k}") -print(f"Chunk size: {best_chunk_size}") -print(f"Temperature: {best_temperature}") \ No newline at end of file +print(f"文本块大小: {best_chunk_size}") +print(f"温度: {best_temperature}") + +# 使用最佳参数再次运行查询引擎，并打印问题与答案 +best_vector_index = _build_index(best_chunk_size, documents) +best_query_engine = best_vector_index.as_query_engine( + similarity_top_k=best_top_k, temperature=best_temperature +) + +best_question_answers = [] +for question in eval_questions: + response = best_query_engine.query(question) + if response is not None: + best_question_answers.append((question, response.response)) + +# 打印最佳参数下的问题与答案 +for i, (question, answer) in enumerate(best_question_answers, start=1): + print(f"最佳参数 - 问题 {i}: {question}\n答案: {answer}\n") \ No newline at end of file