删除重复的名词，指定问题改写时的同义词替换

2025-05-27 15:19:48 +08:00
parent 3dfa8c8a8a
commit 670de2f758
13 changed files with 17093 additions and 51385 deletions
@@ -169,7 +169,7 @@ def main():
    cur_path = os.path.dirname(__file__)
    input_dir = os.path.abspath(os.path.join(cur_path, '../../data/wiki_extracted_nouns'))
    output_path = os.path.join(cur_path, "..", "..", "data", "nouns", 'merged_nouns.json')
-    merger = TermMerger(input_dir=input_dir, output_path=output_path, max_workers=2)
+    merger = TermMerger(input_dir=input_dir, output_path=output_path, max_workers=20)
    merger.merge()

 if __name__ == "__main__":