合并Dev分支代码

This commit is contained in:
wanyaokun
2024-08-30 10:49:05 +08:00
parent e9ccd7db35
commit 73565b26e4
16 changed files with 486 additions and 409 deletions
@@ -6,7 +6,7 @@ from app.settings import init_settings
from app.engine.loaders import get_document_Types, get_documents,getFileCacahePath
from app.engine.vectordb import get_vector_store
from app.engine.generate import get_doc_store,run_pipeline,persist_storage
import tempfile
STORAGE_DIR = os.getenv("STORAGE_DIR", "storage")
@@ -25,31 +25,31 @@ class FileLoadService:
url = load_url,
data=response1.text
)
with open('example.zip','wb') as file:
tempFilePath:str = tempfile.gettempdir() + f"\\{str(uuid4())}.zip"
with open(tempFilePath,'wb') as file:
file.write(response2.content)
prjID = str(uuid4())
filePath = getFileCacahePath() + f'/Projects/{prjID}'
os.makedirs(filePath)
import zipfile
with zipfile.ZipFile('example.zip','r') as zip_File:
with zipfile.ZipFile(tempFilePath,'r') as zip_File:
for zip_info in zip_File.infolist():
zip_info.filename = zip_info.filename.encode('cp437').decode('gbk')
zip_File.extract(zip_info,filePath)
os.remove('example.zip')
os.remove(tempFilePath)
return f'Projects_{prjID}'
@staticmethod
def process_file(base64_content: str) -> List[str]:
docType = FileLoadService.store_and_parse_file(base64_content)
def process_file(base64_content: str) -> str:
prjFlag = FileLoadService.store_and_parse_file(base64_content)
#生成向量并持久化至本地
init_settings()
documents = get_documents(docType)
documents = get_documents(prjFlag)
for doc in documents:
doc.metadata["private"] = "false"
docstore = get_doc_store(docType)
vector_store = get_vector_store(docType)
docstore = get_doc_store(prjFlag)
vector_store = get_vector_store(prjFlag)
_ = run_pipeline(docstore, vector_store, documents)
persist_storage(docstore, vector_store)
return prjFlag