chentianrui
41 lines
1.2 KiB
Python
41 lines
1.2 KiB
Python
import logging
|
|
# import yaml
|
|
from app.engine.loaders.file import FileLoaderConfig, get_file_documents
|
|
from app.engine.loaders.web import WebLoaderConfig, get_web_documents
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# 注释掉 load_configs 函数
|
|
# def load_configs():
|
|
# with open("config/loaders.yaml") as f:
|
|
# configs = yaml.safe_load(f)
|
|
# return configs
|
|
|
|
|
|
def get_documents():
|
|
documents = []
|
|
# 注释掉对 load_configs 的调用
|
|
# config = load_configs()
|
|
# if config is None or len(config.items()) == 0:
|
|
# return documents
|
|
|
|
# 使用一个空的 config 替代原有的配置加载逻辑
|
|
config = {}
|
|
|
|
for loader_type, loader_config in config.items():
|
|
logger.info(
|
|
f"Loading documents from loader: {loader_type}, config: {loader_config}"
|
|
)
|
|
|
|
loader_config = loader_config or []
|
|
match loader_type:
|
|
case "file":
|
|
document = get_file_documents(FileLoaderConfig(**loader_config))
|
|
case "web":
|
|
document = get_web_documents(WebLoaderConfig(**loader_config))
|
|
case _:
|
|
raise ValueError(f"Invalid loader type: {loader_type}")
|
|
documents.extend(document)
|
|
|
|
return documents
|