Merge pull request '增加新的前端子模块' (#4) from dev into main

Reviewed-on: #4

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "webapp"]
+	path = webapp
+	url = https://git.97id.com/ly/webapp.git

backend/app/api/routers/app.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+import time
 from typing import Dict, List, Any, Optional, AsyncGenerator
 
 from aiostream import stream
@@ -12,19 +13,16 @@ from llama_index.core.callbacks import CBEventType
 from llama_index.core.chat_engine.types import StreamingAgentChatResponse
 from llama_index.core.tools import ToolOutput
 from pydantic import BaseModel
-
-from app.api.routers.events import EventCallbackHandler
-from app.api.routers.request.base import userMng, conversations,message
-from app.api.routers.request.models import ChatRequestData
+from app.api.routers.request.base import userMng, conversations,message,parameter
+from app.api.routers.request.models import ChatRequestData,ChatFileUploadRequest
 from app.engine import get_chat_engine
 import uuid
 
 logger = logging.getLogger("uvicorn")
 
+api_router = r = APIRouter()
 v1_router = v = APIRouter()
 
-default_conversation_id = '82e8417f-2c3b-4bb5-ab22-2ad318bbd29a'
-
 class ChatCallbackEvent(BaseModel):
     event_type: CBEventType
     payload: Optional[Dict[str, Any]] = None
@@ -113,11 +111,11 @@ class ChatEventCallbackHandler(BaseCallbackHandler):
     ):
         """Initialize the base callback handler."""
         ignored_events = [
-            CBEventType.CHUNKING,
-            CBEventType.NODE_PARSING,
-            CBEventType.EMBEDDING,
-            CBEventType.LLM,
-            CBEventType.TEMPLATING,
+            # CBEventType.CHUNKING,
+            # CBEventType.NODE_PARSING,
+            # CBEventType.EMBEDDING,
+            # CBEventType.LLM,
+            # CBEventType.TEMPLATING,
         ]
         super().__init__(ignored_events, ignored_events)
         self._aqueue = asyncio.Queue()
@@ -129,6 +127,8 @@ class ChatEventCallbackHandler(BaseCallbackHandler):
             event_id: str = "",
             **kwargs: Any,
     ) -> str:
+        logger.info("event_start:{} type:{} payload:{}\n".format(event_id, event_type, payload))
+
         event = ChatCallbackEvent(event_id=event_id, event_type=event_type, payload=payload)
         if event.to_response() is not None:
             self._aqueue.put_nowait(event)
@@ -140,12 +140,14 @@ class ChatEventCallbackHandler(BaseCallbackHandler):
             event_id: str = "",
             **kwargs: Any,
     ) -> None:
+        logger.info("event_end:{} type:{} payload:{}\n".format(event_id, event_type, payload))
         event = ChatCallbackEvent(event_id=event_id, event_type=event_type, payload=payload)
         if event.to_response() is not None:
             self._aqueue.put_nowait(event)
 
     def start_trace(self, trace_id: Optional[str] = None) -> None:
         """No-op."""
+        logger.info("trace_start:{}\n".format(trace_id))
 
     def end_trace(
             self,
@@ -153,6 +155,7 @@ class ChatEventCallbackHandler(BaseCallbackHandler):
             trace_map: Optional[Dict[str, List[str]]] = None,
     ) -> None:
         """No-op."""
+        logger.info("trace_end:{} trace_map:{}\n".format(trace_id, trace_map))
 
     async def async_event_gen(self) -> AsyncGenerator[ChatCallbackEvent, None]:
         while not self._aqueue.empty() or not self.is_done:
@@ -174,7 +177,7 @@ class DifyChatResponseEvent(BaseModel):
     event: str
     conversation_id: str 
     message_id: str 
-    created_at: int = 1724406492
+    created_at: int = int(time.time())
     task_id: str 
 
 class Workflow_started_DifyChatResponseEvent(DifyChatResponseEvent):
@@ -191,7 +194,7 @@ class Workflow_started_DifyChatResponseEvent(DifyChatResponseEvent):
                 "sys.conversation_id":  args['conversation_id'],
                 "sys.user_id": args['use_id']
             },
-            "created_at": 1724406492
+            "created_at": int(time.time())
         }
         args['event'] = 'workflow_started'
         super().__init__(**args)
@@ -217,8 +220,8 @@ class Workflow_finished_DifyChatResponseEvent(DifyChatResponseEvent):
 			"id": str(uuid.uuid4()),
 			"user": args['use_id']
 		    },
-		    "created_at": 1724406492,
-		    "finished_at": 1724406528,
+		    "created_at": int(time.time()),
+		    "finished_at": int(time.time()),
 		    "files": []
         }
         super().__init__(**args)
@@ -240,26 +243,26 @@ class MessageEnd_DifyChatResponseEvent(DifyChatResponseEvent):
         super().__init__(**args)
         
 class ChatStreamResponse(StreamingResponse):
-    TEXT_PREFIX = "data:"
-    DATA_PREFIX = "data:"
+    TEXT_PREFIX = "data: "
+    DATA_PREFIX = "data: "
 
     @classmethod
     def convert_text(cls, token: str):
         # Escape newlines and double quotes to avoid breaking the stream
-        token = json.dumps(token)
+        #token = json.dumps(token)
 
         #return f"data: {{"event": "message", "conversation_id": "80d85523-de92-4b9d-aca0-c48a5eacb068", "message_id": "16a06b1b-a89b-49c0-bc15-123bd999f6d6", "created_at": 1724406492, "task_id": "802f3064-030d-42ac-a882-0e1293712d04", "id": "16a06b1b-a89b-49c0-bc15-123bd999f6d6", "answer": "{token}"}}"
-        return ""
+        return "\n"
 
     @classmethod
     def convert_data(cls, data: dict):
         data_str = json.dumps(data)
-        return f"{cls.DATA_PREFIX}{data_str}\n"
+        return f"{cls.DATA_PREFIX}{data_str}\n\n"
 
     @classmethod
     def convert_event(cls, event: DifyChatResponseEvent):
         data_str = json.dumps(event.dict())
-        return f"{cls.DATA_PREFIX}{data_str}\n"
+        return f"{cls.DATA_PREFIX}{data_str}\n\n"
 
     def __init__(
             self,
@@ -315,7 +318,7 @@ class ChatStreamResponse(StreamingResponse):
             async for event in event_handler.async_event_gen():
                 event_response = event.to_response()
                 if event_response is not None:
-                    yield ChatStreamResponse.convert_data(event_response)
+                    yield ChatStreamResponse.convert_text("")
 
         combine = stream.merge(_chat_response_generator(), _event_generator())
         is_stream_started = False
@@ -341,15 +344,17 @@ class ChatStreamResponse(StreamingResponse):
                 if await request.is_disconnected():
                     break
 
+
+
 @v.post("/chat-messages")
 async def post_conversations(request: Request, data: ChatRequestData):
     userMng.findNoExistCreate(data.user)
-    data.conversation_id = default_conversation_id if data.conversation_id is None else data.conversation_id
+    data.conversation_id = data.conversation_id if data.conversation_id else str(uuid.uuid4())
 
     conversaObj = conversations()
-    conversationinfo = conversaObj.get(data.user, data.conversation_id)
+    conversationinfo = conversaObj.get(data.conversation_id)
     if conversationinfo is None:
-        conversationinfo = conversaObj.add(data.user, "新建会话", data.conversation_id)
+        conversationinfo = conversaObj.add(data.conversation_id, data.user, "新建会话")
 
     # 生成聊天参数
     last_message_content = ChatMessage.from_str(data.query)
@@ -371,9 +376,16 @@ async def post_conversations(request: Request, data: ChatRequestData):
 
 @v.get("/messages")
 async def query_messages(user:str, conversation_id:str):
-    conversation_id = default_conversation_id if conversation_id is None else conversation_id
+    #conversation_id = default_conversation_id if conversation_id is None else conversation_id
     datas = []
     records = message().gets(user,conversation_id)
+    if records is None:
+        return {
+            "limit": 20,
+            "has_more": False,
+            "data": []
+        }
+
     for record in records:
         res = record.dict()
         res["message_files"] = []
@@ -392,11 +404,29 @@ async def query_messages(user:str, conversation_id:str):
     }  
     
 @v.post("/conversations/{itemid}/name")
-async def post_conversations(user:str):
-    pass
+async def post_conversations(request: Request,itemid:str,params:Dict[str,Any]):
+    consaObj = conversations()
+    consaObj.rename(itemid,'知识问答')
+    cond = {
+        'id':itemid,
+        'user_id':params['user']
+    }
+    results = consaObj.query(**cond)
+    if len(results) > 0:
+        res = results[0]
+        return {
+            "id": res['id'],
+            "name": res['name'],
+            "inputs": res['inputs'],
+            "status": res['status'],
+            "introduction": res['introduction'],
+            "created_at": res['created_at'],
+            #"工程位置"
+        }
+    return 'null'
 
 @v.get("/conversations")
-async def query_conversations(user:str):
+async def query_conversations(user:str, first_id:str = None, limit:str = None, pinned:str = None):
     user_id = '' if user is None else user
     userMng.findNoExistCreate(user_id)
 
@@ -405,3 +435,53 @@ async def query_conversations(user:str):
         "has_more": False,
         "data": conversations().gets(user_id)
     }    
+
+@v.get("/parameters")
+async def query_parameters(user:str):
+    params = parameter().get(user)
+    if len(params) == 0:
+        params = {
+                "opening_statement": "您好，我是配网D3造价软件小助手，您可以问我有关配网造价软件的相关问题！",
+                "suggested_questions": [],
+                "suggested_questions_after_answer": {
+                    "enabled": False
+                },
+                "speech_to_text": {
+                    "enabled": False
+                },
+                "text_to_speech": {
+                    "enabled": False,
+                    "language": "",
+                    "voice": ""
+                },
+                "retriever_resource": {
+                    "enabled": True
+                },
+                "annotation_reply": {
+                    "enabled": False
+                },
+                "more_like_this": {
+                    "enabled": False
+                },
+                "user_input_form": [],
+                "sensitive_word_avoidance": {
+                    "enabled": False
+                },
+                "file_upload": {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "transfer_methods": [
+                            "remote_url"
+                        ]
+                    }
+                },
+                "system_parameters": {
+                    "image_file_size_limit": "10"
+                }
+            }  
+    return params
+
+@r.post("")
+def upload_file(request: ChatFileUploadRequest) -> List[str]:
+    pass

backend/app/api/routers/request/base.py

@@ -18,13 +18,13 @@ class conversations:
 
         return datas
 
-    def get(self,user_id:str,id:str = ''):
-        records = dbManage.query(self._tableName,user_id = user_id,id=id)
+    def get(self, id:str):
+        records = dbManage.query(self._tableName, id=id)
         if len(records) >0:
             return records[0]
         return None
 
-    def add(self,user_id:str,name:str,id:str = ''):
+    def add(self,id:str, user_id:str, name:str):
         template = BaseConfig.ConversationCfg
         template['id'] = id
         template['user_id'] = user_id
@@ -35,10 +35,17 @@ class conversations:
     def delete(self,id:str):
         dbManage.delete(self._tableName,id=id)
     
-    def rename(self,id:str):
-        data = {'name':''}
+    def rename(self,id:str,name:str):
+        data = {'name':name}
         dbManage.update(self._tableName,data,id=id)
 
+    def query(self,**condition):
+        results = []
+        records = dbManage.query(self._tableName,**condition)
+        for record in records:
+            results.append(record.dict())
+        return results
+
 class user:
     def __init__(self) -> None:
         self._tableName = 'user'
@@ -83,22 +90,7 @@ class parameter:
             key = record['name']
             value = record['value']
             data[key] = value
-        
-        return {
-            'opening_statement':data['opening_statement'],
-            'suggested_questions':data['suggested_questions'],
-            'suggested_questions_after_answer':data['suggested_questions_after_answer'],
-            'speech_to_text':data['speech_to_text'],
-            'text_to_speech':data['text_to_speech'],
-            'retriever_resource':data['retriever_resource'],
-            'annotation_reply':data['annotation_reply'],
-            'more_like_this':data['more_like_this'],
-            'user_input_form':data['user_input_form'],
-            'sensitive_word_avoidance':data['sensitive_word_avoidance'],
-            'file_upload':data['file_upload'],
-            'system_parameters':data['system_parameters'],
-            'opening_statement':data['opening_statement'],
-        }
+        return data
 
     def set(self,user_id:str):
         dbManage.addRecord(self._tableName,{})

backend/app/api/routers/request/dbOrm.py

@@ -20,6 +20,14 @@ class ConversationOrm(Base):
     introduction = Column(String)
     created_at = Column(Integer)
 
+    def update(self,data:Dict[str,Any]):
+        if 'name' in data:
+            self.name = data['name']
+
+
+
+
+
 class UserOrm(Base):
     __tablename__ = "user"
 
@@ -143,14 +151,20 @@ class DBManager:
             session.commit()
     
     def update(self,tableName:str,data:Dict[str,Any],**filter):
+        if not self.exist(tableName):
+            return
         session = self.SessionLocal()
         ormCls = self._get_orm(tableName)
         if ormCls is None:
             return 
-        record = session.query(ormCls).filter_by(**filter).first()
-        if record is not None:
-            record.update(data)
-            session.commit()    
+        if len(filter) > 0:
+            records = session.query(ormCls).filter_by(**filter).all()
+        else:
+            records = session.query(ormCls).all()
+        for record in records:
+            if record is not None:
+                record.update(data)
+                session.commit()    
 
     def query(self,tableName:str,**filter):
         session = self.SessionLocal()

backend/app/api/routers/request/models.py

@@ -1,6 +1,5 @@
 
 from typing import Dict, Any
-
 from pydantic import BaseModel
 
 
@@ -11,3 +10,6 @@ class ChatRequestData(BaseModel):
     response_mode: str
     files: Any
     conversation_id: str = None
+
+class ChatFileUploadRequest(BaseModel):
+    base64: str

backend/app/engine/loaders/__init__.py

@@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 
 def load_configs():
-    with open("config/loaders.yaml") as f:
+    with open("config/loaders.yaml",encoding='UTF-8') as f:
         configs = yaml.safe_load(f)
     return configs
 

backend/app/engine/tools/__init__.py

@@ -1,10 +1,9 @@
-import os
-import yaml
-import json
 import importlib
-from cachetools import cached, LRUCache
-from llama_index.core.tools.tool_spec.base import BaseToolSpec
+import os
+
+import yaml
 from llama_index.core.tools.function_tool import FunctionTool
+from llama_index.core.tools.tool_spec.base import BaseToolSpec
 
 
 class ToolType:
@@ -46,7 +45,7 @@ class ToolFactory:
     def from_env() -> list[FunctionTool]:
         tools = []
         if os.path.exists("config/tools.yaml"):
-            with open("config/tools.yaml", "r") as f:
+            with open("config/tools.yaml", "r", encoding='UTF-8') as f:
                 tool_configs = yaml.safe_load(f)
                 if tool_configs != None and len(tool_configs.items()) != 0:
                     for tool_type, config_entries in tool_configs.items():