From 54f19a20fc7643964120ffae5b955405e210a0de Mon Sep 17 00:00:00 2001 From: wanyaokun <12345678> Date: Mon, 9 Sep 2024 19:10:16 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4MarkDown=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E7=9A=84=E6=A0=BC=E5=BC=8F=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/loaders/markdownReader.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/app/engine/loaders/markdownReader.py b/backend/app/engine/loaders/markdownReader.py index e5e465d..bf30a16 100644 --- a/backend/app/engine/loaders/markdownReader.py +++ b/backend/app/engine/loaders/markdownReader.py @@ -27,17 +27,19 @@ class ChunkMarkdownReader(MarkdownReader): for line in lines: tokensNum += self._token_size(line) if tokensNum > self._chunkSize and len(current_lines) > 0: - markdown_tups.append((strTitle + strheader , "\n".join(current_lines))) + if len(markdown_tups) == 0: + markdown_tups.append((strTitle + strheader , "\n".join(current_lines))) + else: + markdown_tups.append((strheader , "\n".join(current_lines))) tokensNum = headerSize current_lines.clear() - current_lines.append(line) if line == '\n' or line == '\r': if tokensNum > self._chunkSize: raise ValueError('标题Token数大于chunkSize大小') strTitle = "\n".join(current_lines) - headerSize = headerSize + self._token_size(strTitle) + #headerSize = headerSize + self._token_size(strTitle) current_lines.clear() if line.startswith("|---"): @@ -46,7 +48,11 @@ class ChunkMarkdownReader(MarkdownReader): current_lines.clear() if len(current_lines) > 0: - markdown_tups.append((strTitle + strheader , "\n".join(current_lines))) + if len(markdown_tups) == 0: + markdown_tups.append((strTitle + strheader , "\n".join(current_lines))) + else: + markdown_tups.append((strheader , "\n".join(current_lines))) + return [ ( key if key is None else re.sub(r"#", "", key).strip(),