lanyuanxiaoyao/lyxy-document
1
0
Fork 0
Code Releases Activity

refactor: 重构 Reader 内部工具函数到独立模块

Browse Source 
- 新增 scripts/readers/_utils.py 作为 Reader 内部共享工具模块
- 将 parse_with_markitdown 等函数从 core/markdown.py 迁移到 _utils.py
- 函数重命名：parse_with_xxx → parse_via_xxx，_unstructured_elements_to_markdown → convert_unstructured_to_markdown
- 更新 17 个 Reader 实现文件的 import 路径
- 从 core/__init__.py 移除已迁移函数的导出
- 新增测试文件 tests/test_readers/test_utils.py
- 新增 spec 文档 openspec/specs/reader-internal-utils/spec.md

这次重构明确了模块边界：core/ 提供公共 API，readers/_utils.py 提供 Reader 内部工具
This commit is contained in:
兰缘小妖
2026-03-09 00:56:05 +08:00
parent b80c635f07
commit 1aea561277
22 changed files with 536 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/core/__init__.py
View File

@@ -8,18 +8,12 @@ from .exceptions import (
DownloadError,
)
from .markdown import (
parse_with_markitdown,
parse_with_docling,
build_markdown_table,
flush_list_stack,
safe_open_zip,
normalize_markdown_whitespace,
remove_markdown_images,
get_heading_level,
extract_titles,
extract_title_content,
search_markdown,
_unstructured_elements_to_markdown,
)
from .parser import parse_input, process_content, output_result
@@ -29,18 +23,12 @@ __all__ = [
"ReaderNotFoundError",
"ParseError",
"DownloadError",
"parse_with_markitdown",
"parse_with_docling",
"build_markdown_table",
"flush_list_stack",
"safe_open_zip",
"normalize_markdown_whitespace",
"remove_markdown_images",
"get_heading_level",
"extract_titles",
"extract_title_content",
"search_markdown",
"_unstructured_elements_to_markdown",
"parse_input",
"process_content",
"output_result",

140
scripts/core/markdown.py
View File

@@ -1,94 +1,11 @@
"""Markdown 后处理模块，包含所有格式共享的工具函数。"""
"""Markdown 后处理模块，包含 Markdown 格式化的工具函数。"""
import re
import zipfile
from pathlib import Path
from typing import List, Optional, Tuple
from typing import List, Optional
IMAGE_PATTERN = re.compile(r"!\[[^\]]*\]\([^)]+\)")
_CONSECUTIVE_BLANK_LINES = re.compile(r"\n{3,}")
# unstructured 噪声匹配: pptx 中的 RGB 颜色值（如 "R:255 G:128 B:0"）
_RGB_PATTERN = re.compile(r"^R:\d+\s+G:\d+\s+B:\d+$")
# unstructured 噪声匹配: 破折号页码（如 "— 3 —"）
_PAGE_NUMBER_PATTERN = re.compile(r"^—\s*\d+\s*—$")
def parse_with_markitdown(
file_path: str,
) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析文件"""
try:
from markitdown import MarkItDown
md = MarkItDown()
result = md.convert(file_path)
if not result.text_content.strip():
return None, "文档为空"
return result.text_content, None
except ImportError:
return None, "MarkItDown 库未安装"
except Exception as e:
return None, f"MarkItDown 解析失败: {str(e)}"
def parse_with_docling(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 docling 库解析文件"""
try:
from docling.document_converter import DocumentConverter
except ImportError:
return None, "docling 库未安装"
try:
converter = DocumentConverter()
result = converter.convert(file_path)
markdown_content = result.document.export_to_markdown()
if not markdown_content.strip():
return None, "文档为空"
return markdown_content, None
except Exception as e:
return None, f"docling 解析失败: {str(e)}"
def build_markdown_table(rows_data: List[List[str]]) -> str:
"""将二维列表转换为 Markdown 表格格式"""
if not rows_data or not rows_data[0]:
return ""
md_lines = []
for i, row_data in enumerate(rows_data):
row_text = [cell if cell else "" for cell in row_data]
md_lines.append("| " + " | ".join(row_text) + " |")
if i == 0:
md_lines.append("| " + " | ".join(["---"] * len(row_text)) + " |")
return "\n".join(md_lines) + "\n\n"
def flush_list_stack(list_stack: List[str], target: List[str]) -> None:
"""将列表堆栈中的非空项添加到目标列表并清空堆栈"""
for item in list_stack:
if item:
target.append(item + "\n")
list_stack.clear()
def safe_open_zip(zip_file: zipfile.ZipFile, name: str) -> Optional[zipfile.ZipExtFile]:
"""安全地从 ZipFile 中打开文件，防止路径遍历攻击"""
if not name:
return None
try:
normalized = Path(name).as_posix()
# 检查是否包含父目录引用
if ".." in Path(normalized).parts:
return None
# 检查是否为绝对路径
if Path(normalized).is_absolute():
return None
return zip_file.open(name)
except (ValueError, OSError):
return None
def normalize_markdown_whitespace(content: str) -> str:
"""规范化 Markdown 空白字符，保留单行空行"""
@@ -235,56 +152,3 @@ def search_markdown(
results.append("\n".join(result_lines))
return "\n---\n".join(results)
def _unstructured_elements_to_markdown(
elements: list, trust_titles: bool = True
) -> str:
"""将 unstructured 解析出的元素列表转换为 Markdown 文本"""
try:
import markdownify as md_lib
from unstructured.documents.elements import (
Footer,
Header,
Image,
ListItem,
PageBreak,
PageNumber,
Table,
Title,
)
except ImportError:
return "\n\n".join(
el.text for el in elements if hasattr(el, "text") and el.text and el.text.strip()
)
skip_types = (Header, Footer, PageBreak, PageNumber)
parts = []
for el in elements:
if isinstance(el, skip_types):
continue
text = el.text.strip() if hasattr(el, "text") else str(el).strip()
if not text or _RGB_PATTERN.match(text) or _PAGE_NUMBER_PATTERN.match(text):
continue
if isinstance(el, Table):
html = getattr(el.metadata, "text_as_html", None)
if html:
parts.append(md_lib.markdownify(html, strip=["img"]).strip())
else:
parts.append(str(el))
elif isinstance(el, Title) and trust_titles:
depth = getattr(el.metadata, "category_depth", None) or 1
depth = min(max(depth, 1), 4)
parts.append(f"{'#' * depth} {text}")
elif isinstance(el, ListItem):
parts.append(f"- {text}")
elif isinstance(el, Image):
path = getattr(el.metadata, "image_path", None) or ""
if path:
parts.append(f"![image]({path})")
else:
parts.append(text)
return "\n\n".join(parts)

207
scripts/readers/_utils.py Normal file
View File

@@ -0,0 +1,207 @@
"""Reader 内部共享工具模块。
此模块包含各 reader 实现共享的内部工具函数，仅供 readers 包内部使用。
"""
import re
import zipfile
from pathlib import Path
from typing import List, Optional, Tuple
# ============================================================================
# 通用解析器包装函数
# ============================================================================
def parse_via_markitdown(
file_path: str,
) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析文件。
Args:
file_path: 文件路径
Returns:
(markdown_content, error_message): 成功时 (content, None)，失败时 (None, error)
"""
try:
from markitdown import MarkItDown
md = MarkItDown()
result = md.convert(file_path)
if not result.text_content.strip():
return None, "文档为空"
return result.text_content, None
except ImportError:
return None, "MarkItDown 库未安装"
except Exception as e:
return None, f"MarkItDown 解析失败: {str(e)}"
def parse_via_docling(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 docling 库解析文件。
Args:
file_path: 文件路径
Returns:
(markdown_content, error_message): 成功时 (content, None)，失败时 (None, error)
"""
try:
from docling.document_converter import DocumentConverter
except ImportError:
return None, "docling 库未安装"
try:
converter = DocumentConverter()
result = converter.convert(file_path)
markdown_content = result.document.export_to_markdown()
if not markdown_content.strip():
return None, "文档为空"
return markdown_content, None
except Exception as e:
return None, f"docling 解析失败: {str(e)}"
# ============================================================================
# 格式化工具
# ============================================================================
def build_markdown_table(rows_data: List[List[str]]) -> str:
"""将二维列表格式化为 Markdown 表格。
Args:
rows_data: 二维列表，第一行为表头
Returns:
Markdown 格式的表格字符串
"""
if not rows_data or not rows_data[0]:
return ""
md_lines = []
for i, row_data in enumerate(rows_data):
row_text = [cell if cell else "" for cell in row_data]
md_lines.append("| " + " | ".join(row_text) + " |")
if i == 0:
md_lines.append("| " + " | ".join(["---"] * len(row_text)) + " |")
return "\n".join(md_lines) + "\n\n"
# ============================================================================
# 列表处理工具
# ============================================================================
def flush_list_stack(list_stack: List[str], target: List[str]) -> None:
"""将列表堆栈中的非空项添加到目标列表并清空堆栈。
用于处理嵌套列表的格式化输出。
Args:
list_stack: 列表堆栈
target: 目标列表
"""
for item in list_stack:
if item:
target.append(item + "\n")
list_stack.clear()
# ============================================================================
# ZIP 文件安全处理
# ============================================================================
def safe_open_zip(zip_file: zipfile.ZipFile, name: str) -> Optional[zipfile.ZipExtFile]:
"""安全地从 ZipFile 中打开文件，防止路径遍历攻击。
Args:
zip_file: ZipFile 对象
name: 文件名
Returns:
ZipExtFile 对象，如果路径不安全则返回 None
"""
if not name:
return None
try:
normalized = Path(name).as_posix()
# 检查是否包含父目录引用
if ".." in Path(normalized).parts:
return None
# 检查是否为绝对路径
if Path(normalized).is_absolute():
return None
return zip_file.open(name)
except (ValueError, OSError, KeyError):
return None
# ============================================================================
# unstructured 库相关
# ============================================================================
# unstructured 噪声匹配模式
_UNSTRUCTURED_RGB_PATTERN = re.compile(r"^R:\d+\s+G:\d+\s+B:\d+$")
_UNSTRUCTURED_PAGE_NUMBER_PATTERN = re.compile(r"^—\s*\d+\s*—$")
def convert_unstructured_to_markdown(
elements: list, trust_titles: bool = True
) -> str:
"""将 unstructured 解析出的元素列表转换为 Markdown 文本。
Args:
elements: unstructured 解析的元素列表
trust_titles: 是否信任 unstructured 的标题检测
Returns:
Markdown 格式的文本
"""
try:
import markdownify as md_lib
from unstructured.documents.elements import (
Footer,
Header,
Image,
ListItem,
PageBreak,
PageNumber,
Table,
Title,
)
except ImportError:
return "\n\n".join(
el.text for el in elements if hasattr(el, "text") and el.text and el.text.strip()
)
skip_types = (Header, Footer, PageBreak, PageNumber)
parts = []
for el in elements:
if isinstance(el, skip_types):
continue
text = el.text.strip() if hasattr(el, "text") else str(el).strip()
if not text or _UNSTRUCTURED_RGB_PATTERN.match(text) or _UNSTRUCTURED_PAGE_NUMBER_PATTERN.match(text):
continue
if isinstance(el, Table):
html = getattr(el.metadata, "text_as_html", None)
if html:
parts.append(md_lib.markdownify(html, strip=["img"]).strip())
else:
parts.append(str(el))
elif isinstance(el, Title) and trust_titles:
depth = getattr(el.metadata, "category_depth", None) or 1
depth = min(max(depth, 1), 4)
parts.append(f"{'#' * depth} {text}")
elif isinstance(el, ListItem):
parts.append(f"- {text}")
elif isinstance(el, Image):
path = getattr(el.metadata, "image_path", None) or ""
if path:
parts.append(f"![image]({path})")
else:
parts.append(text)
return "\n\n".join(parts)

4
scripts/readers/docx/docling.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_docling
from scripts.readers._utils import parse_via_docling
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 docling 库解析 DOCX 文件"""
return parse_with_docling(file_path)
return parse_via_docling(file_path)

4
scripts/readers/docx/markitdown.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_markitdown
from scripts.readers._utils import parse_via_markitdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析 DOCX 文件"""
return parse_with_markitdown(file_path)
return parse_via_markitdown(file_path)

2
scripts/readers/docx/native_xml.py
View File

@@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET
import zipfile
from typing import Any, Dict, List, Optional, Tuple
from scripts.core import build_markdown_table, safe_open_zip
from scripts.readers._utils import build_markdown_table, safe_open_zip
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:

2
scripts/readers/docx/python_docx.py
View File

@@ -2,7 +2,7 @@
from typing import Any, List, Optional, Tuple
from scripts.core import build_markdown_table
from scripts.readers._utils import build_markdown_table
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:

4
scripts/readers/docx/unstructured.py
View File

@@ -2,7 +2,7 @@
from typing import Optional, Tuple
from scripts.core import _unstructured_elements_to_markdown
from scripts.readers._utils import convert_unstructured_to_markdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
@@ -14,7 +14,7 @@ def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
try:
elements = partition_docx(filename=file_path, infer_table_structure=True)
content = _unstructured_elements_to_markdown(elements)
content = convert_unstructured_to_markdown(elements)
if not content.strip():
return None, "文档为空"
return content, None

4
scripts/readers/pdf/markitdown.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_markitdown
from scripts.readers._utils import parse_via_markitdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析 PDF 文件"""
return parse_with_markitdown(file_path)
return parse_via_markitdown(file_path)

4
scripts/readers/pdf/unstructured.py
View File

@@ -2,7 +2,7 @@
from typing import Optional, Tuple
from scripts.core import _unstructured_elements_to_markdown
from scripts.readers._utils import convert_unstructured_to_markdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
@@ -20,7 +20,7 @@ def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
languages=["chi_sim"],
)
# fast 策略不做版面分析，Title 类型标注不可靠
content = _unstructured_elements_to_markdown(elements, trust_titles=False)
content = convert_unstructured_to_markdown(elements, trust_titles=False)
if not content.strip():
return None, "文档为空"
return content, None

4
scripts/readers/pdf/unstructured_ocr.py
View File

@@ -2,7 +2,7 @@
from typing import Optional, Tuple
from scripts.core import _unstructured_elements_to_markdown
from scripts.readers._utils import convert_unstructured_to_markdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
@@ -26,7 +26,7 @@ def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
ocr_agent=OCR_AGENT_PADDLE,
table_ocr_agent=OCR_AGENT_PADDLE,
)
content = _unstructured_elements_to_markdown(elements, trust_titles=True)
content = convert_unstructured_to_markdown(elements, trust_titles=True)
if not content.strip():
return None, "文档为空"
return content, None

4
scripts/readers/pptx/docling.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_docling
from scripts.readers._utils import parse_via_docling
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 docling 库解析 PPTX 文件"""
return parse_with_docling(file_path)
return parse_via_docling(file_path)

4
scripts/readers/pptx/markitdown.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_markitdown
from scripts.readers._utils import parse_via_markitdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析 PPTX 文件"""
return parse_with_markitdown(file_path)
return parse_via_markitdown(file_path)

2
scripts/readers/pptx/native_xml.py
View File

@@ -5,7 +5,7 @@ import xml.etree.ElementTree as ET
import zipfile
from typing import Any, List, Optional, Tuple
from scripts.core import build_markdown_table, flush_list_stack
from scripts.readers._utils import build_markdown_table, flush_list_stack
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:

2
scripts/readers/pptx/python_pptx.py
View File

@@ -2,7 +2,7 @@
from typing import Any, List, Optional, Tuple
from scripts.core import build_markdown_table, flush_list_stack
from scripts.readers._utils import build_markdown_table, flush_list_stack
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:

4
scripts/readers/pptx/unstructured.py
View File

@@ -2,7 +2,7 @@
from typing import Optional, Tuple
from scripts.core import _unstructured_elements_to_markdown
from scripts.readers._utils import convert_unstructured_to_markdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
@@ -16,7 +16,7 @@ def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
elements = partition_pptx(
filename=file_path, infer_table_structure=True, include_metadata=True
)
content = _unstructured_elements_to_markdown(elements)
content = convert_unstructured_to_markdown(elements)
if not content.strip():
return None, "文档为空"
return content, None

4
scripts/readers/xlsx/docling.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_docling
from scripts.readers._utils import parse_via_docling
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 docling 库解析 XLSX 文件"""
return parse_with_docling(file_path)
return parse_via_docling(file_path)

4
scripts/readers/xlsx/markitdown.py
View File

@@ -2,9 +2,9 @@
from typing import Optional, Tuple
from scripts.core import parse_with_markitdown
from scripts.readers._utils import parse_via_markitdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
"""使用 MarkItDown 库解析 XLSX 文件"""
return parse_with_markitdown(file_path)
return parse_via_markitdown(file_path)

2
scripts/readers/xlsx/native_xml.py
View File

@@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET
import zipfile
from typing import List, Optional, Tuple
from scripts.core import build_markdown_table, safe_open_zip
from scripts.readers._utils import build_markdown_table, safe_open_zip
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:

4
scripts/readers/xlsx/unstructured.py
View File

@@ -2,7 +2,7 @@
from typing import Optional, Tuple
from scripts.core import _unstructured_elements_to_markdown
from scripts.readers._utils import convert_unstructured_to_markdown
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
@@ -14,7 +14,7 @@ def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
try:
elements = partition_xlsx(filename=file_path, infer_table_structure=True)
content = _unstructured_elements_to_markdown(elements)
content = convert_unstructured_to_markdown(elements)
if not content.strip():
return None, "文档为空"
return content, None