refactor: 统一 HTML Reader 的 parse 签名,使用文件路径参数
将所有 HTML Parser 的函数签名从接收 HTML 字符串改为接收文件路径, 与其他 Reader(PDF、DOCX 等)保持一致。 主要变更: - 修改 PARSERS 列表,移除 lambda 表达式,直接传递函数引用 - 在 HtmlReader.parse() 中统一管理临时文件(UTF-8 编码) - 每个 Parser 使用独立的临时文件副本,用完即清理 - 移除 download_and_parse() 方法,逻辑合并到 parse() 中 - 更新相关测试,改为直接传递文件路径 受影响的 Parser: - trafilatura.parse(html_content) -> parse(file_path) - domscribe.parse(html_content) -> parse(file_path) - markitdown.parse(html_content, temp_file_path) -> parse(file_path) - html2text.parse(html_content) -> parse(file_path)
This commit is contained in:
@@ -1,39 +1,24 @@
|
||||
"""使用 MarkItDown 解析 HTML"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
def parse(html_content: str, temp_file_path: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""使用 MarkItDown 解析 HTML"""
|
||||
def parse(file_path: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""使用 MarkItDown 解析 HTML 文件"""
|
||||
try:
|
||||
from markitdown import MarkItDown
|
||||
except ImportError:
|
||||
return None, "MarkItDown 库未安装"
|
||||
|
||||
try:
|
||||
input_path = temp_file_path
|
||||
if not input_path or not os.path.exists(input_path):
|
||||
# 创建临时文件
|
||||
fd, input_path = tempfile.mkstemp(suffix='.html')
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||||
f.write(html_content)
|
||||
|
||||
md = MarkItDown()
|
||||
result = md.convert(
|
||||
input_path,
|
||||
file_path,
|
||||
heading_style="ATX",
|
||||
strip=["img", "script", "style", "noscript"],
|
||||
)
|
||||
markdown_content = result.text_content
|
||||
|
||||
if not temp_file_path:
|
||||
try:
|
||||
os.unlink(input_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not markdown_content.strip():
|
||||
return None, "解析内容为空"
|
||||
return markdown_content, None
|
||||
|
||||
Reference in New Issue
Block a user