lanyuanxiaoyao/lyxy-document
1
0
Fork 0
Code Releases Activity

refactor: 统一 HTML Reader 的 parse 签名，使用文件路径参数

Browse Source 
将所有 HTML Parser 的函数签名从接收 HTML 字符串改为接收文件路径，
与其他 Reader（PDF、DOCX 等）保持一致。

主要变更：
- 修改 PARSERS 列表，移除 lambda 表达式，直接传递函数引用
- 在 HtmlReader.parse() 中统一管理临时文件（UTF-8 编码）
- 每个 Parser 使用独立的临时文件副本，用完即清理
- 移除 download_and_parse() 方法，逻辑合并到 parse() 中
- 更新相关测试，改为直接传递文件路径

受影响的 Parser：
- trafilatura.parse(html_content) -> parse(file_path)
- domscribe.parse(html_content) -> parse(file_path)
- markitdown.parse(html_content, temp_file_path) -> parse(file_path)
- html2text.parse(html_content) -> parse(file_path)
This commit is contained in:
兰缘小妖
2026-03-09 00:05:23 +08:00
parent 09904aefdc
commit 2b81dd49fe
10 changed files with 132 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tests/test_readers/test_html/test_consistency.py
View File

@@ -25,20 +25,16 @@ class TestHtmlReadersConsistency:
</html>
""")
# 读取 HTML 内容
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
parsers = [
("html2text", lambda c: html2text.parse(c)),
("markitdown", lambda c: markitdown.parse(c, file_path)),
("trafilatura", lambda c: trafilatura.parse(c)),
("domscribe", lambda c: domscribe.parse(c)),
("html2text", html2text.parse),
("markitdown", markitdown.parse),
("trafilatura", trafilatura.parse),
("domscribe", domscribe.parse),
]
successful_results = []
for name, parser in parsers:
content, error = parser(html_content)
content, error = parser(file_path)
if content is not None and content.strip():
successful_results.append((name, content))

25
tests/test_readers/test_html/test_domscribe_html.py
View File

@@ -10,36 +10,27 @@ class TestDomscribeHtmlReaderParse:
def test_normal_file(self, temp_html):
"""测试正常 HTML 文件解析。"""
file_path = temp_html(content="<h1>标题</h1><p>段落内容</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = domscribe.parse(html_content)
content, error = domscribe.parse(file_path)
if content is not None:
assert "标题" in content or "段落" in content
def test_file_not_exists(self, tmp_path):
"""测试文件不存在的情况。"""
html_content = "<p>测试</p>"
content, error = domscribe.parse(html_content)
assert content is not None or error is not None
non_existent_path = str(tmp_path / "non_existent.html")
content, error = domscribe.parse(non_existent_path)
assert content is None
# 如果库未安装，也会返回 None，但错误信息不同
assert error is not None
def test_empty_file(self, temp_html):
"""测试空 HTML 文件。"""
file_path = temp_html(content="<html><body></body></html>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = domscribe.parse(html_content)
content, error = domscribe.parse(file_path)
assert content is None or content.strip() == ""
def test_corrupted_file(self, temp_html, tmp_path):
"""测试损坏的 HTML 文件。"""
html_content = "\xff\xfe\x00\x00"
content, error = domscribe.parse(html_content)
def test_special_chars(self, temp_html):
"""测试特殊字符处理。"""
file_path = temp_html(content="<p>中文测试 😀 ©®</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = domscribe.parse(html_content)
content, error = domscribe.parse(file_path)
if content is not None:
assert "中文" in content or "测试" in content

26
tests/test_readers/test_html/test_markitdown_html.py
View File

@@ -10,38 +10,26 @@ class TestMarkitdownHtmlReaderParse:
def test_normal_file(self, temp_html):
"""测试正常 HTML 文件解析。"""
file_path = temp_html(content="<h1>标题</h1><p>段落内容</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = markitdown.parse(html_content, file_path)
content, error = markitdown.parse(file_path)
if content is not None:
assert "标题" in content or "段落" in content
def test_file_not_exists(self, tmp_path):
"""测试文件不存在的情况。"""
html_content = "<p>测试</p>"
content, error = markitdown.parse(html_content, None)
# markitdown 应该能解析内容
assert content is not None or error is not None
non_existent_path = str(tmp_path / "non_existent.html")
content, error = markitdown.parse(non_existent_path)
# markitdown 库自己会处理文件不存在的情况
assert content is None or error is not None
def test_empty_file(self, temp_html):
"""测试空 HTML 文件。"""
file_path = temp_html(content="<html><body></body></html>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = markitdown.parse(html_content, file_path)
content, error = markitdown.parse(file_path)
assert content is None or content.strip() == ""
def test_corrupted_file(self, temp_html, tmp_path):
"""测试损坏的 HTML 文件。"""
html_content = "\xff\xfe\x00\x00"
content, error = markitdown.parse(html_content, None)
# HTML 解析器通常比较宽容，可能仍能解析
def test_special_chars(self, temp_html):
"""测试特殊字符处理。"""
file_path = temp_html(content="<p>中文测试 😀 ©®</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = markitdown.parse(html_content, file_path)
content, error = markitdown.parse(file_path)
if content is not None:
assert "中文" in content or "测试" in content

25
tests/test_readers/test_html/test_trafilatura_html.py
View File

@@ -10,36 +10,27 @@ class TestTrafilaturaHtmlReaderParse:
def test_normal_file(self, temp_html):
"""测试正常 HTML 文件解析。"""
file_path = temp_html(content="<h1>标题</h1><p>段落内容</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = trafilatura.parse(html_content)
content, error = trafilatura.parse(file_path)
if content is not None:
assert "标题" in content or "段落" in content
def test_file_not_exists(self, tmp_path):
"""测试文件不存在的情况。"""
html_content = "<p>测试</p>"
content, error = trafilatura.parse(html_content)
assert content is not None or error is not None
non_existent_path = str(tmp_path / "non_existent.html")
content, error = trafilatura.parse(non_existent_path)
assert content is None
# 如果库未安装，也会返回 None，但错误信息不同
assert error is not None
def test_empty_file(self, temp_html):
"""测试空 HTML 文件。"""
file_path = temp_html(content="<html><body></body></html>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = trafilatura.parse(html_content)
content, error = trafilatura.parse(file_path)
assert content is None or content.strip() == ""
def test_corrupted_file(self, temp_html, tmp_path):
"""测试损坏的 HTML 文件。"""
html_content = "\xff\xfe\x00\x00"
content, error = trafilatura.parse(html_content)
def test_special_chars(self, temp_html):
"""测试特殊字符处理。"""
file_path = temp_html(content="<p>中文测试 😀 ©®</p>")
with open(file_path, 'r', encoding='utf-8') as f:
html_content = f.read()
content, error = trafilatura.parse(html_content)
content, error = trafilatura.parse(file_path)
if content is not None:
assert "中文" in content or "测试" in content