aboutsummaryrefslogtreecommitdiff
path: root/tests/tasks
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 12:04:22 +0100
committerDennis Fink2026-10-30 12:04:22 +0100
commitd6ac8fb9768352fbb8a5633c6f357d877512bb5f (patch)
treee649a37a7759b093739723a1e299dcc6487b3195 /tests/tasks
parent37bc3853dded34e3d1323874927979ce510e4910 (diff)
downloadwebmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.tar.gz
webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.zip
fix(sender): isolate source parser failures
Prevent malformed pages or unexpected third-party parser exceptions from aborting an entire source scan or causing previously known sources to be treated as deleted. Keep persistence failures visible so database errors still fail the scan.
Diffstat (limited to 'tests/tasks')
-rw-r--r--tests/tasks/test_scanner.py124
1 files changed, 124 insertions, 0 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 5823e9e..8dee47d 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -4,6 +4,7 @@
from pathlib import Path
from types import ModuleType
+from typing import Any
from unittest.mock import Mock
from uuid import UUID
@@ -1336,6 +1337,129 @@ def test_invalid_known_source_is_not_deleted(
assert "Expected exactly one h-entry, found 0" in caplog.text
+@pytest.mark.parametrize(
+ ("parser", "failure"),
+ (
+ ("bs4", ValueError("BeautifulSoup failed")),
+ ("mf2py", RuntimeError("mf2py failed")),
+ ("mf2py", RecursionError("HTML nesting exceeded recursion limit")),
+ ),
+ ids=("bs4-error", "mf2py-error", "recursion-error"),
+)
+def test_unexpected_parser_error_does_not_abort_other_sources(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ parser: str,
+ failure: Exception,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "bad", "<p>Cannot parse this page.</p>")
+ write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>')
+
+ if parser == "bs4":
+ original_soup = scanner_module.BeautifulSoup
+
+ def failing_soup(markup: bytes, features: str) -> BeautifulSoup:
+ if b"/bad/" in markup:
+ raise failure
+ return original_soup(markup, features)
+
+ monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup)
+ else:
+ original_parse = scanner_module.mf2py.parse
+
+ def failing_parse(*, doc: str, url: str) -> dict[str, Any]:
+ if "/bad/" in doc:
+ raise failure
+ return original_parse(doc=doc, url=url)
+
+ monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse)
+
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+
+ assert [source.path for source in sources] == ["good/index.html"]
+ assert sources[0].revision == 1
+ assert webmention is not None
+ assert webmention.target == "https://example.com/target"
+ assert queued == [webmention.uuid]
+
+ assert "Unexpected error scanning source" in caplog.text
+ assert str(failure) in caplog.text
+
+
+def test_unexpected_parser_error_does_not_delete_known_source(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "bad", "<p>Original bad page.</p>")
+ write_post(tmp_path, "good", "<p>Original good page.</p>")
+
+ monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None)
+ run_scan(app, scanner_module)
+
+ write_post(tmp_path, "good", '<a href="https://example.com/new-target">Link</a>')
+ original_scan_source_file = scanner_module.scan_source_file
+
+ def failing_scan_source_file(path: Path, **kwargs: Any) -> Any:
+ if path.parent.name == "bad":
+ raise RecursionError("nested page")
+ return original_scan_source_file(path, **kwargs)
+
+ monkeypatch.setattr(scanner_module, "scan_source_file", failing_scan_source_file)
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = {
+ source.path: source for source in db.session.scalars(sa.select(Source))
+ }
+ bad = sources["bad/index.html"]
+ good = sources["good/index.html"]
+ webmention = db.session.scalar(sa.select(SentWebmention))
+
+ assert bad.revision == 1
+ assert bad.deleted_at is None
+ assert good.revision == 2
+ assert good.deleted_at is None
+ assert webmention is not None
+ assert webmention.target == "https://example.com/new-target"
+ assert queued == [webmention.uuid]
+
+
+def test_unexpected_persistence_error_is_not_suppressed(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "example", "<p>Content</p>")
+
+ monkeypatch.setattr(
+ scanner_module,
+ "create_source",
+ Mock(side_effect=RuntimeError("database failed")),
+ )
+
+ with pytest.raises(RuntimeError, match="database failed"):
+ run_scan(app, scanner_module)
+
+
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None: