diff options
| author | Dennis Fink | 2026-10-30 12:04:22 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 12:04:22 +0100 |
| commit | d6ac8fb9768352fbb8a5633c6f357d877512bb5f (patch) | |
| tree | e649a37a7759b093739723a1e299dcc6487b3195 | |
| parent | 37bc3853dded34e3d1323874927979ce510e4910 (diff) | |
| download | webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.tar.gz webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.zip | |
fix(sender): isolate source parser failures
Prevent malformed pages or unexpected third-party parser exceptions from
aborting an entire source scan or causing previously known sources to be
treated as deleted.
Keep persistence failures visible so database errors still fail the
scan.
| -rw-r--r-- | tests/tasks/test_scanner.py | 124 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 14 |
2 files changed, 136 insertions, 2 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 5823e9e..8dee47d 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -4,6 +4,7 @@ from pathlib import Path from types import ModuleType +from typing import Any from unittest.mock import Mock from uuid import UUID @@ -1336,6 +1337,129 @@ def test_invalid_known_source_is_not_deleted( assert "Expected exactly one h-entry, found 0" in caplog.text +@pytest.mark.parametrize( + ("parser", "failure"), + ( + ("bs4", ValueError("BeautifulSoup failed")), + ("mf2py", RuntimeError("mf2py failed")), + ("mf2py", RecursionError("HTML nesting exceeded recursion limit")), + ), + ids=("bs4-error", "mf2py-error", "recursion-error"), +) +def test_unexpected_parser_error_does_not_abort_other_sources( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, + parser: str, + failure: Exception, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "bad", "<p>Cannot parse this page.</p>") + write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>') + + if parser == "bs4": + original_soup = scanner_module.BeautifulSoup + + def failing_soup(markup: bytes, features: str) -> BeautifulSoup: + if b"/bad/" in markup: + raise failure + return original_soup(markup, features) + + monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup) + else: + original_parse = scanner_module.mf2py.parse + + def failing_parse(*, doc: str, url: str) -> dict[str, Any]: + if "/bad/" in doc: + raise failure + return original_parse(doc=doc, url=url) + + monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse) + + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert [source.path for source in sources] == ["good/index.html"] + assert sources[0].revision == 1 + assert webmention is not None + assert webmention.target == "https://example.com/target" + assert queued == [webmention.uuid] + + assert "Unexpected error scanning source" in caplog.text + assert str(failure) in caplog.text + + +def test_unexpected_parser_error_does_not_delete_known_source( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "bad", "<p>Original bad page.</p>") + write_post(tmp_path, "good", "<p>Original good page.</p>") + + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + run_scan(app, scanner_module) + + write_post(tmp_path, "good", '<a href="https://example.com/new-target">Link</a>') + original_scan_source_file = scanner_module.scan_source_file + + def failing_scan_source_file(path: Path, **kwargs: Any) -> Any: + if path.parent.name == "bad": + raise RecursionError("nested page") + return original_scan_source_file(path, **kwargs) + + monkeypatch.setattr(scanner_module, "scan_source_file", failing_scan_source_file) + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = { + source.path: source for source in db.session.scalars(sa.select(Source)) + } + bad = sources["bad/index.html"] + good = sources["good/index.html"] + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert bad.revision == 1 + assert bad.deleted_at is None + assert good.revision == 2 + assert good.deleted_at is None + assert webmention is not None + assert webmention.target == "https://example.com/new-target" + assert queued == [webmention.uuid] + + +def test_unexpected_persistence_error_is_not_suppressed( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "example", "<p>Content</p>") + + monkeypatch.setattr( + scanner_module, + "create_source", + Mock(side_effect=RuntimeError("database failed")), + ) + + with pytest.raises(RuntimeError, match="database failed"): + run_scan(app, scanner_module) + + def test_scan_sources_accepts_valid_base_url( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 5ab2521..4bb6736 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -459,7 +459,18 @@ def scan_sources() -> None: base_url=base_url, ignored_hostnames=ignored_hostnames, ) + except (OSError, SourceScanError) as exc: + current_app.logger.warning("Could not scan source %s: %s", index_path, exc) + continue + except Exception: + # Third-party HTML parsers can raise unexpected exceptions (including + # RecursionError). A bad page must not prevent other sources being saved. + current_app.logger.exception( + "Unexpected error scanning source %s", index_path + ) + continue + try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) db.session.add(source) @@ -468,8 +479,7 @@ def scan_sources() -> None: sources_by_path[source.path] = source scanned_count += 1 - - except (OSError, SourceScanError) as exc: + except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) for path, source in sources_by_path.items(): |
