diff options
| author | Dennis Fink | 2026-10-30 12:04:22 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 12:04:22 +0100 |
| commit | d6ac8fb9768352fbb8a5633c6f357d877512bb5f (patch) | |
| tree | e649a37a7759b093739723a1e299dcc6487b3195 | |
| parent | 37bc3853dded34e3d1323874927979ce510e4910 (diff) | |
| download | webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.tar.gz webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.zip | |
fix(sender): isolate source parser failures
Prevent malformed pages or unexpected third-party parser exceptions from
aborting an entire source scan or causing previously known sources to be
treated as deleted.
Keep persistence failures visible so database errors still fail the
scan.
Diffstat (limited to '')
| -rw-r--r-- | tests/tasks/test_scanner.py | 124 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 14 |
2 files changed, 136 insertions, 2 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 5823e9e..8dee47d 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -4,6 +4,7 @@ from pathlib import Path from types import ModuleType +from typing import Any from unittest.mock import Mock from uuid import UUID @@ -1336,6 +1337,129 @@ def test_invalid_known_source_is_not_deleted( assert "Expected exactly one h-entry, found 0" in caplog.text +@pytest.mark.parametrize( + ("parser", "failure"), + ( + ("bs4", ValueError("BeautifulSoup failed")), + ("mf2py", RuntimeError("mf2py failed")), + ("mf2py", RecursionError("HTML nesting exceeded recursion limit")), + ), + ids=("bs4-error", "mf2py-error", "recursion-error"), +) +def test_unexpected_parser_error_does_not_abort_other_sources( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, + parser: str, + failure: Exception, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "bad", "<p>Cannot parse this page.</p>") + write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>') + + if parser == "bs4": + original_soup = scanner_module.BeautifulSoup + + def failing_soup(markup: bytes, features: str) -> BeautifulSoup: + if b"/bad/" in markup: + raise failure + return original_soup(markup, features) + + monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup) + else: + original_parse = scanner_module.mf2py.parse + + def failing_parse(*, doc: str, url: str) -> dict[str, Any]: + if "/bad/" in doc: + raise failure + return original_parse(doc=doc, url=url) + + monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse) + + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert [source.path for source in sources] == ["good/index.html"] + assert sources[0].revision == 1 + assert webmention is not None + assert webmention.target == "https://example.com/target" + assert queued == [webmention.uuid] + + assert "Unexpected error scanning source" in caplog.text + assert str(failure) in caplog.text + + +def test_unexpected_parser_error_does_not_delete_known_source( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "bad", "<p>Original bad page.</p>") + write_post(tmp_path, "good", "<p>Original good page.</p>") + + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + run_scan(app, scanner_module) + + write_post(tmp_path, "good", '<a href="https://example.com/new-target">Link</a>') + original_scan_source_file = scanner_module.scan_source_file + + def failing_scan_source_file(path: Path, **kwargs: Any) -> Any: + if path.parent.name == "bad": + raise RecursionError("nested page") + return original_scan_source_file(path, **kwargs) + + monkeypatch.setattr(scanner_module, "scan_source_file", failing_scan_source_file) + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = { + source.path: source for source in db.session.scalars(sa.select(Source)) + } + bad = sources["bad/index.html"] + good = sources["good/index.html"] + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert bad.revision == 1 + assert bad.deleted_at is None + assert good.revision == 2 + assert good.deleted_at is None + assert webmention is not None + assert webmention.target == "https://example.com/new-target" + assert queued == [webmention.uuid] + + +def test_unexpected_persistence_error_is_not_suppressed( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "example", "<p>Content</p>") + + monkeypatch.setattr( + scanner_module, + "create_source", + Mock(side_effect=RuntimeError("database failed")), + ) + + with pytest.raises(RuntimeError, match="database failed"): + run_scan(app, scanner_module) + + def test_scan_sources_accepts_valid_base_url( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 5ab2521..4bb6736 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -459,7 +459,18 @@ def scan_sources() -> None: base_url=base_url, ignored_hostnames=ignored_hostnames, ) + except (OSError, SourceScanError) as exc: + current_app.logger.warning("Could not scan source %s: %s", index_path, exc) + continue + except Exception: + # Third-party HTML parsers can raise unexpected exceptions (including + # RecursionError). A bad page must not prevent other sources being saved. + current_app.logger.exception( + "Unexpected error scanning source %s", index_path + ) + continue + try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) db.session.add(source) @@ -468,8 +479,7 @@ def scan_sources() -> None: sources_by_path[source.path] = source scanned_count += 1 - - except (OSError, SourceScanError) as exc: + except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) for path, source in sources_by_path.items(): |
