aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 12:04:22 +0100
committerDennis Fink2026-10-30 12:04:22 +0100
commitd6ac8fb9768352fbb8a5633c6f357d877512bb5f (patch)
treee649a37a7759b093739723a1e299dcc6487b3195
parent37bc3853dded34e3d1323874927979ce510e4910 (diff)
downloadwebmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.tar.gz
webmentions-ssg-d6ac8fb9768352fbb8a5633c6f357d877512bb5f.zip
fix(sender): isolate source parser failures
Prevent malformed pages or unexpected third-party parser exceptions from aborting an entire source scan or causing previously known sources to be treated as deleted. Keep persistence failures visible so database errors still fail the scan.
Diffstat (limited to '')
-rw-r--r--tests/tasks/test_scanner.py124
-rw-r--r--webmentions_ssg/tasks/scanner.py14
2 files changed, 136 insertions, 2 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 5823e9e..8dee47d 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -4,6 +4,7 @@
from pathlib import Path
from types import ModuleType
+from typing import Any
from unittest.mock import Mock
from uuid import UUID
@@ -1336,6 +1337,129 @@ def test_invalid_known_source_is_not_deleted(
assert "Expected exactly one h-entry, found 0" in caplog.text
+@pytest.mark.parametrize(
+ ("parser", "failure"),
+ (
+ ("bs4", ValueError("BeautifulSoup failed")),
+ ("mf2py", RuntimeError("mf2py failed")),
+ ("mf2py", RecursionError("HTML nesting exceeded recursion limit")),
+ ),
+ ids=("bs4-error", "mf2py-error", "recursion-error"),
+)
+def test_unexpected_parser_error_does_not_abort_other_sources(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ parser: str,
+ failure: Exception,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "bad", "<p>Cannot parse this page.</p>")
+ write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>')
+
+ if parser == "bs4":
+ original_soup = scanner_module.BeautifulSoup
+
+ def failing_soup(markup: bytes, features: str) -> BeautifulSoup:
+ if b"/bad/" in markup:
+ raise failure
+ return original_soup(markup, features)
+
+ monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup)
+ else:
+ original_parse = scanner_module.mf2py.parse
+
+ def failing_parse(*, doc: str, url: str) -> dict[str, Any]:
+ if "/bad/" in doc:
+ raise failure
+ return original_parse(doc=doc, url=url)
+
+ monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse)
+
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+
+ assert [source.path for source in sources] == ["good/index.html"]
+ assert sources[0].revision == 1
+ assert webmention is not None
+ assert webmention.target == "https://example.com/target"
+ assert queued == [webmention.uuid]
+
+ assert "Unexpected error scanning source" in caplog.text
+ assert str(failure) in caplog.text
+
+
+def test_unexpected_parser_error_does_not_delete_known_source(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "bad", "<p>Original bad page.</p>")
+ write_post(tmp_path, "good", "<p>Original good page.</p>")
+
+ monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None)
+ run_scan(app, scanner_module)
+
+ write_post(tmp_path, "good", '<a href="https://example.com/new-target">Link</a>')
+ original_scan_source_file = scanner_module.scan_source_file
+
+ def failing_scan_source_file(path: Path, **kwargs: Any) -> Any:
+ if path.parent.name == "bad":
+ raise RecursionError("nested page")
+ return original_scan_source_file(path, **kwargs)
+
+ monkeypatch.setattr(scanner_module, "scan_source_file", failing_scan_source_file)
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = {
+ source.path: source for source in db.session.scalars(sa.select(Source))
+ }
+ bad = sources["bad/index.html"]
+ good = sources["good/index.html"]
+ webmention = db.session.scalar(sa.select(SentWebmention))
+
+ assert bad.revision == 1
+ assert bad.deleted_at is None
+ assert good.revision == 2
+ assert good.deleted_at is None
+ assert webmention is not None
+ assert webmention.target == "https://example.com/new-target"
+ assert queued == [webmention.uuid]
+
+
+def test_unexpected_persistence_error_is_not_suppressed(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(tmp_path, "example", "<p>Content</p>")
+
+ monkeypatch.setattr(
+ scanner_module,
+ "create_source",
+ Mock(side_effect=RuntimeError("database failed")),
+ )
+
+ with pytest.raises(RuntimeError, match="database failed"):
+ run_scan(app, scanner_module)
+
+
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 5ab2521..4bb6736 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -459,7 +459,18 @@ def scan_sources() -> None:
base_url=base_url,
ignored_hostnames=ignored_hostnames,
)
+ except (OSError, SourceScanError) as exc:
+ current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
+ continue
+ except Exception:
+ # Third-party HTML parsers can raise unexpected exceptions (including
+ # RecursionError). A bad page must not prevent other sources being saved.
+ current_app.logger.exception(
+ "Unexpected error scanning source %s", index_path
+ )
+ continue
+ try:
if (source := sources_by_path.get(scanned.path)) is None:
source = create_source(scanned, scan_time)
db.session.add(source)
@@ -468,8 +479,7 @@ def scan_sources() -> None:
sources_by_path[source.path] = source
scanned_count += 1
-
- except (OSError, SourceScanError) as exc:
+ except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
for path, source in sources_by_path.items():