diff options
| author | Dennis Fink | 2026-10-30 20:13:42 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 20:13:42 +0100 |
| commit | 8fd569c6c0a39761f7ddda3c6e145d90534b4300 (patch) | |
| tree | f92209df1918fd5c9ac27efe25463afaef2a1a5f /tests | |
| parent | a2992533873cb53acb052cdcd4288430b24af587 (diff) | |
| download | webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.tar.gz webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.zip | |
fix(receiver): recover stale Webmention verifications
Add a periodic recovery sweep for Webmentions left in received or
verifying states after worker crashes, lost queue entries, or
interrupted processing.
Use atomic lease-style updates so stale workers cannot overwrite newer
results, and index verification state for efficient recovery scans.
Diffstat (limited to '')
| -rw-r--r-- | tests/cli/test_database.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_consumer.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 139 |
3 files changed, 151 insertions, 4 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index f25aade..1c36d1f 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -82,9 +82,15 @@ def test_create_db_builds_schema_and_records_latest_migration( with database_app.app_context(): inspector = inspect(DATABASE.engine) table_names = set(inspector.get_table_names()) - columns = {column["name"] for column in inspector.get_columns("sent_webmentions")} + columns = { + column["name"] for column in inspector.get_columns("sent_webmentions") + } + received_indexes = { + index["name"] for index in inspector.get_indexes("received_webmentions") + } assert {"alembic_version", "users", "received_webmentions"} <= table_names assert {"attempt_count", "next_retry_at"} <= columns + assert "ix_received_webmentions_status_updated_at" in received_indexes def test_create_db_refuses_existing_database_without_modifying_it( diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py index 6b79e37..dd93331 100644 --- a/tests/tasks/test_consumer.py +++ b/tests/tasks/test_consumer.py @@ -13,13 +13,14 @@ from flask import Flask from webmentions_ssg import HUEY -def test_consumer_registers_scanner_and_digest( +def test_consumer_registers_scanner_digest_and_verification_reaper( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: import webmentions_ssg app.config["SCANNER_SCHEDULE"] = "*/15 * * * *" app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *" + app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *" registered_tasks: list[object] = [] @@ -35,11 +36,12 @@ def test_consumer_registers_scanner_and_digest( monkeypatch.setattr(HUEY, "periodic_task", periodic_task) consumer = importlib.import_module("webmentions_ssg.tasks.consumer") - from webmentions_ssg.tasks import notifications, scanner + from webmentions_ssg.tasks import notifications, receiver, scanner assert consumer.huey is HUEY.huey - assert periodic_task.call_count == 2 + assert periodic_task.call_count == 3 assert registered_tasks == [ scanner.scan_sources, + receiver.recover_stale_webmentions, notifications.send_webmention_digest, ] diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 6858be9..4daca58 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -4,11 +4,13 @@ import logging import uuid +from datetime import UTC, datetime, timedelta from types import ModuleType from unittest.mock import Mock, patch import httpx import pytest +import sqlalchemy as sa from flask import Flask from webmentions_ssg import DATABASE as db @@ -530,6 +532,143 @@ def test_verify_webmention_persists_retryable_failure_and_reraises( assert get_webmention_state(app, identifier) == ("failed", str(exception)) +@patch("webmentions_ssg.tasks.receiver.source_mentions_target") +@pytest.mark.parametrize("status", ["verifying", "verified", "deleted"]) +def test_verify_webmention_skips_active_and_terminal_states( + source_mentions_target: Mock, app: Flask, receiver: ModuleType, status: str +) -> None: + identifier = create_webmention(app, status=status) + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == (status, None) + source_mentions_target.assert_not_called() + + +def set_webmention_updated_at( + app: Flask, identifier: uuid.UUID, when: datetime +) -> None: + with app.app_context(): + db.session.execute( + sa.update(ReceivedWebmention) + .where(ReceivedWebmention.uuid == identifier) + .values(updated_at=when) + ) + db.session.commit() + + +@pytest.mark.parametrize("status", ["verifying", "received"]) +def test_reaper_requeues_stale_verifications( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch, status: str +) -> None: + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 900 + identifier = create_webmention(app, status=status, failure_reason="Interrupted") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + assert receiver.recover_stale_webmentions() == 0 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.updated_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta( + seconds=10 + ) + + assert get_webmention_state(app, identifier) == ("received", None) + enqueue.assert_called_once_with(identifier) + + +def test_reaper_does_not_touch_recent_or_terminal_webmentions( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + recent_verifying = create_webmention(app, status="verifying") + recent_received = create_webmention( + app, source="https://source.example/received", status="received" + ) + old_verified = create_webmention( + app, source="https://source.example/verified", status="verified" + ) + set_webmention_updated_at(app, old_verified, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + + assert get_webmention_state(app, recent_verifying) == ("verifying", None) + assert get_webmention_state(app, recent_received) == ("received", None) + assert get_webmention_state(app, old_verified) == ("verified", None) + enqueue.assert_not_called() + + +def test_reaper_recovers_after_enqueue_failure( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app, status="verifying") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + monkeypatch.setattr(receiver, "verify_webmention", Mock(side_effect=RuntimeError)) + + with app.app_context(), pytest.raises(RuntimeError): + receiver.recover_stale_webmentions() + + assert get_webmention_state(app, identifier) == ("received", None) + + # The reaper can recover the row again if the process died after the commit + # but before Huey persisted the task. + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0 + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + + enqueue.assert_called_once_with(identifier) + + +def test_late_worker_cannot_overwrite_recovered_verification( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app) + original_verify = receiver.verify_webmention + + def concurrent_recovery(_source: str, _target: str) -> bool: + # The reaper supersedes this still-running attempt; a newer attempt + # subsequently completes before the original fetch returns. + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0 + with app.app_context(): + with patch.object(receiver, "verify_webmention", Mock()) as enqueue: + assert receiver.recover_stale_webmentions() == 1 + enqueue.assert_called_once_with(identifier) + db.session.execute( + sa.update(ReceivedWebmention) + .where(ReceivedWebmention.uuid == identifier) + .values(status="verified", updated_at=datetime.now(UTC)) + ) + db.session.commit() + return False + + monkeypatch.setattr(receiver, "source_mentions_target", concurrent_recovery) + original_verify.call_local(identifier) + + assert get_webmention_state(app, identifier) == ("verified", None) + + +def test_reaper_requeues_and_verifies_integration( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app, status="verifying") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + check = Mock(return_value=True) + monkeypatch.setattr(receiver, "source_mentions_target", check) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + + assert get_webmention_state(app, identifier) == ("verified", None) + check.assert_called_once_with(SOURCE_URL, TARGET_URL) + + def test_verify_webmention_ignores_unknown_identifier( receiver: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: |
