diff options
| author | Dennis Fink | 2026-10-30 20:13:42 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 20:13:42 +0100 |
| commit | 8fd569c6c0a39761f7ddda3c6e145d90534b4300 (patch) | |
| tree | f92209df1918fd5c9ac27efe25463afaef2a1a5f | |
| parent | a2992533873cb53acb052cdcd4288430b24af587 (diff) | |
| download | webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.tar.gz webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.zip | |
fix(receiver): recover stale Webmention verifications
Add a periodic recovery sweep for Webmentions left in received or
verifying states after worker crashes, lost queue entries, or
interrupted processing.
Use atomic lease-style updates so stale workers cannot overwrite newer
results, and index verification state for efficient recovery scans.
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 4 | ||||
| -rw-r--r-- | tests/cli/test_database.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_consumer.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 139 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 3 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py | 26 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 9 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 125 |
9 files changed, 298 insertions, 30 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 7dfdd7a..bf363b8 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -48,6 +48,10 @@ ALLOWED_HOSTNAMES = {"example.com", "www.example.com"} REQUEST_TIMEOUT = 5.0 # seconds; also used for outgoing requests MAX_REDIRECTS = 20 # also used for outgoing requests MAX_SOURCE_BYTES = 1_000_000 +# Requeue stuck verifications and lost queued tasks after 15 minutes. +VERIFICATION_TIMEOUT_SECONDS = 900 +# Huey cron expression: every five minutes (in UTC). +VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" # --------------------------------------------------------------------------- # Rate limiting diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index f25aade..1c36d1f 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -82,9 +82,15 @@ def test_create_db_builds_schema_and_records_latest_migration( with database_app.app_context(): inspector = inspect(DATABASE.engine) table_names = set(inspector.get_table_names()) - columns = {column["name"] for column in inspector.get_columns("sent_webmentions")} + columns = { + column["name"] for column in inspector.get_columns("sent_webmentions") + } + received_indexes = { + index["name"] for index in inspector.get_indexes("received_webmentions") + } assert {"alembic_version", "users", "received_webmentions"} <= table_names assert {"attempt_count", "next_retry_at"} <= columns + assert "ix_received_webmentions_status_updated_at" in received_indexes def test_create_db_refuses_existing_database_without_modifying_it( diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py index 6b79e37..dd93331 100644 --- a/tests/tasks/test_consumer.py +++ b/tests/tasks/test_consumer.py @@ -13,13 +13,14 @@ from flask import Flask from webmentions_ssg import HUEY -def test_consumer_registers_scanner_and_digest( +def test_consumer_registers_scanner_digest_and_verification_reaper( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: import webmentions_ssg app.config["SCANNER_SCHEDULE"] = "*/15 * * * *" app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *" + app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *" registered_tasks: list[object] = [] @@ -35,11 +36,12 @@ def test_consumer_registers_scanner_and_digest( monkeypatch.setattr(HUEY, "periodic_task", periodic_task) consumer = importlib.import_module("webmentions_ssg.tasks.consumer") - from webmentions_ssg.tasks import notifications, scanner + from webmentions_ssg.tasks import notifications, receiver, scanner assert consumer.huey is HUEY.huey - assert periodic_task.call_count == 2 + assert periodic_task.call_count == 3 assert registered_tasks == [ scanner.scan_sources, + receiver.recover_stale_webmentions, notifications.send_webmention_digest, ] diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 6858be9..4daca58 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -4,11 +4,13 @@ import logging import uuid +from datetime import UTC, datetime, timedelta from types import ModuleType from unittest.mock import Mock, patch import httpx import pytest +import sqlalchemy as sa from flask import Flask from webmentions_ssg import DATABASE as db @@ -530,6 +532,143 @@ def test_verify_webmention_persists_retryable_failure_and_reraises( assert get_webmention_state(app, identifier) == ("failed", str(exception)) +@patch("webmentions_ssg.tasks.receiver.source_mentions_target") +@pytest.mark.parametrize("status", ["verifying", "verified", "deleted"]) +def test_verify_webmention_skips_active_and_terminal_states( + source_mentions_target: Mock, app: Flask, receiver: ModuleType, status: str +) -> None: + identifier = create_webmention(app, status=status) + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == (status, None) + source_mentions_target.assert_not_called() + + +def set_webmention_updated_at( + app: Flask, identifier: uuid.UUID, when: datetime +) -> None: + with app.app_context(): + db.session.execute( + sa.update(ReceivedWebmention) + .where(ReceivedWebmention.uuid == identifier) + .values(updated_at=when) + ) + db.session.commit() + + +@pytest.mark.parametrize("status", ["verifying", "received"]) +def test_reaper_requeues_stale_verifications( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch, status: str +) -> None: + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 900 + identifier = create_webmention(app, status=status, failure_reason="Interrupted") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + assert receiver.recover_stale_webmentions() == 0 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.updated_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta( + seconds=10 + ) + + assert get_webmention_state(app, identifier) == ("received", None) + enqueue.assert_called_once_with(identifier) + + +def test_reaper_does_not_touch_recent_or_terminal_webmentions( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + recent_verifying = create_webmention(app, status="verifying") + recent_received = create_webmention( + app, source="https://source.example/received", status="received" + ) + old_verified = create_webmention( + app, source="https://source.example/verified", status="verified" + ) + set_webmention_updated_at(app, old_verified, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + + assert get_webmention_state(app, recent_verifying) == ("verifying", None) + assert get_webmention_state(app, recent_received) == ("received", None) + assert get_webmention_state(app, old_verified) == ("verified", None) + enqueue.assert_not_called() + + +def test_reaper_recovers_after_enqueue_failure( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app, status="verifying") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + monkeypatch.setattr(receiver, "verify_webmention", Mock(side_effect=RuntimeError)) + + with app.app_context(), pytest.raises(RuntimeError): + receiver.recover_stale_webmentions() + + assert get_webmention_state(app, identifier) == ("received", None) + + # The reaper can recover the row again if the process died after the commit + # but before Huey persisted the task. + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0 + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + + enqueue.assert_called_once_with(identifier) + + +def test_late_worker_cannot_overwrite_recovered_verification( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app) + original_verify = receiver.verify_webmention + + def concurrent_recovery(_source: str, _target: str) -> bool: + # The reaper supersedes this still-running attempt; a newer attempt + # subsequently completes before the original fetch returns. + app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0 + with app.app_context(): + with patch.object(receiver, "verify_webmention", Mock()) as enqueue: + assert receiver.recover_stale_webmentions() == 1 + enqueue.assert_called_once_with(identifier) + db.session.execute( + sa.update(ReceivedWebmention) + .where(ReceivedWebmention.uuid == identifier) + .values(status="verified", updated_at=datetime.now(UTC)) + ) + db.session.commit() + return False + + monkeypatch.setattr(receiver, "source_mentions_target", concurrent_recovery) + original_verify.call_local(identifier) + + assert get_webmention_state(app, identifier) == ("verified", None) + + +def test_reaper_requeues_and_verifies_integration( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app, status="verifying") + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + check = Mock(return_value=True) + monkeypatch.setattr(receiver, "source_mentions_target", check) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + + assert get_webmention_state(app, identifier) == ("verified", None) + check.assert_called_once_with(SOURCE_URL, TARGET_URL) + + def test_verify_webmention_ignores_unknown_identifier( receiver: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 3aec00d..e8288db 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -45,6 +45,9 @@ class DefaultConfig: MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 + # Requeue abandoned verification attempts after 15 minutes. + VERIFICATION_TIMEOUT_SECONDS = 900 + VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" SOURCE_DIRECTORY = None SOURCE_BASE_URL = None diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py new file mode 100644 index 0000000..cbb40ef --- /dev/null +++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py @@ -0,0 +1,26 @@ +"""Index received webmentions for expired verification recovery. + +Revision ID: c0272f25ce17 +Revises: 85f2b226ec99 +""" + +from alembic import op + +revision = "c0272f25ce17" +down_revision = "85f2b226ec99" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + "ix_received_webmentions_status_updated_at", + "received_webmentions", + ["status", "updated_at"], + ) + + +def downgrade(): + op.drop_index( + "ix_received_webmentions_status_updated_at", table_name="received_webmentions" + ) diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 4f688e5..58f95bc 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -17,6 +17,7 @@ from sqlalchemy import ( DateTime, Enum, ForeignKey, + Index, Integer, String, Text, @@ -116,6 +117,7 @@ class ReceivedWebmention(Base): __table_args__ = ( UniqueConstraint("source", "target", name="uq_webmention_source_target"), + Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) def __repr__(self) -> str: @@ -246,9 +248,7 @@ class SentWebmention(Base): status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) # Retry state belongs to the desired revision, not to the target forever. - attempt_count: Mapped[int] = mapped_column( - Integer(), nullable=False, default=0 - ) + attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) next_retry_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 559ef5d..afae6e0 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -14,8 +14,8 @@ def create_consumer() -> Huey: Create and configure the Huey consumer. The application is initialized, task modules are imported to register their - tasks, and the source scanner is registered with the configured periodic - schedule. + tasks, and periodic scanner, notification and verification recovery tasks + are registered. :return: Configured Huey instance. """ @@ -26,6 +26,11 @@ def create_consumer() -> Huey: schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) + recovery_schedule = crontab( + *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True + ) + HUEY.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions) + notification_schedule = crontab( *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index d195cf5..4e8df74 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -5,10 +5,12 @@ """Fetch and verify incoming Webmentions against their source pages.""" import uuid +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx import rfc3987 +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -241,17 +243,18 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: """ - Verify a received webmention and store its verification status. + Verify a received webmention and persist the result. - Temporary fetch failures are stored before being re-raised so that Huey can - retry the task. + The transition into ``verifying`` is an atomic claim. Its ``updated_at`` + timestamp acts as a lease version: if a reaper has reclaimed the row while + the fetch was in progress, the old worker cannot overwrite the new result. + Temporary fetch failures are persisted and re-raised for Huey's retries. :param webmention_uuid: Identifier of the received webmention to verify. :raises TemporaryFetchError: If fetching the source fails temporarily. :raises httpx.RequestError: If an HTTP request error occurs while fetching the source. """ - webmention = db.session.get(ReceivedWebmention, webmention_uuid) if webmention is None: @@ -260,31 +263,111 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - webmention.status = "verifying" - webmention.failure_reason = None + # Failed tasks may be retried by Huey. Completed and already-running tasks + # should not start another concurrent verification. + if webmention.status not in {"received", "failed"}: + return + + previous_status = webmention.status + previous_updated_at = webmention.updated_at + source, target = webmention.source, webmention.target + started_at = datetime.now(UTC) + + claimed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == previous_status, + ReceivedWebmention.updated_at == previous_updated_at, + ) + .values(status="verifying", failure_reason=None, updated_at=started_at) + ) + if claimed.rowcount != 1: + db.session.rollback() + return db.session.commit() + def finish(status: str, failure_reason: str | None) -> None: + # A timeout or a newer verification may have superseded this attempt. + # Do not let a late worker overwrite its state. + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == "verifying", + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() + try: - mentions_target = source_mentions_target(webmention.source, webmention.target) + mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - webmention.status = "deleted" - webmention.failure_reason = str(exc) + finish("deleted", str(exc)) except VerificationError as exc: - webmention.status = "failed" - webmention.failure_reason = str(exc) + finish("failed", str(exc)) except (TemporaryFetchError, httpx.RequestError) as exc: - webmention.status = "failed" - webmention.failure_reason = str(exc) or "Source could not be fetched" - db.session.commit() - - # Huey retries the task because the exception escapes. + finish("failed", str(exc) or "Source could not be fetched") raise else: if mentions_target: - webmention.status = "verified" - webmention.failure_reason = None + finish("verified", None) else: - webmention.status = "deleted" - webmention.failure_reason = "Source does not mention target" + finish("deleted", "Source does not mention target") - db.session.commit() + +def recover_stale_webmentions() -> int: + """ + Requeue verification attempts that were abandoned or lost from the queue. + + Recover both stale ``verifying`` rows (workers that died) and stale + ``received`` rows (tasks that were never enqueued or were lost). Conditional + updates prevent overlapping reapers from enqueueing the same row. Each + reclaimed row is committed *before* enqueueing, so the worker can claim it. + A crash between commit and enqueue is recovered on a later sweep. + + :return: Number of webmentions requeued. + """ + now = datetime.now(UTC) + cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( + sa.select( + ReceivedWebmention.uuid, + ReceivedWebmention.status, + ReceivedWebmention.updated_at, + ).where( + ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.updated_at <= cutoff, + ) + ).all() + + recovered = 0 + for identifier, status, updated_at in stale: + changed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == identifier, + ReceivedWebmention.status == status, + ReceivedWebmention.updated_at == updated_at, + ) + .values(status="received", failure_reason=None, updated_at=now) + ) + if changed.rowcount != 1: + db.session.rollback() + continue + + db.session.commit() + verify_webmention(identifier) + recovered += 1 + + if recovered: + current_app.logger.warning( + "Requeued %d stale received webmention verifications", recovered + ) + + return recovered |
