aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 20:13:42 +0100
committerDennis Fink2026-10-30 20:13:42 +0100
commit8fd569c6c0a39761f7ddda3c6e145d90534b4300 (patch)
treef92209df1918fd5c9ac27efe25463afaef2a1a5f /tests
parenta2992533873cb53acb052cdcd4288430b24af587 (diff)
downloadwebmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.tar.gz
webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.zip
fix(receiver): recover stale Webmention verifications
Add a periodic recovery sweep for Webmentions left in received or verifying states after worker crashes, lost queue entries, or interrupted processing. Use atomic lease-style updates so stale workers cannot overwrite newer results, and index verification state for efficient recovery scans.
Diffstat (limited to '')
-rw-r--r--tests/cli/test_database.py8
-rw-r--r--tests/tasks/test_consumer.py8
-rw-r--r--tests/tasks/test_receiver.py139
3 files changed, 151 insertions, 4 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
index f25aade..1c36d1f 100644
--- a/tests/cli/test_database.py
+++ b/tests/cli/test_database.py
@@ -82,9 +82,15 @@ def test_create_db_builds_schema_and_records_latest_migration(
with database_app.app_context():
inspector = inspect(DATABASE.engine)
table_names = set(inspector.get_table_names())
- columns = {column["name"] for column in inspector.get_columns("sent_webmentions")}
+ columns = {
+ column["name"] for column in inspector.get_columns("sent_webmentions")
+ }
+ received_indexes = {
+ index["name"] for index in inspector.get_indexes("received_webmentions")
+ }
assert {"alembic_version", "users", "received_webmentions"} <= table_names
assert {"attempt_count", "next_retry_at"} <= columns
+ assert "ix_received_webmentions_status_updated_at" in received_indexes
def test_create_db_refuses_existing_database_without_modifying_it(
diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py
index 6b79e37..dd93331 100644
--- a/tests/tasks/test_consumer.py
+++ b/tests/tasks/test_consumer.py
@@ -13,13 +13,14 @@ from flask import Flask
from webmentions_ssg import HUEY
-def test_consumer_registers_scanner_and_digest(
+def test_consumer_registers_scanner_digest_and_verification_reaper(
app: Flask, monkeypatch: pytest.MonkeyPatch
) -> None:
import webmentions_ssg
app.config["SCANNER_SCHEDULE"] = "*/15 * * * *"
app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *"
+ app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *"
registered_tasks: list[object] = []
@@ -35,11 +36,12 @@ def test_consumer_registers_scanner_and_digest(
monkeypatch.setattr(HUEY, "periodic_task", periodic_task)
consumer = importlib.import_module("webmentions_ssg.tasks.consumer")
- from webmentions_ssg.tasks import notifications, scanner
+ from webmentions_ssg.tasks import notifications, receiver, scanner
assert consumer.huey is HUEY.huey
- assert periodic_task.call_count == 2
+ assert periodic_task.call_count == 3
assert registered_tasks == [
scanner.scan_sources,
+ receiver.recover_stale_webmentions,
notifications.send_webmention_digest,
]
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 6858be9..4daca58 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -4,11 +4,13 @@
import logging
import uuid
+from datetime import UTC, datetime, timedelta
from types import ModuleType
from unittest.mock import Mock, patch
import httpx
import pytest
+import sqlalchemy as sa
from flask import Flask
from webmentions_ssg import DATABASE as db
@@ -530,6 +532,143 @@ def test_verify_webmention_persists_retryable_failure_and_reraises(
assert get_webmention_state(app, identifier) == ("failed", str(exception))
+@patch("webmentions_ssg.tasks.receiver.source_mentions_target")
+@pytest.mark.parametrize("status", ["verifying", "verified", "deleted"])
+def test_verify_webmention_skips_active_and_terminal_states(
+ source_mentions_target: Mock, app: Flask, receiver: ModuleType, status: str
+) -> None:
+ identifier = create_webmention(app, status=status)
+
+ receiver.verify_webmention.call_local(identifier)
+
+ assert get_webmention_state(app, identifier) == (status, None)
+ source_mentions_target.assert_not_called()
+
+
+def set_webmention_updated_at(
+ app: Flask, identifier: uuid.UUID, when: datetime
+) -> None:
+ with app.app_context():
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(ReceivedWebmention.uuid == identifier)
+ .values(updated_at=when)
+ )
+ db.session.commit()
+
+
+@pytest.mark.parametrize("status", ["verifying", "received"])
+def test_reaper_requeues_stale_verifications(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch, status: str
+) -> None:
+ app.config["VERIFICATION_TIMEOUT_SECONDS"] = 900
+ identifier = create_webmention(app, status=status, failure_reason="Interrupted")
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ enqueue = Mock()
+ monkeypatch.setattr(receiver, "verify_webmention", enqueue)
+
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 1
+ assert receiver.recover_stale_webmentions() == 0
+ row = db.session.get(ReceivedWebmention, identifier)
+ assert row is not None
+ assert row.updated_at.replace(tzinfo=UTC) > datetime.now(UTC) - timedelta(
+ seconds=10
+ )
+
+ assert get_webmention_state(app, identifier) == ("received", None)
+ enqueue.assert_called_once_with(identifier)
+
+
+def test_reaper_does_not_touch_recent_or_terminal_webmentions(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ recent_verifying = create_webmention(app, status="verifying")
+ recent_received = create_webmention(
+ app, source="https://source.example/received", status="received"
+ )
+ old_verified = create_webmention(
+ app, source="https://source.example/verified", status="verified"
+ )
+ set_webmention_updated_at(app, old_verified, datetime.now(UTC) - timedelta(hours=1))
+ enqueue = Mock()
+ monkeypatch.setattr(receiver, "verify_webmention", enqueue)
+
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 0
+
+ assert get_webmention_state(app, recent_verifying) == ("verifying", None)
+ assert get_webmention_state(app, recent_received) == ("received", None)
+ assert get_webmention_state(app, old_verified) == ("verified", None)
+ enqueue.assert_not_called()
+
+
+def test_reaper_recovers_after_enqueue_failure(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_webmention(app, status="verifying")
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ monkeypatch.setattr(receiver, "verify_webmention", Mock(side_effect=RuntimeError))
+
+ with app.app_context(), pytest.raises(RuntimeError):
+ receiver.recover_stale_webmentions()
+
+ assert get_webmention_state(app, identifier) == ("received", None)
+
+ # The reaper can recover the row again if the process died after the commit
+ # but before Huey persisted the task.
+ app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0
+ enqueue = Mock()
+ monkeypatch.setattr(receiver, "verify_webmention", enqueue)
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 1
+
+ enqueue.assert_called_once_with(identifier)
+
+
+def test_late_worker_cannot_overwrite_recovered_verification(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_webmention(app)
+ original_verify = receiver.verify_webmention
+
+ def concurrent_recovery(_source: str, _target: str) -> bool:
+ # The reaper supersedes this still-running attempt; a newer attempt
+ # subsequently completes before the original fetch returns.
+ app.config["VERIFICATION_TIMEOUT_SECONDS"] = 0
+ with app.app_context():
+ with patch.object(receiver, "verify_webmention", Mock()) as enqueue:
+ assert receiver.recover_stale_webmentions() == 1
+ enqueue.assert_called_once_with(identifier)
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(ReceivedWebmention.uuid == identifier)
+ .values(status="verified", updated_at=datetime.now(UTC))
+ )
+ db.session.commit()
+ return False
+
+ monkeypatch.setattr(receiver, "source_mentions_target", concurrent_recovery)
+ original_verify.call_local(identifier)
+
+ assert get_webmention_state(app, identifier) == ("verified", None)
+
+
+def test_reaper_requeues_and_verifies_integration(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_webmention(app, status="verifying")
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ check = Mock(return_value=True)
+ monkeypatch.setattr(receiver, "source_mentions_target", check)
+
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 1
+
+ assert get_webmention_state(app, identifier) == ("verified", None)
+ check.assert_called_once_with(SOURCE_URL, TARGET_URL)
+
+
def test_verify_webmention_ignores_unknown_identifier(
receiver: ModuleType, caplog: pytest.LogCaptureFixture
) -> None: