aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 20:13:42 +0100
committerDennis Fink2026-10-30 20:13:42 +0100
commit8fd569c6c0a39761f7ddda3c6e145d90534b4300 (patch)
treef92209df1918fd5c9ac27efe25463afaef2a1a5f /webmentions_ssg
parenta2992533873cb53acb052cdcd4288430b24af587 (diff)
downloadwebmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.tar.gz
webmentions-ssg-8fd569c6c0a39761f7ddda3c6e145d90534b4300.zip
fix(receiver): recover stale Webmention verifications
Add a periodic recovery sweep for Webmentions left in received or verifying states after worker crashes, lost queue entries, or interrupted processing. Use atomic lease-style updates so stale workers cannot overwrite newer results, and index verification state for efficient recovery scans.
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/config.py3
-rw-r--r--webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py26
-rw-r--r--webmentions_ssg/models.py6
-rw-r--r--webmentions_ssg/tasks/consumer.py9
-rw-r--r--webmentions_ssg/tasks/receiver.py125
5 files changed, 143 insertions, 26 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 3aec00d..e8288db 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -45,6 +45,9 @@ class DefaultConfig:
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
+ # Requeue abandoned verification attempts after 15 minutes.
+ VERIFICATION_TIMEOUT_SECONDS = 900
+ VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *"
SOURCE_DIRECTORY = None
SOURCE_BASE_URL = None
diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
new file mode 100644
index 0000000..cbb40ef
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
@@ -0,0 +1,26 @@
+"""Index received webmentions for expired verification recovery.
+
+Revision ID: c0272f25ce17
+Revises: 85f2b226ec99
+"""
+
+from alembic import op
+
+revision = "c0272f25ce17"
+down_revision = "85f2b226ec99"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.create_index(
+ "ix_received_webmentions_status_updated_at",
+ "received_webmentions",
+ ["status", "updated_at"],
+ )
+
+
+def downgrade():
+ op.drop_index(
+ "ix_received_webmentions_status_updated_at", table_name="received_webmentions"
+ )
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 4f688e5..58f95bc 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -17,6 +17,7 @@ from sqlalchemy import (
DateTime,
Enum,
ForeignKey,
+ Index,
Integer,
String,
Text,
@@ -116,6 +117,7 @@ class ReceivedWebmention(Base):
__table_args__ = (
UniqueConstraint("source", "target", name="uq_webmention_source_target"),
+ Index("ix_received_webmentions_status_updated_at", "status", "updated_at"),
)
def __repr__(self) -> str:
@@ -246,9 +248,7 @@ class SentWebmention(Base):
status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None)
# Retry state belongs to the desired revision, not to the target forever.
- attempt_count: Mapped[int] = mapped_column(
- Integer(), nullable=False, default=0
- )
+ attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0)
next_retry_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, default=None
)
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 559ef5d..afae6e0 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -14,8 +14,8 @@ def create_consumer() -> Huey:
Create and configure the Huey consumer.
The application is initialized, task modules are imported to register their
- tasks, and the source scanner is registered with the configured periodic
- schedule.
+ tasks, and periodic scanner, notification and verification recovery tasks
+ are registered.
:return: Configured Huey instance.
"""
@@ -26,6 +26,11 @@ def create_consumer() -> Huey:
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
+ recovery_schedule = crontab(
+ *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True
+ )
+ HUEY.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
+
notification_schedule = crontab(
*app.config["NOTIFICATION_SCHEDULE"].split(), strict=True
)
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index d195cf5..4e8df74 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -5,10 +5,12 @@
"""Fetch and verify incoming Webmentions against their source pages."""
import uuid
+from datetime import UTC, datetime, timedelta
from urllib.parse import urljoin
import httpx
import rfc3987
+import sqlalchemy as sa
from bs4 import BeautifulSoup
from flask import current_app
@@ -241,17 +243,18 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
@huey.task(retries=2, retry_delay=50)
def verify_webmention(webmention_uuid: uuid.UUID) -> None:
"""
- Verify a received webmention and store its verification status.
+ Verify a received webmention and persist the result.
- Temporary fetch failures are stored before being re-raised so that Huey can
- retry the task.
+ The transition into ``verifying`` is an atomic claim. Its ``updated_at``
+ timestamp acts as a lease version: if a reaper has reclaimed the row while
+ the fetch was in progress, the old worker cannot overwrite the new result.
+ Temporary fetch failures are persisted and re-raised for Huey's retries.
:param webmention_uuid: Identifier of the received webmention to verify.
:raises TemporaryFetchError: If fetching the source fails temporarily.
:raises httpx.RequestError: If an HTTP request error occurs while fetching
the source.
"""
-
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
if webmention is None:
@@ -260,31 +263,111 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- webmention.status = "verifying"
- webmention.failure_reason = None
+ # Failed tasks may be retried by Huey. Completed and already-running tasks
+ # should not start another concurrent verification.
+ if webmention.status not in {"received", "failed"}:
+ return
+
+ previous_status = webmention.status
+ previous_updated_at = webmention.updated_at
+ source, target = webmention.source, webmention.target
+ started_at = datetime.now(UTC)
+
+ claimed = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == previous_status,
+ ReceivedWebmention.updated_at == previous_updated_at,
+ )
+ .values(status="verifying", failure_reason=None, updated_at=started_at)
+ )
+ if claimed.rowcount != 1:
+ db.session.rollback()
+ return
db.session.commit()
+ def finish(status: str, failure_reason: str | None) -> None:
+ # A timeout or a newer verification may have superseded this attempt.
+ # Do not let a late worker overwrite its state.
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == "verifying",
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
+
try:
- mentions_target = source_mentions_target(webmention.source, webmention.target)
+ mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- webmention.status = "deleted"
- webmention.failure_reason = str(exc)
+ finish("deleted", str(exc))
except VerificationError as exc:
- webmention.status = "failed"
- webmention.failure_reason = str(exc)
+ finish("failed", str(exc))
except (TemporaryFetchError, httpx.RequestError) as exc:
- webmention.status = "failed"
- webmention.failure_reason = str(exc) or "Source could not be fetched"
- db.session.commit()
-
- # Huey retries the task because the exception escapes.
+ finish("failed", str(exc) or "Source could not be fetched")
raise
else:
if mentions_target:
- webmention.status = "verified"
- webmention.failure_reason = None
+ finish("verified", None)
else:
- webmention.status = "deleted"
- webmention.failure_reason = "Source does not mention target"
+ finish("deleted", "Source does not mention target")
- db.session.commit()
+
+def recover_stale_webmentions() -> int:
+ """
+ Requeue verification attempts that were abandoned or lost from the queue.
+
+ Recover both stale ``verifying`` rows (workers that died) and stale
+ ``received`` rows (tasks that were never enqueued or were lost). Conditional
+ updates prevent overlapping reapers from enqueueing the same row. Each
+ reclaimed row is committed *before* enqueueing, so the worker can claim it.
+ A crash between commit and enqueue is recovered on a later sweep.
+
+ :return: Number of webmentions requeued.
+ """
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+ stale = db.session.execute(
+ sa.select(
+ ReceivedWebmention.uuid,
+ ReceivedWebmention.status,
+ ReceivedWebmention.updated_at,
+ ).where(
+ ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.updated_at <= cutoff,
+ )
+ ).all()
+
+ recovered = 0
+ for identifier, status, updated_at in stale:
+ changed = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == identifier,
+ ReceivedWebmention.status == status,
+ ReceivedWebmention.updated_at == updated_at,
+ )
+ .values(status="received", failure_reason=None, updated_at=now)
+ )
+ if changed.rowcount != 1:
+ db.session.rollback()
+ continue
+
+ db.session.commit()
+ verify_webmention(identifier)
+ recovered += 1
+
+ if recovered:
+ current_app.logger.warning(
+ "Requeued %d stale received webmention verifications", recovered
+ )
+
+ return recovered