diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/config.py | 3 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py | 26 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 9 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 125 |
5 files changed, 143 insertions, 26 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 3aec00d..e8288db 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -45,6 +45,9 @@ class DefaultConfig: MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 + # Requeue abandoned verification attempts after 15 minutes. + VERIFICATION_TIMEOUT_SECONDS = 900 + VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" SOURCE_DIRECTORY = None SOURCE_BASE_URL = None diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py new file mode 100644 index 0000000..cbb40ef --- /dev/null +++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py @@ -0,0 +1,26 @@ +"""Index received webmentions for expired verification recovery. + +Revision ID: c0272f25ce17 +Revises: 85f2b226ec99 +""" + +from alembic import op + +revision = "c0272f25ce17" +down_revision = "85f2b226ec99" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + "ix_received_webmentions_status_updated_at", + "received_webmentions", + ["status", "updated_at"], + ) + + +def downgrade(): + op.drop_index( + "ix_received_webmentions_status_updated_at", table_name="received_webmentions" + ) diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 4f688e5..58f95bc 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -17,6 +17,7 @@ from sqlalchemy import ( DateTime, Enum, ForeignKey, + Index, Integer, String, Text, @@ -116,6 +117,7 @@ class ReceivedWebmention(Base): __table_args__ = ( UniqueConstraint("source", "target", name="uq_webmention_source_target"), + Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) def __repr__(self) -> str: @@ -246,9 +248,7 @@ class SentWebmention(Base): status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) # Retry state belongs to the desired revision, not to the target forever. - attempt_count: Mapped[int] = mapped_column( - Integer(), nullable=False, default=0 - ) + attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) next_retry_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 559ef5d..afae6e0 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -14,8 +14,8 @@ def create_consumer() -> Huey: Create and configure the Huey consumer. The application is initialized, task modules are imported to register their - tasks, and the source scanner is registered with the configured periodic - schedule. + tasks, and periodic scanner, notification and verification recovery tasks + are registered. :return: Configured Huey instance. """ @@ -26,6 +26,11 @@ def create_consumer() -> Huey: schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) + recovery_schedule = crontab( + *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True + ) + HUEY.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions) + notification_schedule = crontab( *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index d195cf5..4e8df74 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -5,10 +5,12 @@ """Fetch and verify incoming Webmentions against their source pages.""" import uuid +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx import rfc3987 +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -241,17 +243,18 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: """ - Verify a received webmention and store its verification status. + Verify a received webmention and persist the result. - Temporary fetch failures are stored before being re-raised so that Huey can - retry the task. + The transition into ``verifying`` is an atomic claim. Its ``updated_at`` + timestamp acts as a lease version: if a reaper has reclaimed the row while + the fetch was in progress, the old worker cannot overwrite the new result. + Temporary fetch failures are persisted and re-raised for Huey's retries. :param webmention_uuid: Identifier of the received webmention to verify. :raises TemporaryFetchError: If fetching the source fails temporarily. :raises httpx.RequestError: If an HTTP request error occurs while fetching the source. """ - webmention = db.session.get(ReceivedWebmention, webmention_uuid) if webmention is None: @@ -260,31 +263,111 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - webmention.status = "verifying" - webmention.failure_reason = None + # Failed tasks may be retried by Huey. Completed and already-running tasks + # should not start another concurrent verification. + if webmention.status not in {"received", "failed"}: + return + + previous_status = webmention.status + previous_updated_at = webmention.updated_at + source, target = webmention.source, webmention.target + started_at = datetime.now(UTC) + + claimed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == previous_status, + ReceivedWebmention.updated_at == previous_updated_at, + ) + .values(status="verifying", failure_reason=None, updated_at=started_at) + ) + if claimed.rowcount != 1: + db.session.rollback() + return db.session.commit() + def finish(status: str, failure_reason: str | None) -> None: + # A timeout or a newer verification may have superseded this attempt. + # Do not let a late worker overwrite its state. + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == "verifying", + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() + try: - mentions_target = source_mentions_target(webmention.source, webmention.target) + mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - webmention.status = "deleted" - webmention.failure_reason = str(exc) + finish("deleted", str(exc)) except VerificationError as exc: - webmention.status = "failed" - webmention.failure_reason = str(exc) + finish("failed", str(exc)) except (TemporaryFetchError, httpx.RequestError) as exc: - webmention.status = "failed" - webmention.failure_reason = str(exc) or "Source could not be fetched" - db.session.commit() - - # Huey retries the task because the exception escapes. + finish("failed", str(exc) or "Source could not be fetched") raise else: if mentions_target: - webmention.status = "verified" - webmention.failure_reason = None + finish("verified", None) else: - webmention.status = "deleted" - webmention.failure_reason = "Source does not mention target" + finish("deleted", "Source does not mention target") - db.session.commit() + +def recover_stale_webmentions() -> int: + """ + Requeue verification attempts that were abandoned or lost from the queue. + + Recover both stale ``verifying`` rows (workers that died) and stale + ``received`` rows (tasks that were never enqueued or were lost). Conditional + updates prevent overlapping reapers from enqueueing the same row. Each + reclaimed row is committed *before* enqueueing, so the worker can claim it. + A crash between commit and enqueue is recovered on a later sweep. + + :return: Number of webmentions requeued. + """ + now = datetime.now(UTC) + cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( + sa.select( + ReceivedWebmention.uuid, + ReceivedWebmention.status, + ReceivedWebmention.updated_at, + ).where( + ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.updated_at <= cutoff, + ) + ).all() + + recovered = 0 + for identifier, status, updated_at in stale: + changed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == identifier, + ReceivedWebmention.status == status, + ReceivedWebmention.updated_at == updated_at, + ) + .values(status="received", failure_reason=None, updated_at=now) + ) + if changed.rowcount != 1: + db.session.rollback() + continue + + db.session.commit() + verify_webmention(identifier) + recovered += 1 + + if recovered: + current_app.logger.warning( + "Requeued %d stale received webmention verifications", recovered + ) + + return recovered |
