aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/consumer.py9
-rw-r--r--webmentions_ssg/tasks/receiver.py125
2 files changed, 111 insertions, 23 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 559ef5d..afae6e0 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -14,8 +14,8 @@ def create_consumer() -> Huey:
Create and configure the Huey consumer.
The application is initialized, task modules are imported to register their
- tasks, and the source scanner is registered with the configured periodic
- schedule.
+ tasks, and periodic scanner, notification and verification recovery tasks
+ are registered.
:return: Configured Huey instance.
"""
@@ -26,6 +26,11 @@ def create_consumer() -> Huey:
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
+ recovery_schedule = crontab(
+ *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True
+ )
+ HUEY.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
+
notification_schedule = crontab(
*app.config["NOTIFICATION_SCHEDULE"].split(), strict=True
)
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index d195cf5..4e8df74 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -5,10 +5,12 @@
"""Fetch and verify incoming Webmentions against their source pages."""
import uuid
+from datetime import UTC, datetime, timedelta
from urllib.parse import urljoin
import httpx
import rfc3987
+import sqlalchemy as sa
from bs4 import BeautifulSoup
from flask import current_app
@@ -241,17 +243,18 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
@huey.task(retries=2, retry_delay=50)
def verify_webmention(webmention_uuid: uuid.UUID) -> None:
"""
- Verify a received webmention and store its verification status.
+ Verify a received webmention and persist the result.
- Temporary fetch failures are stored before being re-raised so that Huey can
- retry the task.
+ The transition into ``verifying`` is an atomic claim. Its ``updated_at``
+ timestamp acts as a lease version: if a reaper has reclaimed the row while
+ the fetch was in progress, the old worker cannot overwrite the new result.
+ Temporary fetch failures are persisted and re-raised for Huey's retries.
:param webmention_uuid: Identifier of the received webmention to verify.
:raises TemporaryFetchError: If fetching the source fails temporarily.
:raises httpx.RequestError: If an HTTP request error occurs while fetching
the source.
"""
-
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
if webmention is None:
@@ -260,31 +263,111 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- webmention.status = "verifying"
- webmention.failure_reason = None
+ # Failed tasks may be retried by Huey. Completed and already-running tasks
+ # should not start another concurrent verification.
+ if webmention.status not in {"received", "failed"}:
+ return
+
+ previous_status = webmention.status
+ previous_updated_at = webmention.updated_at
+ source, target = webmention.source, webmention.target
+ started_at = datetime.now(UTC)
+
+ claimed = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == previous_status,
+ ReceivedWebmention.updated_at == previous_updated_at,
+ )
+ .values(status="verifying", failure_reason=None, updated_at=started_at)
+ )
+ if claimed.rowcount != 1:
+ db.session.rollback()
+ return
db.session.commit()
+ def finish(status: str, failure_reason: str | None) -> None:
+ # A timeout or a newer verification may have superseded this attempt.
+ # Do not let a late worker overwrite its state.
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == "verifying",
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
+
try:
- mentions_target = source_mentions_target(webmention.source, webmention.target)
+ mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- webmention.status = "deleted"
- webmention.failure_reason = str(exc)
+ finish("deleted", str(exc))
except VerificationError as exc:
- webmention.status = "failed"
- webmention.failure_reason = str(exc)
+ finish("failed", str(exc))
except (TemporaryFetchError, httpx.RequestError) as exc:
- webmention.status = "failed"
- webmention.failure_reason = str(exc) or "Source could not be fetched"
- db.session.commit()
-
- # Huey retries the task because the exception escapes.
+ finish("failed", str(exc) or "Source could not be fetched")
raise
else:
if mentions_target:
- webmention.status = "verified"
- webmention.failure_reason = None
+ finish("verified", None)
else:
- webmention.status = "deleted"
- webmention.failure_reason = "Source does not mention target"
+ finish("deleted", "Source does not mention target")
- db.session.commit()
+
+def recover_stale_webmentions() -> int:
+ """
+ Requeue verification attempts that were abandoned or lost from the queue.
+
+ Recover both stale ``verifying`` rows (workers that died) and stale
+ ``received`` rows (tasks that were never enqueued or were lost). Conditional
+ updates prevent overlapping reapers from enqueueing the same row. Each
+ reclaimed row is committed *before* enqueueing, so the worker can claim it.
+ A crash between commit and enqueue is recovered on a later sweep.
+
+ :return: Number of webmentions requeued.
+ """
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+ stale = db.session.execute(
+ sa.select(
+ ReceivedWebmention.uuid,
+ ReceivedWebmention.status,
+ ReceivedWebmention.updated_at,
+ ).where(
+ ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.updated_at <= cutoff,
+ )
+ ).all()
+
+ recovered = 0
+ for identifier, status, updated_at in stale:
+ changed = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == identifier,
+ ReceivedWebmention.status == status,
+ ReceivedWebmention.updated_at == updated_at,
+ )
+ .values(status="received", failure_reason=None, updated_at=now)
+ )
+ if changed.rowcount != 1:
+ db.session.rollback()
+ continue
+
+ db.session.commit()
+ verify_webmention(identifier)
+ recovered += 1
+
+ if recovered:
+ current_app.logger.warning(
+ "Requeued %d stale received webmention verifications", recovered
+ )
+
+ return recovered