aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/receiver.py
diff options
context:
space:
mode:
authorDennis Fink2026-10-31 19:00:17 +0100
committerDennis Fink2026-10-31 19:00:17 +0100
commitdf3b79e2c89978ec21dc2da7f90801dce33aeab2 (patch)
tree146f34d59b2b1957e53856a58777a3453960a2a5 /webmentions_ssg/tasks/receiver.py
parentf7c0691e0da84644ea8b2b83ad5574fe179da59b (diff)
downloadwebmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.tar.gz
webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.zip
fix(receiver): prevent endless verification recovery
Cap stale verification requeues so repeatedly lost or crashing tasks eventually become terminal failures instead of cycling indefinitely. Reject URLs that httpx cannot safely fetch before enqueueing, and persist unexpected verification errors as terminal failures.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/receiver.py43
1 files changed, 41 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index f999d1b..acd66a1 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport
from ..url_security import AddressResolutionError, UnsafeAddressError
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+
+
class VerificationError(Exception):
"""The source permanently failed ReceivedWebmention verification."""
@@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
# should not start another concurrent verification.
if webmention.status not in {"received", "failed"}:
return
+ # A reaper-exhausted row must not be revived by a late Huey retry.
+ if (
+ webmention.status == "failed"
+ and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
+ ):
+ return
previous_status = webmention.status
previous_updated_at = webmention.updated_at
@@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
except (TemporaryFetchError, httpx.RequestError) as exc:
finish("failed", str(exc) or "Source could not be fetched")
raise
+ except Exception as exc:
+ # InvalidURL is not a RequestError; other unexpected source errors must
+ # also leave the row in a terminal state, without an endless retry loop.
+ current_app.logger.exception(
+ "Unexpected error verifying ReceivedWebmention %s", webmention_uuid
+ )
+ finish("failed", str(exc) or type(exc).__name__)
else:
if mentions_target:
finish("verified", None)
@@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int:
updates prevent overlapping reapers from enqueueing the same row. Each
reclaimed row is committed *before* enqueueing, so the worker can claim it.
A crash between commit and enqueue is recovered on a later sweep.
+ Persist a recovery count so repeatedly lost or crashing tasks eventually
+ terminate rather than being requeued forever.
:return: Number of webmentions requeued.
"""
@@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.uuid,
ReceivedWebmention.status,
ReceivedWebmention.updated_at,
+ ReceivedWebmention.recovery_attempts,
).where(
ReceivedWebmention.status.in_(("verifying", "received")),
ReceivedWebmention.updated_at <= cutoff,
@@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int:
).all()
recovered = 0
- for identifier, status, updated_at in stale:
+ exhausted = 0
+ max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+ for identifier, status, updated_at, recovery_attempts in stale:
+ at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == identifier,
ReceivedWebmention.status == status,
ReceivedWebmention.updated_at == updated_at,
+ ReceivedWebmention.recovery_attempts == recovery_attempts,
+ )
+ .values(
+ status="failed" if at_limit else "received",
+ failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
+ updated_at=now,
+ recovery_attempts=(
+ recovery_attempts if at_limit else recovery_attempts + 1
+ ),
)
- .values(status="received", failure_reason=None, updated_at=now)
)
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+ if at_limit:
+ exhausted += 1
+ continue
verify_webmention(identifier)
recovered += 1
+ if exhausted:
+ current_app.logger.warning(
+ "Failed %d stale webmentions after maximum verification recoveries",
+ exhausted,
+ )
+
if recovered:
current_app.logger.warning(
"Requeued %d stale received webmention verifications", recovered