aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/receiver.py43
1 files changed, 41 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index f999d1b..acd66a1 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport
from ..url_security import AddressResolutionError, UnsafeAddressError
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+
+
class VerificationError(Exception):
"""The source permanently failed ReceivedWebmention verification."""
@@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
# should not start another concurrent verification.
if webmention.status not in {"received", "failed"}:
return
+ # A reaper-exhausted row must not be revived by a late Huey retry.
+ if (
+ webmention.status == "failed"
+ and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
+ ):
+ return
previous_status = webmention.status
previous_updated_at = webmention.updated_at
@@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
except (TemporaryFetchError, httpx.RequestError) as exc:
finish("failed", str(exc) or "Source could not be fetched")
raise
+ except Exception as exc:
+ # InvalidURL is not a RequestError; other unexpected source errors must
+ # also leave the row in a terminal state, without an endless retry loop.
+ current_app.logger.exception(
+ "Unexpected error verifying ReceivedWebmention %s", webmention_uuid
+ )
+ finish("failed", str(exc) or type(exc).__name__)
else:
if mentions_target:
finish("verified", None)
@@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int:
updates prevent overlapping reapers from enqueueing the same row. Each
reclaimed row is committed *before* enqueueing, so the worker can claim it.
A crash between commit and enqueue is recovered on a later sweep.
+ Persist a recovery count so repeatedly lost or crashing tasks eventually
+ terminate rather than being requeued forever.
:return: Number of webmentions requeued.
"""
@@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.uuid,
ReceivedWebmention.status,
ReceivedWebmention.updated_at,
+ ReceivedWebmention.recovery_attempts,
).where(
ReceivedWebmention.status.in_(("verifying", "received")),
ReceivedWebmention.updated_at <= cutoff,
@@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int:
).all()
recovered = 0
- for identifier, status, updated_at in stale:
+ exhausted = 0
+ max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+ for identifier, status, updated_at, recovery_attempts in stale:
+ at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == identifier,
ReceivedWebmention.status == status,
ReceivedWebmention.updated_at == updated_at,
+ ReceivedWebmention.recovery_attempts == recovery_attempts,
+ )
+ .values(
+ status="failed" if at_limit else "received",
+ failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
+ updated_at=now,
+ recovery_attempts=(
+ recovery_attempts if at_limit else recovery_attempts + 1
+ ),
)
- .values(status="received", failure_reason=None, updated_at=now)
)
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+ if at_limit:
+ exhausted += 1
+ continue
verify_webmention(identifier)
recovered += 1
+ if exhausted:
+ current_app.logger.warning(
+ "Failed %d stale webmentions after maximum verification recoveries",
+ exhausted,
+ )
+
if recovered:
current_app.logger.warning(
"Requeued %d stale received webmention verifications", recovered