diff options
| author | Dennis Fink | 2026-10-31 19:00:17 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-31 19:00:17 +0100 |
| commit | df3b79e2c89978ec21dc2da7f90801dce33aeab2 (patch) | |
| tree | 146f34d59b2b1957e53856a58777a3453960a2a5 /webmentions_ssg/tasks/receiver.py | |
| parent | f7c0691e0da84644ea8b2b83ad5574fe179da59b (diff) | |
| download | webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.tar.gz webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.zip | |
fix(receiver): prevent endless verification recovery
Cap stale verification requeues so repeatedly lost or crashing tasks
eventually become terminal failures instead of cycling indefinitely.
Reject URLs that httpx cannot safely fetch before enqueueing, and
persist unexpected verification errors as terminal failures.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 43 |
1 files changed, 41 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index f999d1b..acd66a1 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport from ..url_security import AddressResolutionError, UnsafeAddressError +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: # should not start another concurrent verification. if webmention.status not in {"received", "failed"}: return + # A reaper-exhausted row must not be revived by a late Huey retry. + if ( + webmention.status == "failed" + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return previous_status = webmention.status previous_updated_at = webmention.updated_at @@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: except (TemporaryFetchError, httpx.RequestError) as exc: finish("failed", str(exc) or "Source could not be fetched") raise + except Exception as exc: + # InvalidURL is not a RequestError; other unexpected source errors must + # also leave the row in a terminal state, without an endless retry loop. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", webmention_uuid + ) + finish("failed", str(exc) or type(exc).__name__) else: if mentions_target: finish("verified", None) @@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int: updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. :return: Number of webmentions requeued. """ @@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_(("verifying", "received")), ReceivedWebmention.updated_at <= cutoff, @@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int: ).all() recovered = 0 - for identifier, status, updated_at in stale: + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + status="failed" if at_limit else "received", + failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, + updated_at=now, + recovery_attempts=( + recovery_attempts if at_limit else recovery_attempts + 1 + ), ) - .values(status="received", failure_reason=None, updated_at=now) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: + exhausted += 1 + continue verify_webmention(identifier) recovered += 1 + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered |
