diff options
Diffstat (limited to 'webmentions_ssg/tasks')
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 43 |
1 files changed, 41 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index f999d1b..acd66a1 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport from ..url_security import AddressResolutionError, UnsafeAddressError +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: # should not start another concurrent verification. if webmention.status not in {"received", "failed"}: return + # A reaper-exhausted row must not be revived by a late Huey retry. + if ( + webmention.status == "failed" + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return previous_status = webmention.status previous_updated_at = webmention.updated_at @@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: except (TemporaryFetchError, httpx.RequestError) as exc: finish("failed", str(exc) or "Source could not be fetched") raise + except Exception as exc: + # InvalidURL is not a RequestError; other unexpected source errors must + # also leave the row in a terminal state, without an endless retry loop. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", webmention_uuid + ) + finish("failed", str(exc) or type(exc).__name__) else: if mentions_target: finish("verified", None) @@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int: updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. :return: Number of webmentions requeued. """ @@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_(("verifying", "received")), ReceivedWebmention.updated_at <= cutoff, @@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int: ).all() recovered = 0 - for identifier, status, updated_at in stale: + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + status="failed" if at_limit else "received", + failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, + updated_at=now, + recovery_attempts=( + recovery_attempts if at_limit else recovery_attempts + 1 + ), ) - .values(status="received", failure_reason=None, updated_at=now) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: + exhausted += 1 + continue verify_webmention(identifier) recovered += 1 + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered |
