diff options
| author | Dennis Fink | 2026-10-30 13:24:53 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 13:24:53 +0100 |
| commit | f6d96df2c3aecb51ca97426efddb101dce6350d7 (patch) | |
| tree | 2968f65a627fbfd15051277d07eaa321db0e3cc4 /webmentions_ssg/tasks/sender.py | |
| parent | 2b59a3082000842d980f9e19931949dbe5011229 (diff) | |
| download | webmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.tar.gz webmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.zip | |
feat(sender): add persistent retry backoff
Persist retry state per source revision so temporary delivery failures
survive worker restarts and duplicate queued tasks cannot bypass the
retry schedule.
Use capped exponential backoff, stop after a configurable attempt limit,
and reset the retry budget when a source changes or an admin resends
manually.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 117 |
1 files changed, 102 insertions, 15 deletions
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 8834934..4aa59ee 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,10 +6,11 @@ import re import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: ) -@huey.task(retries=2, retry_delay=50) +def retry_delay(attempt: int) -> timedelta: + """Calculate the capped exponential delay after a numbered send attempt.""" + base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) + maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) + return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) + + +@huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: """ Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. - Temporary failures are re-raised so Huey can retry them, while permanent - failures mark the current revision as processed. + Temporary failures are recorded with an exponential retry deadline. The + scanner requeues them when due; Huey does not retry them independently. + The retry budget is per desired source revision. :param webmention_uuid: Identifier of the sent Webmention to process. - :raises TemporarySenderError: If endpoint discovery or delivery fails for a - potentially temporary reason. - :raises httpx.RequestError: If an HTTP request fails. """ webmention = db.session.get(SentWebmention, webmention_uuid) @@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: revision = webmention.desired_revision source = webmention.source.url target = webmention.target + now = datetime.now(UTC) + deadline = webmention.next_retry_at + + # SQLite returns timezone-naive datetimes even for timezone-aware columns. + if deadline is not None: + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + if aware_deadline > now: + return + + max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: + # Recover records whose worker died during the final attempt, without + # touching a newer revision that may have been queued in the meantime. + db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count >= max_attempts, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + processed_revision=revision, + next_retry_at=None, + status=SentWebmentionStatus.FAILED, + failure_reason=( + webmention.failure_reason + or f"Retry limit reached ({max_attempts} attempts)" + ), + ) + .execution_options(synchronize_session=False) + ) + db.session.commit() + return - webmention.last_attempted_at = datetime.now(UTC) - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None + attempt = webmention.attempt_count + 1 + next_retry_at = now + retry_delay(attempt) + # Claim the attempt atomically: duplicate queued tasks must not send the + # same revision concurrently or bypass its retry deadline. + claim = db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count == attempt - 1, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + attempt_count=attempt, + last_attempted_at=now, + next_retry_at=next_retry_at, + endpoint=None, + response_status=None, + status_url=None, + ) + .execution_options(synchronize_session=False) + ) db.session.commit() + if claim.rowcount != 1: + return + + db.session.refresh(webmention) + endpoint: str | None = None response_status: int | None = None @@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.UNSUPPORTED webmention.failure_reason = "No Webmention endpoint discovered" webmention.endpoint = None @@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) or "Webmention request failed" + if attempt >= max_attempts: + webmention.processed_revision = revision + webmention.next_retry_at = None + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + webmention.failure_reason, + ) webmention.endpoint = endpoint webmention.response_status = response_status db.session.commit() - # Leave processed_revision unchanged. Huey may retry the - # attempt, and the periodic scanner will also rediscover - # outstanding work. - raise + # No Huey retries: only the scanner may requeue after next_retry_at. + return except PermanentSenderError as exc: if not attempt_is_current(webmention, revision): return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) webmention.endpoint = endpoint @@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.sent_revision = revision webmention.status = SentWebmentionStatus.SENT |
