diff options
Diffstat (limited to 'webmentions_ssg/tasks')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 117 |
2 files changed, 111 insertions, 16 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 7e163f7..03f4b8d 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -385,6 +385,8 @@ def update_source( for webmention in existing.values(): webmention.active = webmention.target in scanned.targets webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -522,6 +524,8 @@ def scan_sources() -> None: for webmention in source.sent_webmentions: webmention.active = False webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -536,7 +540,11 @@ def scan_sources() -> None: sa.or_( SentWebmention.processed_revision.is_(None), SentWebmention.processed_revision < SentWebmention.desired_revision, - ) + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= scan_time, + ), ) .order_by(SentWebmention.uuid) ): diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 8834934..4aa59ee 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,10 +6,11 @@ import re import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: ) -@huey.task(retries=2, retry_delay=50) +def retry_delay(attempt: int) -> timedelta: + """Calculate the capped exponential delay after a numbered send attempt.""" + base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) + maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) + return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) + + +@huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: """ Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. - Temporary failures are re-raised so Huey can retry them, while permanent - failures mark the current revision as processed. + Temporary failures are recorded with an exponential retry deadline. The + scanner requeues them when due; Huey does not retry them independently. + The retry budget is per desired source revision. :param webmention_uuid: Identifier of the sent Webmention to process. - :raises TemporarySenderError: If endpoint discovery or delivery fails for a - potentially temporary reason. - :raises httpx.RequestError: If an HTTP request fails. """ webmention = db.session.get(SentWebmention, webmention_uuid) @@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: revision = webmention.desired_revision source = webmention.source.url target = webmention.target + now = datetime.now(UTC) + deadline = webmention.next_retry_at + + # SQLite returns timezone-naive datetimes even for timezone-aware columns. + if deadline is not None: + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + if aware_deadline > now: + return + + max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: + # Recover records whose worker died during the final attempt, without + # touching a newer revision that may have been queued in the meantime. + db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count >= max_attempts, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + processed_revision=revision, + next_retry_at=None, + status=SentWebmentionStatus.FAILED, + failure_reason=( + webmention.failure_reason + or f"Retry limit reached ({max_attempts} attempts)" + ), + ) + .execution_options(synchronize_session=False) + ) + db.session.commit() + return - webmention.last_attempted_at = datetime.now(UTC) - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None + attempt = webmention.attempt_count + 1 + next_retry_at = now + retry_delay(attempt) + # Claim the attempt atomically: duplicate queued tasks must not send the + # same revision concurrently or bypass its retry deadline. + claim = db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count == attempt - 1, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + attempt_count=attempt, + last_attempted_at=now, + next_retry_at=next_retry_at, + endpoint=None, + response_status=None, + status_url=None, + ) + .execution_options(synchronize_session=False) + ) db.session.commit() + if claim.rowcount != 1: + return + + db.session.refresh(webmention) + endpoint: str | None = None response_status: int | None = None @@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.UNSUPPORTED webmention.failure_reason = "No Webmention endpoint discovered" webmention.endpoint = None @@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) or "Webmention request failed" + if attempt >= max_attempts: + webmention.processed_revision = revision + webmention.next_retry_at = None + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + webmention.failure_reason, + ) webmention.endpoint = endpoint webmention.response_status = response_status db.session.commit() - # Leave processed_revision unchanged. Huey may retry the - # attempt, and the periodic scanner will also rediscover - # outstanding work. - raise + # No Huey retries: only the scanner may requeue after next_retry_at. + return except PermanentSenderError as exc: if not attempt_is_current(webmention, revision): return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) webmention.endpoint = endpoint @@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.sent_revision = revision webmention.status = SentWebmentionStatus.SENT |
