diff options
Diffstat (limited to 'webmentions_ssg/tasks/sender.py')
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 242 |
1 files changed, 168 insertions, 74 deletions
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 92d8dfa..339dfe6 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -5,6 +5,7 @@ """Claim, send, and persist outgoing Webmention attempts.""" import uuid +from dataclasses import dataclass from datetime import UTC, datetime, timedelta import httpx @@ -76,38 +77,64 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return -def _send_locked(webmention_uuid: uuid.UUID) -> None: - """Process a Webmention while holding its per-UUID task lock.""" - webmention = db.session.get(SentWebmention, webmention_uuid) +@dataclass(frozen=True) +class SendAttempt: + """Claimed source revision and retry budget for one send.""" + + identifier: uuid.UUID + revision: int + number: int + max_attempts: int + source: str + target: str + + +@dataclass(frozen=True) +class DeliveryResult: + """HTTP outcome, with whether the failure is eligible for another attempt. + + ``terminal`` describes the failure type. The retry budget can also make a + temporary failure terminal when its last allowed attempt has been used. + """ + + status: SentWebmentionStatus + reason: str | None = None + endpoint: str | None = None + response_status: int | None = None + status_url: str | None = None + terminal: bool = True + temporary_failure: bool = False - if webmention is None: - current_app.logger.warning( - "Cannot send unknown SentWebmention %s", webmention_uuid - ) - return +def is_due(webmention: SentWebmention, now: datetime) -> bool: + """Return whether an unprocessed Webmention has reached its retry deadline.""" if not webmention.pending: - return + return False - revision = webmention.desired_revision - source = webmention.source.url - target = webmention.target - now = datetime.now(UTC) deadline = webmention.next_retry_at + if deadline is None: + return True + + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + return aware_deadline <= now + - if deadline is not None: - aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) - if aware_deadline > now: - return +def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | None: + """Atomically reserve one send attempt, or finish an exhausted revision. + An attempt's next retry deadline is persisted *before* the HTTP request so + a lost worker cannot cause an immediate duplicate send. + """ + revision = webmention.desired_revision max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: - # Recover records whose worker died during the final attempt, without - # touching a newer revision that may have been queued in the meantime. + # A dead worker may have already consumed the final attempt. Never + # finish a newer revision or a retry whose deadline is still in future. db.session.execute( sa.update(SentWebmention) .where( - SentWebmention.uuid == webmention_uuid, + SentWebmention.uuid == webmention.uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count >= max_attempts, SentWebmention.pending, @@ -128,19 +155,22 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: .execution_options(synchronize_session=False) ) db.session.commit() - return + return None - attempt = webmention.attempt_count + 1 - next_retry_at = now + retry_delay(attempt) - - # Claim the attempt atomically: duplicate queued tasks must not send the - # same revision concurrently or bypass its retry deadline. - claim = db.session.execute( + attempt = SendAttempt( + identifier=webmention.uuid, + revision=revision, + number=webmention.attempt_count + 1, + max_attempts=max_attempts, + source=webmention.source.url, + target=webmention.target, + ) + changed = db.session.execute( sa.update(SentWebmention) .where( - SentWebmention.uuid == webmention_uuid, + SentWebmention.uuid == attempt.identifier, SentWebmention.desired_revision == revision, - SentWebmention.attempt_count == attempt - 1, + SentWebmention.attempt_count == attempt.number - 1, SentWebmention.pending, sa.or_( SentWebmention.next_retry_at.is_(None), @@ -148,9 +178,9 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: ), ) .values( - attempt_count=attempt, + attempt_count=attempt.number, last_attempted_at=now, - next_retry_at=next_retry_at, + next_retry_at=now + retry_delay(attempt.number), endpoint=None, response_status=None, status_url=None, @@ -159,17 +189,20 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: ) db.session.commit() - if rows_affected(claim) != 1: - return + if rows_affected(changed) != 1: + return None db.session.refresh(webmention) + return attempt + +def deliver_webmention( + source: str, target: str, *, webmention_uuid: uuid.UUID +) -> DeliveryResult: + """Discover and POST a Webmention, returning an outcome without DB writes.""" endpoint: str | None = None response_status: int | None = None status_url: str | None = None - reason: str | None = None - terminal = True - temporary_failure = False try: if not is_http_url(target): @@ -179,19 +212,26 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: - outcome = SentWebmentionStatus.UNSUPPORTED - reason = "No Webmention endpoint discovered" - else: - response_status, status_url = post_webmention( - client, endpoint=endpoint, source=source, target=target + return DeliveryResult( + status=SentWebmentionStatus.UNSUPPORTED, + reason="No Webmention endpoint discovered", ) - if not 200 <= response_status <= 299: - message = f"Webmention endpoint returned HTTP {response_status}" - if is_temporary_status(response_status): - raise TemporarySenderError(message) - raise PermanentSenderError(message) - outcome = SentWebmentionStatus.SENT + response_status, status_url = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + if not 200 <= response_status <= 299: + message = f"Webmention endpoint returned HTTP {response_status}" + if is_temporary_status(response_status): + raise TemporarySenderError(message) + raise PermanentSenderError(message) + + return DeliveryResult( + status=SentWebmentionStatus.SENT, + endpoint=endpoint, + response_status=response_status, + status_url=status_url, + ) except ( PermanentSenderError, UnsafeAddressError, @@ -200,54 +240,108 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: httpx.LocalProtocolError, ) as exc: # Known permanent failures cannot improve with retries. - outcome = SentWebmentionStatus.FAILED - reason = str(exc) or type(exc).__name__ + return DeliveryResult( + status=SentWebmentionStatus.FAILED, + reason=str(exc) or type(exc).__name__, + endpoint=endpoint, + response_status=response_status, + ) except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: - outcome = SentWebmentionStatus.FAILED - reason = str(exc) or "Webmention request failed" - terminal = attempt >= max_attempts - temporary_failure = True + return DeliveryResult( + status=SentWebmentionStatus.FAILED, + reason=str(exc) or "Webmention request failed", + endpoint=endpoint, + response_status=response_status, + terminal=False, + temporary_failure=True, + ) except Exception as exc: - # Unexpected errors must not disappear into a generic permanent-failure - # handler without a traceback (including programming ValueErrors). + # Unexpected errors should be terminal and keep their traceback. current_app.logger.exception( "Unexpected error sending SentWebmention %s", webmention_uuid ) - outcome = SentWebmentionStatus.FAILED - reason = str(exc) or type(exc).__name__ + return DeliveryResult( + status=SentWebmentionStatus.FAILED, + reason=str(exc) or type(exc).__name__, + endpoint=endpoint, + response_status=response_status, + ) + +def record_outcome( + webmention: SentWebmention, + attempt: SendAttempt, + result: DeliveryResult, + *, + terminal: bool, +) -> bool: + """Record an outcome only while the claimed source revision is current.""" # A scanner may have reopened or superseded the revision during I/O. - # Refresh only once, and persist one outcome atomically with its metadata. - if not attempt_is_current(webmention, revision): - return + if not attempt_is_current(webmention, attempt.revision): + return False - webmention.status = outcome - webmention.failure_reason = reason - webmention.endpoint = endpoint - webmention.response_status = response_status + webmention.status = result.status + webmention.failure_reason = result.reason + webmention.endpoint = result.endpoint + webmention.response_status = result.response_status if terminal: - webmention.processed_revision = revision + webmention.processed_revision = attempt.revision webmention.next_retry_at = None - if outcome == SentWebmentionStatus.SENT: - webmention.sent_revision = revision + if result.status == SentWebmentionStatus.SENT: + webmention.sent_revision = attempt.revision webmention.status_url = ( - status_url if status_url is not None and is_http_url(status_url) else None + result.status_url + if result.status_url is not None and is_http_url(result.status_url) + else None ) webmention.last_sent_at = datetime.now(UTC) db.session.commit() + return True + - if temporary_failure and not terminal: - # Huey does not retry: scanner requeues after next_retry_at. +def _send_locked(webmention_uuid: uuid.UUID) -> bool: + """Process a due Webmention while holding its per-UUID task lock. + + :return: Whether this worker recorded a delivery outcome. + """ + webmention = db.session.get(SentWebmention, webmention_uuid) + if webmention is None: + current_app.logger.warning( + "Cannot send unknown SentWebmention %s", webmention_uuid + ) + return False + + now = datetime.now(UTC) + if not is_due(webmention, now): + return False + + if (attempt := claim_attempt(webmention, now)) is None: + return False + + result = deliver_webmention( + attempt.source, attempt.target, webmention_uuid=webmention_uuid + ) + terminal = result.terminal or attempt.number >= attempt.max_attempts + + if not record_outcome(webmention, attempt, result, terminal=terminal): + return False + + if result.temporary_failure and not terminal: + # Huey does not retry: the scanner requeues after next_retry_at. current_app.logger.info( - "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt + "Webmention %s send attempt %d failed; will retry", + webmention_uuid, + attempt.number, ) - elif temporary_failure and terminal: + elif result.temporary_failure: current_app.logger.warning( "Webmention %s exhausted %d send attempts: %s", webmention_uuid, - attempt, - reason, + attempt.number, + result.reason, ) + + return True |
