aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/sender.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-13 22:12:02 +0100
committerDennis Fink2026-11-13 22:12:02 +0100
commit2366d455c7d1ae16c84e0bcbffb94ed762c16832 (patch)
tree51de07737ebc12fdcbf29a0a405ff2cd5aa9eb87 /webmentions_ssg/tasks/sender.py
parent64f328d86bba9caa9d0a60aa89e06f70403942b9 (diff)
downloadwebmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.tar.gz
webmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.zip
refactor(core): split task workflows into focused helpers
Decompose receiver verification, source scanning, and Webmention delivery into explicit claim, check, reconciliation, and outcome-recording stages. Extract Huey backend parsing helpers and add focused tests for the new task boundaries while preserving existing processing semantics.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/sender.py242
1 files changed, 168 insertions, 74 deletions
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 92d8dfa..339dfe6 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -5,6 +5,7 @@
"""Claim, send, and persist outgoing Webmention attempts."""
import uuid
+from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import httpx
@@ -76,38 +77,64 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
-def _send_locked(webmention_uuid: uuid.UUID) -> None:
- """Process a Webmention while holding its per-UUID task lock."""
- webmention = db.session.get(SentWebmention, webmention_uuid)
+@dataclass(frozen=True)
+class SendAttempt:
+ """Claimed source revision and retry budget for one send."""
+
+ identifier: uuid.UUID
+ revision: int
+ number: int
+ max_attempts: int
+ source: str
+ target: str
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+ """HTTP outcome, with whether the failure is eligible for another attempt.
+
+ ``terminal`` describes the failure type. The retry budget can also make a
+ temporary failure terminal when its last allowed attempt has been used.
+ """
+
+ status: SentWebmentionStatus
+ reason: str | None = None
+ endpoint: str | None = None
+ response_status: int | None = None
+ status_url: str | None = None
+ terminal: bool = True
+ temporary_failure: bool = False
- if webmention is None:
- current_app.logger.warning(
- "Cannot send unknown SentWebmention %s", webmention_uuid
- )
- return
+def is_due(webmention: SentWebmention, now: datetime) -> bool:
+ """Return whether an unprocessed Webmention has reached its retry deadline."""
if not webmention.pending:
- return
+ return False
- revision = webmention.desired_revision
- source = webmention.source.url
- target = webmention.target
- now = datetime.now(UTC)
deadline = webmention.next_retry_at
+ if deadline is None:
+ return True
+
+ aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
+ return aware_deadline <= now
+
- if deadline is not None:
- aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
- if aware_deadline > now:
- return
+def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | None:
+ """Atomically reserve one send attempt, or finish an exhausted revision.
+ An attempt's next retry deadline is persisted *before* the HTTP request so
+ a lost worker cannot cause an immediate duplicate send.
+ """
+ revision = webmention.desired_revision
max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
+
if webmention.attempt_count >= max_attempts:
- # Recover records whose worker died during the final attempt, without
- # touching a newer revision that may have been queued in the meantime.
+ # A dead worker may have already consumed the final attempt. Never
+ # finish a newer revision or a retry whose deadline is still in future.
db.session.execute(
sa.update(SentWebmention)
.where(
- SentWebmention.uuid == webmention_uuid,
+ SentWebmention.uuid == webmention.uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count >= max_attempts,
SentWebmention.pending,
@@ -128,19 +155,22 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
.execution_options(synchronize_session=False)
)
db.session.commit()
- return
+ return None
- attempt = webmention.attempt_count + 1
- next_retry_at = now + retry_delay(attempt)
-
- # Claim the attempt atomically: duplicate queued tasks must not send the
- # same revision concurrently or bypass its retry deadline.
- claim = db.session.execute(
+ attempt = SendAttempt(
+ identifier=webmention.uuid,
+ revision=revision,
+ number=webmention.attempt_count + 1,
+ max_attempts=max_attempts,
+ source=webmention.source.url,
+ target=webmention.target,
+ )
+ changed = db.session.execute(
sa.update(SentWebmention)
.where(
- SentWebmention.uuid == webmention_uuid,
+ SentWebmention.uuid == attempt.identifier,
SentWebmention.desired_revision == revision,
- SentWebmention.attempt_count == attempt - 1,
+ SentWebmention.attempt_count == attempt.number - 1,
SentWebmention.pending,
sa.or_(
SentWebmention.next_retry_at.is_(None),
@@ -148,9 +178,9 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
),
)
.values(
- attempt_count=attempt,
+ attempt_count=attempt.number,
last_attempted_at=now,
- next_retry_at=next_retry_at,
+ next_retry_at=now + retry_delay(attempt.number),
endpoint=None,
response_status=None,
status_url=None,
@@ -159,17 +189,20 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
)
db.session.commit()
- if rows_affected(claim) != 1:
- return
+ if rows_affected(changed) != 1:
+ return None
db.session.refresh(webmention)
+ return attempt
+
+def deliver_webmention(
+ source: str, target: str, *, webmention_uuid: uuid.UUID
+) -> DeliveryResult:
+ """Discover and POST a Webmention, returning an outcome without DB writes."""
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
- reason: str | None = None
- terminal = True
- temporary_failure = False
try:
if not is_http_url(target):
@@ -179,19 +212,26 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- outcome = SentWebmentionStatus.UNSUPPORTED
- reason = "No Webmention endpoint discovered"
- else:
- response_status, status_url = post_webmention(
- client, endpoint=endpoint, source=source, target=target
+ return DeliveryResult(
+ status=SentWebmentionStatus.UNSUPPORTED,
+ reason="No Webmention endpoint discovered",
)
- if not 200 <= response_status <= 299:
- message = f"Webmention endpoint returned HTTP {response_status}"
- if is_temporary_status(response_status):
- raise TemporarySenderError(message)
- raise PermanentSenderError(message)
- outcome = SentWebmentionStatus.SENT
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+
+ return DeliveryResult(
+ status=SentWebmentionStatus.SENT,
+ endpoint=endpoint,
+ response_status=response_status,
+ status_url=status_url,
+ )
except (
PermanentSenderError,
UnsafeAddressError,
@@ -200,54 +240,108 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
httpx.LocalProtocolError,
) as exc:
# Known permanent failures cannot improve with retries.
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or type(exc).__name__
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or type(exc).__name__,
+ endpoint=endpoint,
+ response_status=response_status,
+ )
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or "Webmention request failed"
- terminal = attempt >= max_attempts
- temporary_failure = True
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or "Webmention request failed",
+ endpoint=endpoint,
+ response_status=response_status,
+ terminal=False,
+ temporary_failure=True,
+ )
except Exception as exc:
- # Unexpected errors must not disappear into a generic permanent-failure
- # handler without a traceback (including programming ValueErrors).
+ # Unexpected errors should be terminal and keep their traceback.
current_app.logger.exception(
"Unexpected error sending SentWebmention %s", webmention_uuid
)
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or type(exc).__name__
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or type(exc).__name__,
+ endpoint=endpoint,
+ response_status=response_status,
+ )
+
+def record_outcome(
+ webmention: SentWebmention,
+ attempt: SendAttempt,
+ result: DeliveryResult,
+ *,
+ terminal: bool,
+) -> bool:
+ """Record an outcome only while the claimed source revision is current."""
# A scanner may have reopened or superseded the revision during I/O.
- # Refresh only once, and persist one outcome atomically with its metadata.
- if not attempt_is_current(webmention, revision):
- return
+ if not attempt_is_current(webmention, attempt.revision):
+ return False
- webmention.status = outcome
- webmention.failure_reason = reason
- webmention.endpoint = endpoint
- webmention.response_status = response_status
+ webmention.status = result.status
+ webmention.failure_reason = result.reason
+ webmention.endpoint = result.endpoint
+ webmention.response_status = result.response_status
if terminal:
- webmention.processed_revision = revision
+ webmention.processed_revision = attempt.revision
webmention.next_retry_at = None
- if outcome == SentWebmentionStatus.SENT:
- webmention.sent_revision = revision
+ if result.status == SentWebmentionStatus.SENT:
+ webmention.sent_revision = attempt.revision
webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
+ result.status_url
+ if result.status_url is not None and is_http_url(result.status_url)
+ else None
)
webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+ return True
+
- if temporary_failure and not terminal:
- # Huey does not retry: scanner requeues after next_retry_at.
+def _send_locked(webmention_uuid: uuid.UUID) -> bool:
+ """Process a due Webmention while holding its per-UUID task lock.
+
+ :return: Whether this worker recorded a delivery outcome.
+ """
+ webmention = db.session.get(SentWebmention, webmention_uuid)
+ if webmention is None:
+ current_app.logger.warning(
+ "Cannot send unknown SentWebmention %s", webmention_uuid
+ )
+ return False
+
+ now = datetime.now(UTC)
+ if not is_due(webmention, now):
+ return False
+
+ if (attempt := claim_attempt(webmention, now)) is None:
+ return False
+
+ result = deliver_webmention(
+ attempt.source, attempt.target, webmention_uuid=webmention_uuid
+ )
+ terminal = result.terminal or attempt.number >= attempt.max_attempts
+
+ if not record_outcome(webmention, attempt, result, terminal=terminal):
+ return False
+
+ if result.temporary_failure and not terminal:
+ # Huey does not retry: the scanner requeues after next_retry_at.
current_app.logger.info(
- "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ "Webmention %s send attempt %d failed; will retry",
+ webmention_uuid,
+ attempt.number,
)
- elif temporary_failure and terminal:
+ elif result.temporary_failure:
current_app.logger.warning(
"Webmention %s exhausted %d send attempts: %s",
webmention_uuid,
- attempt,
- reason,
+ attempt.number,
+ result.reason,
)
+
+ return True