aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/extension.py124
-rw-r--r--webmentions_ssg/tasks/receiver.py148
-rw-r--r--webmentions_ssg/tasks/scanner.py115
-rw-r--r--webmentions_ssg/tasks/sender.py242
4 files changed, 389 insertions, 240 deletions
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index 94875ff..b77a140 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -9,7 +9,7 @@ from __future__ import annotations
from collections.abc import Callable
from functools import update_wrapper, wraps
from typing import Any
-from urllib.parse import urlsplit, urlunsplit
+from urllib.parse import SplitResult, urlsplit, urlunsplit
from flask import Flask
from huey import Huey as BaseHuey
@@ -18,6 +18,63 @@ from huey.api import TaskWrapper
from .json_serializer import JSONSerializer
+def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]:
+ """Select a Redis Huey flavour while retaining the underlying URL scheme."""
+ from huey import (
+ PriorityRedisExpireHuey,
+ PriorityRedisHuey,
+ RedisExpireHuey,
+ RedisHuey,
+ )
+
+ scheme = parsed.scheme.lower()
+ base_scheme, separator, flavour = scheme.partition("+")
+ backends = {
+ "": RedisHuey,
+ "priority": PriorityRedisHuey,
+ "expire": RedisExpireHuey,
+ "priority+expire": PriorityRedisExpireHuey,
+ }
+ if flavour not in backends or (separator and not flavour):
+ raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}")
+
+ fixed_url = urlunsplit(parsed._replace(scheme=base_scheme))
+ return backends[flavour], {"url": fixed_url}
+
+
+def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]:
+ """Extract a relative or absolute SQLite/file storage path."""
+ from huey import FileHuey, SqliteHuey
+
+ if scheme == "sqlite":
+ backend, option, description, example = (
+ SqliteHuey,
+ "filename",
+ "SQLite Huey",
+ "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)",
+ )
+ missing = "SQLite Huey URL must include a database path."
+ else:
+ backend, option, description, example = (
+ FileHuey,
+ "path",
+ "File Huey",
+ "file:///huey-queue (relative) or file:////var/huey-queue (absolute)",
+ )
+ missing = "File Huey URL must include a directory path."
+
+ prefix = f"{scheme}:///"
+ if not url.startswith(prefix):
+ raise RuntimeError(f"{description} URLs must look like {example}")
+
+ # Four slashes retain the initial '/' of an absolute path; three do not.
+ path = url.removeprefix(prefix)
+ if not path:
+ raise RuntimeError(missing)
+
+ return backend, {option: path}
+
+
class _LazyTask:
"""Stable public task handle which follows the extension's current backend.
@@ -211,75 +268,18 @@ class Huey:
parsed = urlsplit(url)
scheme = parsed.scheme.lower()
- base_scheme, separator, flavour = scheme.partition("+")
- if base_scheme in {"redis", "rediss"}:
- from huey import (
- PriorityRedisExpireHuey,
- PriorityRedisHuey,
- RedisExpireHuey,
- RedisHuey,
- )
-
- backends = {
- "": RedisHuey,
- "priority": PriorityRedisHuey,
- "expire": RedisExpireHuey,
- "priority+expire": PriorityRedisExpireHuey,
- }
- if flavour not in backends or (separator and not flavour):
- raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}")
-
- fixed_url = urlunsplit(parsed._replace(scheme=base_scheme))
- return backends[flavour], {"url": fixed_url}
-
- if scheme == "sqlite":
- from huey import SqliteHuey
-
- prefix = "sqlite:///"
-
- if not url.startswith(prefix):
- raise RuntimeError(
- "SQLite Huey URLs must look like sqlite:///huey.db (relative) "
- "or sqlite:////var/huey.db (absolute)"
- )
-
- # Preserve a leading slash only when the URL has four slashes.
- filename = url.removeprefix(prefix)
-
- if not filename:
- raise RuntimeError("SQLite Huey URL must include a database path.")
-
- return SqliteHuey, {"filename": filename}
-
- if scheme == "file":
- from huey import FileHuey
-
- prefix = "file:///"
-
- if not url.startswith(prefix):
- raise RuntimeError(
- "File Huey URLs must look like file:///huey-queue (relative) "
- "or file:////var/huey-queue (absolute)"
- )
-
- # file:////path is absolute; file:///path is relative here.
- path = url.removeprefix(prefix)
-
- if not path:
- raise RuntimeError("File Huey URL must include a directory path.")
-
- return FileHuey, {"path": path}
-
+ if scheme.partition("+")[0] in {"redis", "rediss"}:
+ return _redis_backend(parsed)
+ if scheme in {"sqlite", "file"}:
+ return _path_backend(url, scheme)
if scheme in {"postgres", "postgresql"}:
from huey import PostgresHuey
return PostgresHuey, {"dsn": url}
-
if scheme == "memory":
from huey import MemoryHuey
return MemoryHuey, {}
-
if scheme == "blackhole":
from huey import BlackHoleHuey
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 1df363f..31fb64c 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -5,6 +5,7 @@
"""Fetch and verify incoming Webmentions with transactional state changes."""
import uuid
+from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
@@ -123,105 +124,124 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
)
-@huey.task(retries=2, retry_delay=50)
-def verify_webmention(webmention_uuid: uuid.UUID) -> None:
- """Verify a received webmention and persist the result.
+@dataclass(frozen=True)
+class VerificationClaim:
+ """Source and lease version for one atomically claimed verification."""
- The transition into ``verifying`` is an atomic claim. Its ``updated_at``
- timestamp acts as a lease version: if a reaper has reclaimed the row while
- the fetch was in progress, the old worker cannot overwrite the new result.
- Temporary fetch failures are persisted and re-raised for Huey's retries.
+ identifier: uuid.UUID
+ source: str
+ target: str
+ started_at: datetime
- :param webmention_uuid: Identifier of the received webmention to verify.
- :raises TemporaryFetchError: If fetching the source fails temporarily.
- :raises httpx.RequestError: If an HTTP request error occurs while fetching
- the source.
- """
+
+def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
+ """Claim an eligible row, returning its source, target, and lease version."""
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
if webmention is None:
current_app.logger.warning(
"Cannot verify unknown ReceivedWebmention %s", webmention_uuid
)
- return
+ return None
if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
- return
+ return None
if (
webmention.status == ReceivedStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
- return
+ return None
- previous_status = webmention.status
- previous_updated_at = webmention.updated_at
- source, target = webmention.source, webmention.target
- started_at = datetime.now(UTC)
-
- claimed = db.session.execute(
+ claim = VerificationClaim(
+ identifier=webmention_uuid,
+ source=webmention.source,
+ target=webmention.target,
+ started_at=datetime.now(UTC),
+ )
+ changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == previous_status,
- ReceivedWebmention.updated_at == previous_updated_at,
+ ReceivedWebmention.status == webmention.status,
+ ReceivedWebmention.updated_at == webmention.updated_at,
)
.values(
- status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at
+ status=ReceivedStatus.VERIFYING,
+ failure_reason=None,
+ updated_at=claim.started_at,
)
)
- if rows_affected(claimed) != 1:
+ if rows_affected(changed) != 1:
db.session.rollback()
- return
+ return None
db.session.commit()
- status = None
- failure_reason = None
+ return claim
+
+def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
+ """Map verification results and permanent errors to a final status/reason."""
try:
- mentions_target = source_mentions_target(source, target)
+ mentions_target = source_mentions_target(claim.source, claim.target)
except SourceGoneError as exc:
- status = ReceivedStatus.DELETED
- failure_reason = str(exc)
+ return ReceivedStatus.DELETED, str(exc)
except VerificationError as exc:
- status = ReceivedStatus.FAILED
- failure_reason = str(exc)
- except (TemporaryFetchError, httpx.RequestError) as exc:
- status = ReceivedStatus.FAILED
- failure_reason = str(exc) or "Source could not be fetched"
- raise
+ return ReceivedStatus.FAILED, str(exc)
+ except TemporaryFetchError, httpx.RequestError:
+ raise # These errors must be persisted and then retried by Huey.
except Exception as exc:
- # InvalidURL is not a RequestError; other unexpected source errors must
- # also leave the row in a terminal state, without an endless retry loop.
+ # InvalidURL is not a RequestError; unexpected source errors are terminal.
current_app.logger.exception(
- "Unexpected error verifying ReceivedWebmention %s", webmention_uuid
+ "Unexpected error verifying ReceivedWebmention %s", claim.identifier
)
+ return ReceivedStatus.FAILED, str(exc) or type(exc).__name__
- status = ReceivedStatus.FAILED
- failure_reason = str(exc) or type(exc).__name__
- else:
- if mentions_target:
- status = ReceivedStatus.VERIFIED
- else:
- status = ReceivedStatus.DELETED
- failure_reason = "Source does not mention target"
- finally:
- if status is not None:
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == ReceivedStatus.VERIFYING,
- ReceivedWebmention.updated_at == started_at,
- )
- .values(
- status=status,
- failure_reason=failure_reason,
- updated_at=datetime.now(UTC),
- )
- )
- db.session.commit()
+ if mentions_target:
+ return ReceivedStatus.VERIFIED, None
+ return ReceivedStatus.DELETED, "Source does not mention target"
+
+
+def release_claim(
+ claim: VerificationClaim, status: ReceivedStatus, reason: str | None
+) -> bool:
+ """Persist an outcome only if this worker still owns the verification lease."""
+ changed = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == claim.identifier,
+ ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.updated_at == claim.started_at,
+ )
+ .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC))
+ )
+ db.session.commit()
+ return rows_affected(changed) == 1
+
+
+@huey.task(retries=2, retry_delay=50)
+def verify_webmention(webmention_uuid: uuid.UUID) -> None:
+ """Claim and verify a received Webmention, preserving its lease on release.
+
+ Temporary fetch failures are recorded and re-raised for Huey's retries.
+
+ :param webmention_uuid: Identifier of the received webmention to verify.
+ :raises TemporaryFetchError: If fetching the source fails temporarily.
+ :raises httpx.RequestError: If an HTTP request fails while fetching.
+ """
+ if (claim := claim_eligible_row(webmention_uuid)) is None:
+ return
+
+ try:
+ status, reason = check_source(claim)
+ except (TemporaryFetchError, httpx.RequestError) as exc:
+ release_claim(
+ claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched"
+ )
+ raise
+
+ release_claim(claim, status, reason)
def recover_stale_webmentions() -> int:
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 5ee2d9e..ce03185 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Reconcile scanned static-site sources with the database."""
+from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
@@ -113,38 +114,24 @@ def scan_sources() -> None:
_scan_locked()
-def _scan_locked() -> None:
- """Scan the sources while the caller holds the shared scan lock."""
- directory = current_app.config.get("SOURCE_DIRECTORY")
-
- if directory is None:
- return
-
- root = Path(directory).expanduser().resolve()
-
- if not root.is_dir():
- raise RuntimeError(f"Source directory is not a directory: {root}")
-
- if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
- if not is_http_url(base_url):
- raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
- base_url = f"{base_url.rstrip('/')}/"
+@dataclass(frozen=True)
+class ScanResult:
+ """Count and discovered paths from a traversal of the source tree."""
- sources_by_path = {
- source.path: source
- for source in db.session.scalars(
- sa.select(Source).options(selectinload(Source.sent_webmentions))
- )
- }
+ scanned_count: int
+ seen_paths: frozenset[str]
- # Existing sources retain their canonical URLs even when marked deleted.
- # Reserve these URLs before scanning so an alias cannot claim one just
- # because it is discovered earlier than its persisted owner.
- paths_by_url = {source.url: source.path for source in sources_by_path.values()}
+def scan_tree(
+ root: Path,
+ base_url: str | None,
+ sources_by_path: dict[str, Source],
+ paths_by_url: dict[str, str],
+ scan_time: datetime,
+) -> ScanResult:
+ """Reconcile found pages and return the paths seen, even on parse failures."""
seen_paths: set[str] = set()
scanned_count = 0
- scan_time = datetime.now(UTC)
for index_path in root.rglob("index.html"):
relative_path = index_path.relative_to(root)
@@ -152,6 +139,7 @@ def _scan_locked() -> None:
if relative_path.parent == Path():
continue
+ # A failed parse still proves the path exists; do not mark it deleted.
seen_paths.add(relative_path.as_posix())
try:
@@ -165,8 +153,7 @@ def _scan_locked() -> None:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
continue
except Exception:
- # Third-party HTML parsers can raise unexpected exceptions (including
- # RecursionError). A bad page must not prevent other sources being saved.
+ # A bad page must not prevent other sources being saved.
current_app.logger.exception(
"Unexpected error scanning source %s", index_path
)
@@ -196,13 +183,14 @@ def _scan_locked() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- if scanned_count == 0:
- current_app.logger.warning(
- "No source documents successfully scanned; "
- "skipping deletion detection and Webmention sends"
- )
- return
+ return ScanResult(scanned_count, frozenset(seen_paths))
+
+def mark_missing(
+ sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
+) -> int:
+ """Mark absent, previously active sources deleted and return their count."""
+ marked = 0
for path, source in sources_by_path.items():
if source.deleted_at is not None or path in seen_paths:
continue
@@ -210,6 +198,7 @@ def _scan_locked() -> None:
source.revision += 1
source.revised_at = scan_time
source.deleted_at = scan_time
+ marked += 1
for webmention in source.sent_webmentions:
webmention.reopen(source.revision, False)
@@ -217,10 +206,12 @@ def _scan_locked() -> None:
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
- db.session.commit()
+ return marked
- pending_count = 0
+def queue_pending(scan_time: datetime) -> int:
+ """Enqueue due Webmentions and return the number of tasks submitted."""
+ queued = 0
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
.where(
@@ -233,11 +224,55 @@ def _scan_locked() -> None:
.order_by(SentWebmention.uuid)
):
send_webmention(identifier)
- pending_count += 1
+ queued += 1
+
+ return queued
+
+
+def _scan_locked() -> None:
+ """Scan sources while the caller holds the shared scan lock."""
+ directory = current_app.config.get("SOURCE_DIRECTORY")
+ if directory is None:
+ return
+
+ root = Path(directory).expanduser().resolve()
+ if not root.is_dir():
+ raise RuntimeError(f"Source directory is not a directory: {root}")
+
+ if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
+ if not is_http_url(base_url):
+ raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
+ base_url = f"{base_url.rstrip('/')}/"
+
+ sources_by_path = {
+ source.path: source
+ for source in db.session.scalars(
+ sa.select(Source).options(selectinload(Source.sent_webmentions))
+ )
+ }
+
+ # Even deleted sources reserve their canonical URL, so a later alias cannot
+ # take ownership merely by appearing earlier in the traversal.
+ paths_by_url = {source.url: source.path for source in sources_by_path.values()}
+ scan_time = datetime.now(UTC)
+ scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time)
+
+ if scan.scanned_count == 0:
+ current_app.logger.warning(
+ "No source documents successfully scanned; "
+ "skipping deletion detection and Webmention sends"
+ )
+ return
+
+ missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time)
+ db.session.commit()
+ pending_count = queue_pending(scan_time)
current_app.logger.info(
- "Webmention source scan complete: %d sources scanned, %d pending sends",
- scanned_count,
+ "Webmention source scan complete: %d sources scanned, "
+ "%d missing sources, %d pending sends",
+ scan.scanned_count,
+ missing_count,
pending_count,
)
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 92d8dfa..339dfe6 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -5,6 +5,7 @@
"""Claim, send, and persist outgoing Webmention attempts."""
import uuid
+from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import httpx
@@ -76,38 +77,64 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
-def _send_locked(webmention_uuid: uuid.UUID) -> None:
- """Process a Webmention while holding its per-UUID task lock."""
- webmention = db.session.get(SentWebmention, webmention_uuid)
+@dataclass(frozen=True)
+class SendAttempt:
+ """Claimed source revision and retry budget for one send."""
+
+ identifier: uuid.UUID
+ revision: int
+ number: int
+ max_attempts: int
+ source: str
+ target: str
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+ """HTTP outcome, with whether the failure is eligible for another attempt.
+
+ ``terminal`` describes the failure type. The retry budget can also make a
+ temporary failure terminal when its last allowed attempt has been used.
+ """
+
+ status: SentWebmentionStatus
+ reason: str | None = None
+ endpoint: str | None = None
+ response_status: int | None = None
+ status_url: str | None = None
+ terminal: bool = True
+ temporary_failure: bool = False
- if webmention is None:
- current_app.logger.warning(
- "Cannot send unknown SentWebmention %s", webmention_uuid
- )
- return
+def is_due(webmention: SentWebmention, now: datetime) -> bool:
+ """Return whether an unprocessed Webmention has reached its retry deadline."""
if not webmention.pending:
- return
+ return False
- revision = webmention.desired_revision
- source = webmention.source.url
- target = webmention.target
- now = datetime.now(UTC)
deadline = webmention.next_retry_at
+ if deadline is None:
+ return True
+
+ aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
+ return aware_deadline <= now
+
- if deadline is not None:
- aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
- if aware_deadline > now:
- return
+def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | None:
+ """Atomically reserve one send attempt, or finish an exhausted revision.
+ An attempt's next retry deadline is persisted *before* the HTTP request so
+ a lost worker cannot cause an immediate duplicate send.
+ """
+ revision = webmention.desired_revision
max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
+
if webmention.attempt_count >= max_attempts:
- # Recover records whose worker died during the final attempt, without
- # touching a newer revision that may have been queued in the meantime.
+ # A dead worker may have already consumed the final attempt. Never
+ # finish a newer revision or a retry whose deadline is still in future.
db.session.execute(
sa.update(SentWebmention)
.where(
- SentWebmention.uuid == webmention_uuid,
+ SentWebmention.uuid == webmention.uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count >= max_attempts,
SentWebmention.pending,
@@ -128,19 +155,22 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
.execution_options(synchronize_session=False)
)
db.session.commit()
- return
+ return None
- attempt = webmention.attempt_count + 1
- next_retry_at = now + retry_delay(attempt)
-
- # Claim the attempt atomically: duplicate queued tasks must not send the
- # same revision concurrently or bypass its retry deadline.
- claim = db.session.execute(
+ attempt = SendAttempt(
+ identifier=webmention.uuid,
+ revision=revision,
+ number=webmention.attempt_count + 1,
+ max_attempts=max_attempts,
+ source=webmention.source.url,
+ target=webmention.target,
+ )
+ changed = db.session.execute(
sa.update(SentWebmention)
.where(
- SentWebmention.uuid == webmention_uuid,
+ SentWebmention.uuid == attempt.identifier,
SentWebmention.desired_revision == revision,
- SentWebmention.attempt_count == attempt - 1,
+ SentWebmention.attempt_count == attempt.number - 1,
SentWebmention.pending,
sa.or_(
SentWebmention.next_retry_at.is_(None),
@@ -148,9 +178,9 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
),
)
.values(
- attempt_count=attempt,
+ attempt_count=attempt.number,
last_attempted_at=now,
- next_retry_at=next_retry_at,
+ next_retry_at=now + retry_delay(attempt.number),
endpoint=None,
response_status=None,
status_url=None,
@@ -159,17 +189,20 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
)
db.session.commit()
- if rows_affected(claim) != 1:
- return
+ if rows_affected(changed) != 1:
+ return None
db.session.refresh(webmention)
+ return attempt
+
+def deliver_webmention(
+ source: str, target: str, *, webmention_uuid: uuid.UUID
+) -> DeliveryResult:
+ """Discover and POST a Webmention, returning an outcome without DB writes."""
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
- reason: str | None = None
- terminal = True
- temporary_failure = False
try:
if not is_http_url(target):
@@ -179,19 +212,26 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- outcome = SentWebmentionStatus.UNSUPPORTED
- reason = "No Webmention endpoint discovered"
- else:
- response_status, status_url = post_webmention(
- client, endpoint=endpoint, source=source, target=target
+ return DeliveryResult(
+ status=SentWebmentionStatus.UNSUPPORTED,
+ reason="No Webmention endpoint discovered",
)
- if not 200 <= response_status <= 299:
- message = f"Webmention endpoint returned HTTP {response_status}"
- if is_temporary_status(response_status):
- raise TemporarySenderError(message)
- raise PermanentSenderError(message)
- outcome = SentWebmentionStatus.SENT
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+
+ return DeliveryResult(
+ status=SentWebmentionStatus.SENT,
+ endpoint=endpoint,
+ response_status=response_status,
+ status_url=status_url,
+ )
except (
PermanentSenderError,
UnsafeAddressError,
@@ -200,54 +240,108 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
httpx.LocalProtocolError,
) as exc:
# Known permanent failures cannot improve with retries.
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or type(exc).__name__
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or type(exc).__name__,
+ endpoint=endpoint,
+ response_status=response_status,
+ )
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or "Webmention request failed"
- terminal = attempt >= max_attempts
- temporary_failure = True
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or "Webmention request failed",
+ endpoint=endpoint,
+ response_status=response_status,
+ terminal=False,
+ temporary_failure=True,
+ )
except Exception as exc:
- # Unexpected errors must not disappear into a generic permanent-failure
- # handler without a traceback (including programming ValueErrors).
+ # Unexpected errors should be terminal and keep their traceback.
current_app.logger.exception(
"Unexpected error sending SentWebmention %s", webmention_uuid
)
- outcome = SentWebmentionStatus.FAILED
- reason = str(exc) or type(exc).__name__
+ return DeliveryResult(
+ status=SentWebmentionStatus.FAILED,
+ reason=str(exc) or type(exc).__name__,
+ endpoint=endpoint,
+ response_status=response_status,
+ )
+
+def record_outcome(
+ webmention: SentWebmention,
+ attempt: SendAttempt,
+ result: DeliveryResult,
+ *,
+ terminal: bool,
+) -> bool:
+ """Record an outcome only while the claimed source revision is current."""
# A scanner may have reopened or superseded the revision during I/O.
- # Refresh only once, and persist one outcome atomically with its metadata.
- if not attempt_is_current(webmention, revision):
- return
+ if not attempt_is_current(webmention, attempt.revision):
+ return False
- webmention.status = outcome
- webmention.failure_reason = reason
- webmention.endpoint = endpoint
- webmention.response_status = response_status
+ webmention.status = result.status
+ webmention.failure_reason = result.reason
+ webmention.endpoint = result.endpoint
+ webmention.response_status = result.response_status
if terminal:
- webmention.processed_revision = revision
+ webmention.processed_revision = attempt.revision
webmention.next_retry_at = None
- if outcome == SentWebmentionStatus.SENT:
- webmention.sent_revision = revision
+ if result.status == SentWebmentionStatus.SENT:
+ webmention.sent_revision = attempt.revision
webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
+ result.status_url
+ if result.status_url is not None and is_http_url(result.status_url)
+ else None
)
webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+ return True
+
- if temporary_failure and not terminal:
- # Huey does not retry: scanner requeues after next_retry_at.
+def _send_locked(webmention_uuid: uuid.UUID) -> bool:
+ """Process a due Webmention while holding its per-UUID task lock.
+
+ :return: Whether this worker recorded a delivery outcome.
+ """
+ webmention = db.session.get(SentWebmention, webmention_uuid)
+ if webmention is None:
+ current_app.logger.warning(
+ "Cannot send unknown SentWebmention %s", webmention_uuid
+ )
+ return False
+
+ now = datetime.now(UTC)
+ if not is_due(webmention, now):
+ return False
+
+ if (attempt := claim_attempt(webmention, now)) is None:
+ return False
+
+ result = deliver_webmention(
+ attempt.source, attempt.target, webmention_uuid=webmention_uuid
+ )
+ terminal = result.terminal or attempt.number >= attempt.max_attempts
+
+ if not record_outcome(webmention, attempt, result, terminal=terminal):
+ return False
+
+ if result.temporary_failure and not terminal:
+ # Huey does not retry: the scanner requeues after next_retry_at.
current_app.logger.info(
- "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ "Webmention %s send attempt %d failed; will retry",
+ webmention_uuid,
+ attempt.number,
)
- elif temporary_failure and terminal:
+ elif result.temporary_failure:
current_app.logger.warning(
"Webmention %s exhausted %d send attempts: %s",
webmention_uuid,
- attempt,
- reason,
+ attempt.number,
+ result.reason,
)
+
+ return True