aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/sender.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/sender.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg/tasks/sender.py')
-rw-r--r--webmentions_ssg/tasks/sender.py405
1 files changed, 67 insertions, 338 deletions
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 9330247..3c4c717 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -2,272 +2,33 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Discover endpoints and deliver outgoing Webmentions."""
+"""Claim, send, and persist outgoing Webmention attempts."""
-import re
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from huey.exceptions import TaskLockedException
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
-
-LINK_SPLIT = re.compile(r",\s*(?=<)")
-
-DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
-
-
-class SenderError(Exception):
- """A Webmention cannot be sent."""
-
-
-class TemporarySenderError(SenderError):
- """A Webmention send failed for a potentially temporary reason."""
-
-
-class PermanentSenderError(SenderError):
- """A Webmention send failed permanently for this source revision."""
-
-
-def temporary_http_status(status: int) -> bool:
- """
- Return whether an HTTP status indicates a temporary failure.
-
- :param status: HTTP response status code.
- :return: Whether the request should be retried.
- """
- return status in {408, 425, 429} or 500 <= status <= 599
-
-
-def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """
- Resolve and validate a discovered Webmention endpoint.
-
- :param response: Response from which the endpoint was discovered.
- :param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
- """
- try:
- endpoint = urljoin(str(response.url), href.strip())
- except ValueError as exc:
- raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
-
- if not is_http_url(endpoint):
- raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
-
- return endpoint
-
-
-def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """
- Parse a Link header value into its target and relations.
-
- :param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
- """
- value = value.strip()
-
- if not value.startswith("<"):
- return None
-
- href, separator, parameters = value[1:].partition(">")
-
- if not separator:
- return None
-
- relations: set[str] = set()
-
- for parameter in parameters.split(";"):
- name, separator, value = parameter.partition("=")
-
- if separator and name.strip().lower() == "rel":
- relations.update(
- relation.lower() for relation in value.strip().strip("\"'").split()
- )
-
- return href, relations
-
-
-def endpoint_from_headers(response: httpx.Response) -> str | None:
- """
- Find a Webmention endpoint in the response Link headers.
-
- :param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- for header in response.headers.get_list("Link"):
- for value in LINK_SPLIT.split(header):
- if (link := parse_link_value(value)) is None:
- continue
-
- href, relations = link
-
- if "webmention" in relations:
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """
- Find a Webmention endpoint in an HTML document.
-
- ``link`` and ``a`` elements are inspected in document order.
-
- :param response: HTTP response from which the document was retrieved.
- :param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- document = BeautifulSoup(body, "html5lib")
-
- for element in document.find_all(["link", "a"], href=True):
- relations = element.get("rel")
-
- if isinstance(relations, str):
- relations = relations.split()
-
- if not relations:
- continue
-
- if not any(
- isinstance(relation, str) and relation.lower() == "webmention"
- for relation in relations
- ):
- continue
-
- href = element.get("href")
-
- if not isinstance(href, str):
- continue
-
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def read_target_body(response: httpx.Response) -> bytes:
- """
- Read a target document up to the configured size limit.
-
- :param response: Streaming HTTP response to read.
- :return: Response body.
- :raises PermanentSenderError: If the document exceeds the configured maximum
- size.
- """
- max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_bytes:
- raise PermanentSenderError("Target document is too large")
- except ValueError:
- pass
-
- body = bytearray()
-
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_bytes:
- raise PermanentSenderError("Target document is too large")
-
- return bytes(body)
-
-
-def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """
- Discover the Webmention endpoint advertised by a target URL.
-
- Endpoint discovery first checks the response to a HEAD request and then falls
- back to a GET request, inspecting both HTTP Link headers and supported HTML
- documents.
-
- :param client: HTTP client to use for discovery requests.
- :param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
- :raises TemporarySenderError: If the target returns a temporary HTTP failure.
- :raises PermanentSenderError: If GET returns a permanent HTTP failure.
- """
- try:
- head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
- # HEAD is only an optimization. Servers and intermediaries may reject
- # or drop it even when fetching the document with GET would succeed.
- pass
- else:
- if endpoint := endpoint_from_headers(head_response):
- return endpoint
-
- with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
- if endpoint := endpoint_from_headers(response):
- return endpoint
-
- if temporary_http_status(response.status_code):
- raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
-
- if not response.is_success:
- raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
-
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
-
- if media_type not in {"text/html", "application/xhtml+xml"}:
- return None
-
- return endpoint_from_html(response, read_target_body(response))
-
-
-def post_webmention(
- client: httpx.Client, *, endpoint: str, source: str, target: str
-) -> tuple[int, str | None]:
- """
- Send a Webmention to a discovered endpoint.
-
- :param client: HTTP client to use for the request.
- :param endpoint: Webmention endpoint URL.
- :param source: Source URL of the Webmention.
- :param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
- """
- with client.stream(
- "POST", endpoint, data={"source": source, "target": target}
- ) as response:
- status_url = None
-
- if response.status_code == 201 and (
- location := response.headers.get("Location")
- ):
- try:
- candidate = urljoin(str(response.url), location)
- except ValueError:
- # A malformed Location must not fail an otherwise successful send.
- pass
- else:
- if is_http_url(candidate):
- status_url = candidate
-
- return (response.status_code, status_url)
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_http_url,
+ is_temporary_status,
+)
+from .discovery import (
+ PermanentSenderError,
+ TemporarySenderError,
+ discover_webmention_endpoint,
+ post_webmention,
+)
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
now = datetime.now(UTC)
deadline = webmention.next_retry_at
- # SQLite returns timezone-naive datetimes even for timezone-aware columns.
if deadline is not None:
aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
if aware_deadline > now:
@@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
endpoint: str | None = None
response_status: int | None = None
+ status_url: str | None = None
+ reason: str | None = None
+ terminal = True
+ temporary_failure = False
try:
if not is_http_url(target):
raise PermanentSenderError(f"Invalid target URL: {target!r}")
- with httpx.Client(
- headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")},
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client:
+ with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.UNSUPPORTED
- webmention.failure_reason = "No Webmention endpoint discovered"
- webmention.endpoint = None
- webmention.response_status = None
-
- db.session.commit()
- return
-
- (response_status, status_url) = post_webmention(
- client, endpoint=endpoint, source=source, target=target
- )
-
- if not (200 <= response_status <= 299):
- message = f"Webmention endpoint returned HTTP {response_status}"
-
- if temporary_http_status(response_status):
- raise TemporarySenderError(message)
-
- raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.UNSUPPORTED
+ reason = "No Webmention endpoint discovered"
+ else:
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.SENT
except (
PermanentSenderError,
@@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
ValueError,
UnicodeError,
) as exc:
- # Malformed URLs and local request/protocol errors cannot be repaired
- # by retrying this source revision. InvalidURL is not a RequestError;
- # protocol errors are, so handle them before the retryable branch.
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or type(exc).__name__
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
- return
-
+ # Invalid URLs and local protocol errors cannot improve with retries.
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or type(exc).__name__
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or "Webmention request failed"
- if attempt >= max_attempts:
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- current_app.logger.warning(
- "Webmention %s exhausted %d send attempts: %s",
- webmention_uuid,
- attempt,
- webmention.failure_reason,
- )
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
-
- # No Huey retries: only the scanner may requeue after next_retry_at.
- return
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or "Webmention request failed"
+ terminal = attempt >= max_attempts
+ temporary_failure = True
+ # A scanner may have reopened or superseded the revision during I/O.
+ # Refresh only once, and persist one outcome atomically with its metadata.
if not attempt_is_current(webmention, revision):
return
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.sent_revision = revision
-
- webmention.status = SentWebmentionStatus.SENT
- webmention.failure_reason = None
-
+ webmention.status = outcome
+ webmention.failure_reason = reason
webmention.endpoint = endpoint
webmention.response_status = response_status
- webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
- )
- webmention.last_sent_at = datetime.now(UTC)
+ if terminal:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+
+ if outcome == SentWebmentionStatus.SENT:
+ webmention.sent_revision = revision
+ webmention.status_url = (
+ status_url if status_url is not None and is_http_url(status_url) else None
+ )
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+
+ if temporary_failure and not terminal:
+ # Huey does not retry: scanner requeues after next_retry_at.
+ current_app.logger.info(
+ "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ )
+ elif temporary_failure and terminal:
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ reason,
+ )