diff options
| author | Dennis Fink | 2026-11-12 19:57:08 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-12 19:57:08 +0100 |
| commit | 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch) | |
| tree | f7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/sender.py | |
| parent | 7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff) | |
| download | webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip | |
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and
application extension setup so each module owns a focused concern.
Reorganize the test suite around the new boundaries while preserving
existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg/tasks/sender.py')
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 405 |
1 files changed, 67 insertions, 338 deletions
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 9330247..3c4c717 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -2,272 +2,33 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Discover endpoints and deliver outgoing Webmentions.""" +"""Claim, send, and persist outgoing Webmention attempts.""" -import re import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from huey.exceptions import TaskLockedException from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, SentWebmentionStatus -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url - -LINK_SPLIT = re.compile(r",\s*(?=<)") - -DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} - - -class SenderError(Exception): - """A Webmention cannot be sent.""" - - -class TemporarySenderError(SenderError): - """A Webmention send failed for a potentially temporary reason.""" - - -class PermanentSenderError(SenderError): - """A Webmention send failed permanently for this source revision.""" - - -def temporary_http_status(status: int) -> bool: - """ - Return whether an HTTP status indicates a temporary failure. - - :param status: HTTP response status code. - :return: Whether the request should be retried. - """ - return status in {408, 425, 429} or 500 <= status <= 599 - - -def resolve_endpoint(response: httpx.Response, href: str) -> str: - """ - Resolve and validate a discovered Webmention endpoint. - - :param response: Response from which the endpoint was discovered. - :param href: Endpoint reference to resolve. - :return: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or - HTTPS URL. - """ - try: - endpoint = urljoin(str(response.url), href.strip()) - except ValueError as exc: - raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc - - if not is_http_url(endpoint): - raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") - - return endpoint - - -def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """ - Parse a Link header value into its target and relations. - - :param value: Link header value to parse. - :return: Link target and relation names, or ``None`` if the value is invalid. - """ - value = value.strip() - - if not value.startswith("<"): - return None - - href, separator, parameters = value[1:].partition(">") - - if not separator: - return None - - relations: set[str] = set() - - for parameter in parameters.split(";"): - name, separator, value = parameter.partition("=") - - if separator and name.strip().lower() == "rel": - relations.update( - relation.lower() for relation in value.strip().strip("\"'").split() - ) - - return href, relations - - -def endpoint_from_headers(response: httpx.Response) -> str | None: - """ - Find a Webmention endpoint in the response Link headers. - - :param response: HTTP response whose headers should be inspected. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - for header in response.headers.get_list("Link"): - for value in LINK_SPLIT.split(header): - if (link := parse_link_value(value)) is None: - continue - - href, relations = link - - if "webmention" in relations: - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """ - Find a Webmention endpoint in an HTML document. - - ``link`` and ``a`` elements are inspected in document order. - - :param response: HTTP response from which the document was retrieved. - :param body: HTML document body. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - document = BeautifulSoup(body, "html5lib") - - for element in document.find_all(["link", "a"], href=True): - relations = element.get("rel") - - if isinstance(relations, str): - relations = relations.split() - - if not relations: - continue - - if not any( - isinstance(relation, str) and relation.lower() == "webmention" - for relation in relations - ): - continue - - href = element.get("href") - - if not isinstance(href, str): - continue - - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def read_target_body(response: httpx.Response) -> bytes: - """ - Read a target document up to the configured size limit. - - :param response: Streaming HTTP response to read. - :return: Response body. - :raises PermanentSenderError: If the document exceeds the configured maximum - size. - """ - max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_bytes: - raise PermanentSenderError("Target document is too large") - except ValueError: - pass - - body = bytearray() - - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_bytes: - raise PermanentSenderError("Target document is too large") - - return bytes(body) - - -def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """ - Discover the Webmention endpoint advertised by a target URL. - - Endpoint discovery first checks the response to a HEAD request and then falls - back to a GET request, inspecting both HTTP Link headers and supported HTML - documents. - - :param client: HTTP client to use for discovery requests. - :param target: Target URL whose Webmention endpoint should be discovered. - :return: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If GET returns a permanent HTTP failure. - """ - try: - head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError, TemporarySenderError: - # HEAD is only an optimization. Servers and intermediaries may reject - # or drop it even when fetching the document with GET would succeed. - pass - else: - if endpoint := endpoint_from_headers(head_response): - return endpoint - - with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := endpoint_from_headers(response): - return endpoint - - if temporary_http_status(response.status_code): - raise TemporarySenderError(f"Target returned HTTP {response.status_code}") - - if not response.is_success: - raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) - - if media_type not in {"text/html", "application/xhtml+xml"}: - return None - - return endpoint_from_html(response, read_target_body(response)) - - -def post_webmention( - client: httpx.Client, *, endpoint: str, source: str, target: str -) -> tuple[int, str | None]: - """ - Send a Webmention to a discovered endpoint. - - :param client: HTTP client to use for the request. - :param endpoint: Webmention endpoint URL. - :param source: Source URL of the Webmention. - :param target: Target URL of the Webmention. - :return: HTTP response status code and optional status URL. - """ - with client.stream( - "POST", endpoint, data={"source": source, "target": target} - ) as response: - status_url = None - - if response.status_code == 201 and ( - location := response.headers.get("Location") - ): - try: - candidate = urljoin(str(response.url), location) - except ValueError: - # A malformed Location must not fail an otherwise successful send. - pass - else: - if is_http_url(candidate): - status_url = candidate - - return (response.status_code, status_url) +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_http_url, + is_temporary_status, +) +from .discovery import ( + PermanentSenderError, + TemporarySenderError, + discover_webmention_endpoint, + post_webmention, +) def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: now = datetime.now(UTC) deadline = webmention.next_retry_at - # SQLite returns timezone-naive datetimes even for timezone-aware columns. if deadline is not None: aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) if aware_deadline > now: @@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: endpoint: str | None = None response_status: int | None = None + status_url: str | None = None + reason: str | None = None + terminal = True + temporary_failure = False try: if not is_http_url(target): raise PermanentSenderError(f"Invalid target URL: {target!r}") - with httpx.Client( - headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")}, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client: + with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.UNSUPPORTED - webmention.failure_reason = "No Webmention endpoint discovered" - webmention.endpoint = None - webmention.response_status = None - - db.session.commit() - return - - (response_status, status_url) = post_webmention( - client, endpoint=endpoint, source=source, target=target - ) - - if not (200 <= response_status <= 299): - message = f"Webmention endpoint returned HTTP {response_status}" - - if temporary_http_status(response_status): - raise TemporarySenderError(message) - - raise PermanentSenderError(message) + outcome = SentWebmentionStatus.UNSUPPORTED + reason = "No Webmention endpoint discovered" + else: + response_status, status_url = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + if not 200 <= response_status <= 299: + message = f"Webmention endpoint returned HTTP {response_status}" + if is_temporary_status(response_status): + raise TemporarySenderError(message) + raise PermanentSenderError(message) + outcome = SentWebmentionStatus.SENT except ( PermanentSenderError, @@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: ValueError, UnicodeError, ) as exc: - # Malformed URLs and local request/protocol errors cannot be repaired - # by retrying this source revision. InvalidURL is not a RequestError; - # protocol errors are, so handle them before the retryable branch. - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or type(exc).__name__ - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - return - + # Invalid URLs and local protocol errors cannot improve with retries. + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or type(exc).__name__ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: - if not attempt_is_current(webmention, revision): - return - - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or "Webmention request failed" - if attempt >= max_attempts: - webmention.processed_revision = revision - webmention.next_retry_at = None - current_app.logger.warning( - "Webmention %s exhausted %d send attempts: %s", - webmention_uuid, - attempt, - webmention.failure_reason, - ) - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - - # No Huey retries: only the scanner may requeue after next_retry_at. - return + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or "Webmention request failed" + terminal = attempt >= max_attempts + temporary_failure = True + # A scanner may have reopened or superseded the revision during I/O. + # Refresh only once, and persist one outcome atomically with its metadata. if not attempt_is_current(webmention, revision): return - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.sent_revision = revision - - webmention.status = SentWebmentionStatus.SENT - webmention.failure_reason = None - + webmention.status = outcome + webmention.failure_reason = reason webmention.endpoint = endpoint webmention.response_status = response_status - webmention.status_url = ( - status_url if status_url is not None and is_http_url(status_url) else None - ) - webmention.last_sent_at = datetime.now(UTC) + if terminal: + webmention.processed_revision = revision + webmention.next_retry_at = None + + if outcome == SentWebmentionStatus.SENT: + webmention.sent_revision = revision + webmention.status_url = ( + status_url if status_url is not None and is_http_url(status_url) else None + ) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() + + if temporary_failure and not terminal: + # Huey does not retry: scanner requeues after next_retry_at. + current_app.logger.info( + "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt + ) + elif temporary_failure and terminal: + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + reason, + ) |
