diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/discovery.py | 231 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 53 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/json_serializer.py | 133 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/mentions.py | 99 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/notifications.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 281 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 414 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 405 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/source_parser.py | 390 |
11 files changed, 1001 insertions, 1029 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index fb1d46d..bf7189f 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,7 +21,7 @@ def create_consumer() -> Huey: """ app = create_app() - from . import notifications, receiver, retention, scanner, sender # noqa: F401 + from . import notifications, receiver, retention, scanner schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py new file mode 100644 index 0000000..98f3fd5 --- /dev/null +++ b/webmentions_ssg/tasks/discovery.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Discover a target Webmention endpoint and submit a POST.""" + +import re +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup +from flask import current_app + +from ..net import is_http_url, is_temporary_status, read_limited_body + +LINK_SPLIT = re.compile(r",\s*(?=<)") +DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} + + +class SenderError(Exception): + """A Webmention cannot be sent.""" + + +class TemporarySenderError(SenderError): + """A Webmention send failed for a potentially temporary reason.""" + + +class PermanentSenderError(SenderError): + """A Webmention send failed permanently for this source revision.""" + + +def resolve_endpoint(response: httpx.Response, href: str) -> str: + """ + Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :return: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or + HTTPS URL. + """ + try: + endpoint = urljoin(str(response.url), href.strip()) + except ValueError as exc: + raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc + + if not is_http_url(endpoint): + raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") + + return endpoint + + +def parse_link_value(value: str) -> tuple[str, set[str]] | None: + """ + Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :return: Link target and relation names, or ``None`` if the value is invalid. + """ + value = value.strip() + + if not value.startswith("<"): + return None + + href, separator, parameters = value[1:].partition(">") + + if not separator: + return None + + relations: set[str] = set() + + for parameter in parameters.split(";"): + name, separator, value = parameter.partition("=") + + if separator and name.strip().lower() == "rel": + relations.update( + relation.lower() for relation in value.strip().strip("\"'").split() + ) + + return href, relations + + +def endpoint_from_headers(response: httpx.Response) -> str | None: + """ + Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + for header in response.headers.get_list("Link"): + for value in LINK_SPLIT.split(header): + if (link := parse_link_value(value)) is None: + continue + + href, relations = link + + if "webmention" in relations: + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: + """ + Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + document = BeautifulSoup(body, "html5lib") + + for element in document.find_all(["link", "a"], href=True): + relations = element.get("rel") + + if isinstance(relations, str): + relations = relations.split() + + if not relations: + continue + + if not any( + isinstance(relation, str) and relation.lower() == "webmention" + for relation in relations + ): + continue + + href = element.get("href") + + if not isinstance(href, str): + continue + + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def read_target_body(response: httpx.Response) -> bytes: + """Return a bounded target body, using the shared HTTP size guard.""" + try: + return read_limited_body( + response, current_app.config["MAX_TARGET_BYTES"], label="Target" + ) + except ValueError as exc: + raise PermanentSenderError(str(exc)) from exc + + +def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: + """ + Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises TemporarySenderError: If the target returns a temporary HTTP failure. + :raises PermanentSenderError: If GET returns a permanent HTTP failure. + """ + try: + head_response = client.head(target, headers=DISCOVERY_HEADERS) + except httpx.RequestError, TemporarySenderError: + # HEAD is only an optimization. Servers and intermediaries may reject + # or drop it even when fetching the document with GET would succeed. + pass + else: + if endpoint := endpoint_from_headers(head_response): + return endpoint + + with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: + if endpoint := endpoint_from_headers(response): + return endpoint + + if is_temporary_status(response.status_code): + raise TemporarySenderError(f"Target returned HTTP {response.status_code}") + + if not response.is_success: + raise PermanentSenderError(f"Target returned HTTP {response.status_code}") + + media_type = ( + response.headers.get("Content-Type", "").partition(";")[0].strip().lower() + ) + + if media_type not in {"text/html", "application/xhtml+xml"}: + return None + + return endpoint_from_html(response, read_target_body(response)) + + +def post_webmention( + client: httpx.Client, *, endpoint: str, source: str, target: str +) -> tuple[int, str | None]: + """ + Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :return: HTTP response status code and optional status URL. + """ + with client.stream( + "POST", endpoint, data={"source": source, "target": target} + ) as response: + status_url = None + + if response.status_code == 201 and ( + location := response.headers.get("Location") + ): + try: + candidate = urljoin(str(response.url), location) + except ValueError: + # A malformed Location must not fail an otherwise successful send. + pass + else: + if is_http_url(candidate): + status_url = candidate + + return (response.status_code, status_url) diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index df6cb21..9f53f3b 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -93,54 +93,37 @@ class Huey: ) return self._huey + def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]: + """Return a task wrapper that executes with the configured Flask app.""" + + @wraps(func) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + if self.app is None: + raise RuntimeError("Flask app is not available.") + with self.app.app_context(): + return func(*args, **kwargs) + + return wrapped + def task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's task decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's task decorator. - :return: Decorator that registers the wrapped function as a Huey task. - """ + """Return a Huey task decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.task(*task_args, **task_kwargs)(wrapper) + return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func)) return decorator def periodic_task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a periodic Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's periodic task - decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's periodic task - decorator. - :return: Decorator that registers the wrapped function as a periodic Huey - task. - """ + """Return a periodic Huey decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper) + return self.huey.periodic_task(*task_args, **task_kwargs)( + self._in_app_context(func) + ) return decorator diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py index 473a544..7e51358 100644 --- a/webmentions_ssg/tasks/json_serializer.py +++ b/webmentions_ssg/tasks/json_serializer.py @@ -11,7 +11,7 @@ from typing import Any from huey.registry import Message from huey.serializer import Serializer -from huey.utils import Error, SKIPPED +from huey.utils import SKIPPED, Error class JSONSerializer(Serializer): @@ -39,66 +39,87 @@ class JSONSerializer(Serializer): @classmethod def _encode(cls, value: Any) -> Any: - if value is None or type(value) in {str, int, float, bool}: - return value - if isinstance(value, uuid.UUID): - return {"type": "uuid", "value": str(value)} - if isinstance(value, datetime): - return {"type": "datetime", "value": value.isoformat()} - if isinstance(value, Message): - return {"type": "message", "value": [cls._encode(v) for v in value]} - if isinstance(value, Error): - return {"type": "error", "value": cls._encode(value.metadata)} - if value is SKIPPED: - return {"type": "skipped", "value": None} - if isinstance(value, tuple): - return {"type": "tuple", "value": [cls._encode(v) for v in value]} - if isinstance(value, list): - return {"type": "list", "value": [cls._encode(v) for v in value]} - if isinstance(value, dict) and all(type(k) is str for k in value): - return { - "type": "dict", - "value": [[k, cls._encode(v)] for k, v in value.items()], - } - raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") + match value: + case _ if value is None or type(value) in {str, int, float, bool}: + return value + + case uuid.UUID(): + return {"type": "uuid", "value": str(value)} + + case datetime(): + return {"type": "datetime", "value": value.isoformat()} + + case Message(): + return {"type": "message", "value": [cls._encode(v) for v in value]} + + case Error(): + return {"type": "error", "value": cls._encode(value.metadata)} + + case _ if value is SKIPPED: + return {"type": "skipped", "value": None} + + case tuple(): + return {"type": "tuple", "value": [cls._encode(v) for v in value]} + + case list(): + return {"type": "list", "value": [cls._encode(v) for v in value]} + + case dict() if all(isinstance(k, str) for k in value): + return { + "type": "dict", + "value": [[k, cls._encode(v)] for k, v in value.items()], + } + + case _: + raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") @classmethod def _decode(cls, value: Any) -> Any: if value is None or type(value) in {str, int, float, bool}: return value + if not isinstance(value, dict) or set(value) != {"type", "value"}: raise ValueError("Invalid Huey JSON data") - kind, payload = value["type"], value["value"] - if kind == "uuid" and isinstance(payload, str): - return uuid.UUID(payload) - if kind == "datetime" and isinstance(payload, str): - return datetime.fromisoformat(payload) - if kind == "message" and isinstance(payload, list): - if len(payload) != len(Message._fields): - raise ValueError("Invalid Huey JSON message field count") - return Message(*(cls._decode(v) for v in payload)) - if kind == "error": - metadata = cls._decode(payload) - if not isinstance(metadata, dict): - raise ValueError("Invalid Huey JSON error metadata") - return Error(metadata) - if kind == "skipped" and payload is None: - return SKIPPED - if kind in {"tuple", "list"} and isinstance(payload, list): - items = [cls._decode(v) for v in payload] - return tuple(items) if kind == "tuple" else items - if kind == "dict" and isinstance(payload, list): - result: dict[str, Any] = {} - for pair in payload: - if ( - not isinstance(pair, list) - or len(pair) != 2 - or type(pair[0]) is not str - ): - raise ValueError("Invalid Huey JSON dictionary entry") - if pair[0] in result: - raise ValueError("Duplicate Huey JSON dictionary key") - result[pair[0]] = cls._decode(pair[1]) - return result - raise ValueError("Unsupported Huey JSON type") + match value["type"], value["value"]: + case "uuid", str() as payload: + return uuid.UUID(payload) + + case "datetime", str() as payload: + return datetime.fromisoformat(payload) + + case "message", list() as payload: + if len(payload) != len(Message._fields): + raise ValueError("Invalid Huey JSON message field count") + return Message(*(cls._decode(v) for v in payload)) + + case "error", payload: + metadata = cls._decode(payload) + if not isinstance(metadata, dict): + raise TypeError("Invalid Huey JSON error metadata") + return Error(metadata) + + case "skipped", None: + return SKIPPED + + case ("tuple" | "list") as kind, list() as payload: + items = [cls._decode(v) for v in payload] + return tuple(items) if kind == "tuple" else items + + case "dict", list() as payload: + result: dict[str, Any] = {} + + for pair in payload: + match pair: + case list([str() as key, encoded]): + if key in result: + raise ValueError("Duplicate Huey JSON dictionary key") + result[key] = cls._decode(encoded) + + case _: + raise ValueError("Invalid Huey JSON dictionary entry") + + return result + + case _: + raise ValueError("Unsupported Huey JSON type") diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py new file mode 100644 index 0000000..4e5a5ca --- /dev/null +++ b/webmentions_ssg/tasks/mentions.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure HTML and text matching for received Webmention targets.""" + +from urllib.parse import urljoin + +import rfc3987 +from bs4 import BeautifulSoup + +IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") +TRAILING_IRI_PUNCTUATION = ".,;:!?" + +HTML_URL_ATTRIBUTES = { + "href": {"a", "area", "link"}, + "src": { + "audio", + "embed", + "iframe", + "img", + 'input[type="image" i]', + "script", + "audio source", + "video source", + "track", + "video", + }, + "cite": {"blockquote", "del", "ins", "q"}, +} + + +def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: + """ + Check whether an HTML document mentions the target URL. + + URL references are resolved against the document's ``base`` element when + present, or against the source URL otherwise. + + :param body: HTML document body. + :param source_url: URL from which the document was retrieved. + :param target_url: Exact URL to look for. + :return: Whether a supported HTML URL attribute references the target URL. + """ + + document = BeautifulSoup(body, "html.parser") + base_url = source_url + + if (base_element := document.select_one("base[href]")) is not None and isinstance( + base_href := base_element.get("href"), str + ): + base_url = urljoin(source_url, base_href.strip()) + + for attribute, selectors in HTML_URL_ATTRIBUTES.items(): + selector = ", ".join( + [f"{selector_string}[{attribute}]" for selector_string in selectors] + ) + + for element in document.select(selector): + if not isinstance(reference := element.get(attribute), str): + continue + + if urljoin(base_url, reference.strip()) == target_url: + return True + + return False + + +def text_mentions_target(body: str, target_url: str) -> bool: + """ + Check whether plain text contains the target IRI, ignoring trailing prose + punctuation but not extra URL path or query characters. + + :param body: Plain-text document body. + :param target_url: Exact IRI to look for. + :return: Whether the target IRI occurs in the document. + """ + for match in IRI_PATTERN.finditer(body): + candidate = match.group() + + # A URL in prose can be followed by punctuation that the IRI pattern + # consumes as part of its match. Only remove trailing punctuation, + # leaving longer URLs (e.g. ``target/more``) unmatched. + if not candidate.startswith(target_url): + continue + + while candidate != target_url: + if candidate[-1] in TRAILING_IRI_PUNCTUATION: + candidate = candidate[:-1] + elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): + # Keep balanced parentheses that belong to the URL itself. + candidate = candidate[:-1] + else: + break + + if candidate == target_url: + return True + + return False diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index 475fad1..889030c 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -10,10 +10,10 @@ import sqlalchemy as sa from flask import current_app from flask_mail import Message -from .. import DATABASE as db -from .. import HUEY as huey -from .. import MAIL as mail -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..extensions import MAIL as mail +from ..models import ReceivedStatus, ReceivedWebmention @huey.lock_task("send-webmention-digest") @@ -27,7 +27,7 @@ def send_webmention_digest() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == "verified", + ReceivedWebmention.status == ReceivedStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) @@ -53,7 +53,6 @@ def send_webmention_digest() -> None: ), ) - # If SMTP fails, the records remain pending for the next digest. mail.send(message) db.session.execute( diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index acd66a1..bed923c 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -2,27 +2,27 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Fetch and verify incoming Webmentions against their source pages.""" +"""Fetch and verify incoming Webmentions with transactional state changes.""" import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx -import rfc3987 import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey -from ..models import ReceivedWebmention -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..models import ReceivedStatus, ReceivedWebmention +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_temporary_status, + read_limited_body, +) +from .mentions import html_mentions_target, text_mentions_target class VerificationError(Exception): @@ -37,95 +37,7 @@ class TemporaryFetchError(Exception): """Fetching the source may succeed when retried later.""" -IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") -TRAILING_IRI_PUNCTUATION = ".,;:!?" - - -HTML_URL_ATTRIBUTES = { - "href": {"a", "area", "link"}, - "src": { - "audio", - "embed", - "iframe", - "img", - 'input[type="image" i]', - "script", - "audio source", - "video source", - "track", - "video", - }, - "cite": {"blockquote", "del", "ins", "q"}, -} - - -def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """ - Check whether an HTML document mentions the target URL. - - URL references are resolved against the document's ``base`` element when - present, or against the source URL otherwise. - - :param body: HTML document body. - :param source_url: URL from which the document was retrieved. - :param target_url: Exact URL to look for. - :return: Whether a supported HTML URL attribute references the target URL. - """ - - document = BeautifulSoup(body, "html.parser") - base_url = source_url - - if (base_element := document.select_one("base[href]")) is not None and isinstance( - base_href := base_element.get("href"), str - ): - base_url = urljoin(source_url, base_href.strip()) - - for attribute, selectors in HTML_URL_ATTRIBUTES.items(): - selector = ", ".join( - [f"{selector_string}[{attribute}]" for selector_string in selectors] - ) - - for element in document.select(selector): - if not isinstance(reference := element.get(attribute), str): - continue - - if urljoin(base_url, reference.strip()) == target_url: - return True - - return False - - -def text_mentions_target(body: str, target_url: str) -> bool: - """ - Check whether plain text contains the target IRI, ignoring trailing prose - punctuation but not extra URL path or query characters. - - :param body: Plain-text document body. - :param target_url: Exact IRI to look for. - :return: Whether the target IRI occurs in the document. - """ - for match in IRI_PATTERN.finditer(body): - candidate = match.group() - - # A URL in prose can be followed by punctuation that the IRI pattern - # consumes as part of its match. Only remove trailing punctuation, - # leaving longer URLs (e.g. ``target/more``) unmatched. - if not candidate.startswith(target_url): - continue - - while candidate != target_url: - if candidate[-1] in TRAILING_IRI_PUNCTUATION: - candidate = candidate[:-1] - elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): - # Keep balanced parentheses that belong to the URL itself. - candidate = candidate[:-1] - else: - break - - if candidate == target_url: - return True - - return False +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: @@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ try: - return _fetch_source(source_url) + with ( + build_client( + f"{APP_NAME}/{VERSION} ReceivedWebmention", + accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if is_temporary_status(status): + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") + + try: + return response, read_limited_body( + response, current_app.config["MAX_SOURCE_BYTES"], label="Source" + ) + except ValueError as exc: + raise VerificationError(str(exc)) from exc + except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc -def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - with ( - httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client, - client.stream("GET", source_url) as response, - ): - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") - - max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass - - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_source_bytes: - raise VerificationError("Source document is too large") - - return response, bytes(body) - - def source_mentions_target(source_url: str, target_url: str) -> bool: """ Fetch a source and check whether it mentions the target URL. @@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - # Failed tasks may be retried by Huey. Completed and already-running tasks - # should not start another concurrent verification. - if webmention.status not in {"received", "failed"}: + if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: return - # A reaper-exhausted row must not be revived by a late Huey retry. + if ( - webmention.status == "failed" + webmention.status == ReceivedStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return @@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ReceivedWebmention.status == previous_status, ReceivedWebmention.updated_at == previous_updated_at, ) - .values(status="verifying", failure_reason=None, updated_at=started_at) + .values( + status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at + ) ) + if claimed.rowcount != 1: db.session.rollback() return - db.session.commit() - def finish(status: str, failure_reason: str | None) -> None: - # A timeout or a newer verification may have superseded this attempt. - # Do not let a late worker overwrite its state. - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == "verifying", - ReceivedWebmention.updated_at == started_at, - ) - .values( - status=status, - failure_reason=failure_reason, - updated_at=datetime.now(UTC), - ) - ) - db.session.commit() + db.session.commit() + status = None + failure_reason = None try: mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - finish("deleted", str(exc)) + status = ReceivedStatus.DELETED + failure_reason = str(exc) except VerificationError as exc: - finish("failed", str(exc)) + status = ReceivedStatus.FAILED + failure_reason = str(exc) except (TemporaryFetchError, httpx.RequestError) as exc: - finish("failed", str(exc) or "Source could not be fetched") + status = ReceivedStatus.FAILED + failure_reason = str(exc) or "Source could not be fetched" raise except Exception as exc: # InvalidURL is not a RequestError; other unexpected source errors must @@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", webmention_uuid ) - finish("failed", str(exc) or type(exc).__name__) + + status = ReceivedStatus.FAILED + failure_reason = str(exc) or type(exc).__name__ else: if mentions_target: - finish("verified", None) + status = ReceivedStatus.VERIFIED else: - finish("deleted", "Source does not mention target") + status = ReceivedStatus.DELETED + failure_reason = "Source does not mention target" + finally: + if status is not None: + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() def recover_stale_webmentions() -> int: @@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int: """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( sa.select( ReceivedWebmention.uuid, @@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.updated_at, ReceivedWebmention.recovery_attempts, ).where( - ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.status.in_( + (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + ), ReceivedWebmention.updated_at <= cutoff, ) ).all() @@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int: recovered = 0 exhausted = 0 max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( @@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.recovery_attempts == recovery_attempts, ) .values( - status="failed" if at_limit else "received", - failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, - updated_at=now, - recovery_attempts=( - recovery_attempts if at_limit else recovery_attempts + 1 - ), + **( + { + "status": ReceivedStatus.FAILED, + "failure_reason": RECOVERY_EXHAUSTED_REASON, + "updated_at": now, + "recovery_attempts": recovery_attempts, + } + if at_limit + else ReceivedWebmention.verification_reset_values( + at=now, recoveries=recovery_attempts + 1 + ) + ) ) ) + if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: exhausted += 1 continue diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 727b186..bf79512 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,8 +11,8 @@ import sqlalchemy as sa from flask import current_app from sqlalchemy.engine import CursorResult -from .. import DATABASE as db -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..models import ReceivedStatus, ReceivedWebmention def purge_old_received_webmentions() -> int: @@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int: cutoff = datetime.now(UTC) - timedelta( days=current_app.config["RECEIVED_RETENTION_DAYS"] ) + result = cast( CursorResult[Any], db.session.execute( sa.delete(ReceivedWebmention).where( - ReceivedWebmention.status.in_(("failed", "deleted")), + ReceivedWebmention.status.in_( + (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ), ReceivedWebmention.updated_at < cutoff, ) ), ) db.session.commit() + removed = result.rowcount if removed: current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + return removed diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 666eba7..b2d6def 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -2,404 +2,22 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Scan generated site pages to track outgoing Webmention targets.""" +"""Reconcile scanned static-site sources with the database.""" -import json -from collections.abc import Iterator -from dataclasses import dataclass from datetime import UTC, datetime -from fnmatch import fnmatchcase -from itertools import chain from pathlib import Path -from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit -import mf2py import sqlalchemy as sa -import xxhash -from bs4 import BeautifulSoup, NavigableString, Tag -from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload -from .. import DATABASE as db -from .. import HUEY as huey -from ..config import normalize_ignored_hostnames +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, Source -from ..url_security import is_http_url +from ..net import is_http_url from .sender import send_webmention - -REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") - -CONTENT_BLOCK_TAGS = frozenset( - { - "address", - "article", - "aside", - "blockquote", - "br", - "dd", - "div", - "dl", - "dt", - "figcaption", - "figure", - "footer", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "header", - "hr", - "li", - "main", - "ol", - "p", - "pre", - "section", - "table", - "td", - "th", - "tr", - "ul", - } -) - - -class SourceScanError(Exception): - """A source document cannot safely be processed.""" - - -@dataclass(frozen=True) -class ScannedSource: - path: str - url: str - content_hash: str - targets: frozenset[str] - - -def source_url_for_path(relative_path: Path, base_url: str) -> str: - """ - Derive the public source URL from a relative filesystem path. - - :param relative_path: Path of the source document relative to the source root. - :param base_url: Base URL under which source documents are published. - :return: Public URL corresponding to the source document. - """ - directory = relative_path.parent.as_posix() - - return urljoin(base_url, f"{quote(directory, safe='/')}/") - - -def canonical_url( - document: BeautifulSoup, *, relative_path: Path, base_url: str | None -) -> str | None: - """ - Return the canonical URL declared by a source document. - - Relative canonical URLs are resolved against the configured source base URL. - - :param document: Parsed HTML source document. - :param relative_path: Path of the source document relative to the source root. - :param base_url: Configured source base URL, if available. - :return: Canonical URL, or ``None`` if none is declared. - :raises SourceScanError: If the canonical URL is empty, cannot be resolved, - or does not resolve to an HTTP or HTTPS URL. - """ - if (link := document.select_one('link[rel~="canonical"][href]')) is not None: - href = link.get("href") - - if not isinstance(href, str) or not (href := href.strip()): - raise SourceScanError("Document contains an empty canonical URL") - - if is_http_url(href): - return href - - if base_url is None: - raise SourceScanError( - "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" - ) - - resolved = urljoin(source_url_for_path(relative_path, base_url), href) - - if not is_http_url(resolved): - raise SourceScanError( - f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" - ) - - return resolved - - -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """ - Yield URL values from a microformats property. - - :param entry: Parsed microformats entry. - :param property_name: Name of the property to inspect. - :return: Iterator over string values of the property. - """ - properties = entry.get("properties") - - if not isinstance(properties, dict): - return - - values = properties.get(property_name) - - if not isinstance(values, list): - return - - yield from (value for value in values if isinstance(value, str)) - - -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """ - Parse and validate an h-entry with mf2py. - - :param element: HTML element containing the h-entry. - :param base_url: Base URL used when parsing the microformats data. - :return: Parsed h-entry. - :raises SourceScanError: If parsing does not produce exactly one valid h-entry. - """ - parsed = mf2py.parse(doc=str(element), url=base_url) - - if not isinstance(parsed, dict): - raise SourceScanError("Microformats parser did not return a document object") - - items = parsed.get("items") - - if not isinstance(items, list) or len(items) != 1: - raise SourceScanError("Expected exactly one parsed h-entry") - - entry = items[0] - - if not isinstance(entry, dict): - raise SourceScanError("Parsed h-entry is not an object") - - types = entry.get("type") - - if not isinstance(types, list) or "h-entry" not in types: - raise SourceScanError("Parsed microformats item is not an h-entry") - - return entry - - -def primary_entry( - document: BeautifulSoup, source_url: str -) -> tuple[Tag, dict[str, Any]]: - """ - Return and validate the primary h-entry of a source document. - - The document must contain exactly one h-entry whose ``u-url`` matches the - source URL. - - :param document: Parsed HTML source document. - :param source_url: Public URL of the source document. - :return: h-entry element and its parsed microformats representation. - :raises SourceScanError: If the document does not contain exactly one h-entry - or its ``u-url`` does not match the source URL. - """ - entries = document.find_all(class_="h-entry") - - if len(entries) != 1: - raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") - - entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) - - urls = tuple(property_urls(mf2_entry, "url")) - - if source_url not in urls: - raise SourceScanError( - f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" - ) - - return entry_element, mf2_entry - - -def content_element(entry: Tag) -> Tag: - """ - Return the e-content element of an h-entry. - - :param entry: HTML element containing the h-entry. - :return: The h-entry's e-content element. - :raises SourceScanError: If the h-entry does not contain exactly one - e-content element. - """ - contents = entry.find_all(class_="e-content") - - if len(contents) != 1: - raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") - - return contents[0] - - -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """ - Resolve and validate a possible Webmention target URL. - - Empty values, non-HTTP URLs, and URLs referring to the source itself are - discarded. - - :param value: URL reference to normalize. - :param base_url: Base URL against which relative references are resolved. - :param source_url: URL of the source document. - :return: Normalized target URL, or ``None`` if the value is not a valid - Webmention target. - """ - value = value.strip() - - if not value: - return None - - target = urljoin(base_url, value) - - if not is_http_url(target): - return None - - if urldefrag(target)[0] == urldefrag(source_url)[0]: - return None - - return target - - -def iter_targets( - content: Tag, - mf2_entry: dict[str, Any], - *, - base_url: str, - source_url: str, - ignored_hostnames: tuple[str, ...] = (), -) -> Iterator[str]: - """ - Yield outgoing Webmention targets from an h-entry. - - Targets are collected from links in the entry content and supported - microformats reaction properties. Invalid, self-referencing, and ignored - targets are excluded. - - :param content: e-content element of the h-entry. - :param mf2_entry: Parsed microformats representation of the h-entry. - :param base_url: Base URL against which relative targets are resolved. - :param source_url: URL of the source document. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Iterator over outgoing Webmention target URLs. - """ - hrefs = ( - href - for element in content.find_all("a", href=True) - if isinstance(href := element.get("href"), str) - ) - - reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES - ) - - for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) - if target is not None: - hostname = urlsplit(target).hostname or "" - if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): - yield target - - -def normalized_content(content: Tag) -> str: - """Extract stable reader-facing content without generated HTML attributes.""" - - def extract(node: PageElement) -> str: - if isinstance(node, NavigableString): - return str(node) - if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: - return "" - - # A relative-time label may change on every site build even though its - # machine-readable date stays the same. - if node.name == "time" and isinstance(value := node.get("datetime"), str): - return value.strip() - - # Image-only posts also have meaningful content, despite having no text. - if node.name == "img": - return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " - - text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text - - return " ".join(extract(content).split()) - - -def scan_source_file( - path: Path, - *, - root: Path, - base_url: str | None, - ignored_hostnames: tuple[str, ...] = (), -) -> ScannedSource: - """ - Scan one generated source document for outgoing Webmentions. - - :param path: Path of the generated HTML document. - :param root: Root directory containing generated source documents. - :param base_url: Configured source base URL, if available. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Scanned source metadata and discovered targets. - :raises SourceScanError: If the document cannot be safely interpreted as a - Webmention source. - :raises OSError: If the source document cannot be read. - """ - # Use the path as published for its URL, but read only files whose - # resolved target stays inside the configured source directory. Resolving - # before reading also protects against symlinked index.html files. - resolved_root = root.resolve() - resolved_path = path.resolve() - if not resolved_path.is_relative_to(resolved_root): - raise SourceScanError(f"Source document escapes source directory: {path}") - - relative_path = path.relative_to(root) - document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) - - if source_url is None: - if base_url is None: - raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" - ) - - source_url = source_url_for_path(relative_path, base_url) - - entry_element, mf2_entry = primary_entry(document, source_url) - - content = content_element(entry_element) - - targets = frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ) - - # Fingerprint the meaningful post text and its outgoing targets rather than - # the full h-entry markup. Relative timestamps, classes, and other volatile - # metadata outside e-content must not cause repeat Webmentions. - stable_content = normalized_content(content) - fingerprint = json.dumps( - {"content": stable_content, "targets": sorted(targets)}, - ensure_ascii=False, - separators=(",", ":"), - ) - - return ScannedSource( - path=relative_path.as_posix(), - url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), - targets=targets, - ) +from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: @@ -466,10 +84,9 @@ def update_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.active = webmention.target in scanned.targets - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen( + source.revision, webmention.target in scanned.targets + ) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -523,10 +140,6 @@ def scan_sources_task() -> None: raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" - ignored_hostnames = normalize_ignored_hostnames( - current_app.config["IGNORED_HOSTNAMES"] - ) - sources_by_path = { source.path: source for source in db.session.scalars( @@ -555,7 +168,7 @@ def scan_sources_task() -> None: index_path, root=root, base_url=base_url, - ignored_hostnames=ignored_hostnames, + ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -592,10 +205,6 @@ def scan_sources_task() -> None: except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - # A static-site generator can temporarily empty its output directory while - # rebuilding. With no successfully scanned pages, absence is not evidence - # of deletion; do not generate removals or send pending Webmentions that - # might be verified while the site is unavailable. if scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " @@ -612,10 +221,7 @@ def scan_sources_task() -> None: source.deleted_at = scan_time for webmention in source.sent_webmentions: - webmention.active = False - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 9330247..3c4c717 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -2,272 +2,33 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Discover endpoints and deliver outgoing Webmentions.""" +"""Claim, send, and persist outgoing Webmention attempts.""" -import re import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from huey.exceptions import TaskLockedException from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, SentWebmentionStatus -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url - -LINK_SPLIT = re.compile(r",\s*(?=<)") - -DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} - - -class SenderError(Exception): - """A Webmention cannot be sent.""" - - -class TemporarySenderError(SenderError): - """A Webmention send failed for a potentially temporary reason.""" - - -class PermanentSenderError(SenderError): - """A Webmention send failed permanently for this source revision.""" - - -def temporary_http_status(status: int) -> bool: - """ - Return whether an HTTP status indicates a temporary failure. - - :param status: HTTP response status code. - :return: Whether the request should be retried. - """ - return status in {408, 425, 429} or 500 <= status <= 599 - - -def resolve_endpoint(response: httpx.Response, href: str) -> str: - """ - Resolve and validate a discovered Webmention endpoint. - - :param response: Response from which the endpoint was discovered. - :param href: Endpoint reference to resolve. - :return: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or - HTTPS URL. - """ - try: - endpoint = urljoin(str(response.url), href.strip()) - except ValueError as exc: - raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc - - if not is_http_url(endpoint): - raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") - - return endpoint - - -def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """ - Parse a Link header value into its target and relations. - - :param value: Link header value to parse. - :return: Link target and relation names, or ``None`` if the value is invalid. - """ - value = value.strip() - - if not value.startswith("<"): - return None - - href, separator, parameters = value[1:].partition(">") - - if not separator: - return None - - relations: set[str] = set() - - for parameter in parameters.split(";"): - name, separator, value = parameter.partition("=") - - if separator and name.strip().lower() == "rel": - relations.update( - relation.lower() for relation in value.strip().strip("\"'").split() - ) - - return href, relations - - -def endpoint_from_headers(response: httpx.Response) -> str | None: - """ - Find a Webmention endpoint in the response Link headers. - - :param response: HTTP response whose headers should be inspected. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - for header in response.headers.get_list("Link"): - for value in LINK_SPLIT.split(header): - if (link := parse_link_value(value)) is None: - continue - - href, relations = link - - if "webmention" in relations: - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """ - Find a Webmention endpoint in an HTML document. - - ``link`` and ``a`` elements are inspected in document order. - - :param response: HTTP response from which the document was retrieved. - :param body: HTML document body. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - document = BeautifulSoup(body, "html5lib") - - for element in document.find_all(["link", "a"], href=True): - relations = element.get("rel") - - if isinstance(relations, str): - relations = relations.split() - - if not relations: - continue - - if not any( - isinstance(relation, str) and relation.lower() == "webmention" - for relation in relations - ): - continue - - href = element.get("href") - - if not isinstance(href, str): - continue - - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def read_target_body(response: httpx.Response) -> bytes: - """ - Read a target document up to the configured size limit. - - :param response: Streaming HTTP response to read. - :return: Response body. - :raises PermanentSenderError: If the document exceeds the configured maximum - size. - """ - max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_bytes: - raise PermanentSenderError("Target document is too large") - except ValueError: - pass - - body = bytearray() - - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_bytes: - raise PermanentSenderError("Target document is too large") - - return bytes(body) - - -def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """ - Discover the Webmention endpoint advertised by a target URL. - - Endpoint discovery first checks the response to a HEAD request and then falls - back to a GET request, inspecting both HTTP Link headers and supported HTML - documents. - - :param client: HTTP client to use for discovery requests. - :param target: Target URL whose Webmention endpoint should be discovered. - :return: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If GET returns a permanent HTTP failure. - """ - try: - head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError, TemporarySenderError: - # HEAD is only an optimization. Servers and intermediaries may reject - # or drop it even when fetching the document with GET would succeed. - pass - else: - if endpoint := endpoint_from_headers(head_response): - return endpoint - - with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := endpoint_from_headers(response): - return endpoint - - if temporary_http_status(response.status_code): - raise TemporarySenderError(f"Target returned HTTP {response.status_code}") - - if not response.is_success: - raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) - - if media_type not in {"text/html", "application/xhtml+xml"}: - return None - - return endpoint_from_html(response, read_target_body(response)) - - -def post_webmention( - client: httpx.Client, *, endpoint: str, source: str, target: str -) -> tuple[int, str | None]: - """ - Send a Webmention to a discovered endpoint. - - :param client: HTTP client to use for the request. - :param endpoint: Webmention endpoint URL. - :param source: Source URL of the Webmention. - :param target: Target URL of the Webmention. - :return: HTTP response status code and optional status URL. - """ - with client.stream( - "POST", endpoint, data={"source": source, "target": target} - ) as response: - status_url = None - - if response.status_code == 201 and ( - location := response.headers.get("Location") - ): - try: - candidate = urljoin(str(response.url), location) - except ValueError: - # A malformed Location must not fail an otherwise successful send. - pass - else: - if is_http_url(candidate): - status_url = candidate - - return (response.status_code, status_url) +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_http_url, + is_temporary_status, +) +from .discovery import ( + PermanentSenderError, + TemporarySenderError, + discover_webmention_endpoint, + post_webmention, +) def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: now = datetime.now(UTC) deadline = webmention.next_retry_at - # SQLite returns timezone-naive datetimes even for timezone-aware columns. if deadline is not None: aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) if aware_deadline > now: @@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: endpoint: str | None = None response_status: int | None = None + status_url: str | None = None + reason: str | None = None + terminal = True + temporary_failure = False try: if not is_http_url(target): raise PermanentSenderError(f"Invalid target URL: {target!r}") - with httpx.Client( - headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")}, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client: + with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.UNSUPPORTED - webmention.failure_reason = "No Webmention endpoint discovered" - webmention.endpoint = None - webmention.response_status = None - - db.session.commit() - return - - (response_status, status_url) = post_webmention( - client, endpoint=endpoint, source=source, target=target - ) - - if not (200 <= response_status <= 299): - message = f"Webmention endpoint returned HTTP {response_status}" - - if temporary_http_status(response_status): - raise TemporarySenderError(message) - - raise PermanentSenderError(message) + outcome = SentWebmentionStatus.UNSUPPORTED + reason = "No Webmention endpoint discovered" + else: + response_status, status_url = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + if not 200 <= response_status <= 299: + message = f"Webmention endpoint returned HTTP {response_status}" + if is_temporary_status(response_status): + raise TemporarySenderError(message) + raise PermanentSenderError(message) + outcome = SentWebmentionStatus.SENT except ( PermanentSenderError, @@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: ValueError, UnicodeError, ) as exc: - # Malformed URLs and local request/protocol errors cannot be repaired - # by retrying this source revision. InvalidURL is not a RequestError; - # protocol errors are, so handle them before the retryable branch. - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or type(exc).__name__ - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - return - + # Invalid URLs and local protocol errors cannot improve with retries. + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or type(exc).__name__ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: - if not attempt_is_current(webmention, revision): - return - - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or "Webmention request failed" - if attempt >= max_attempts: - webmention.processed_revision = revision - webmention.next_retry_at = None - current_app.logger.warning( - "Webmention %s exhausted %d send attempts: %s", - webmention_uuid, - attempt, - webmention.failure_reason, - ) - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - - # No Huey retries: only the scanner may requeue after next_retry_at. - return + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or "Webmention request failed" + terminal = attempt >= max_attempts + temporary_failure = True + # A scanner may have reopened or superseded the revision during I/O. + # Refresh only once, and persist one outcome atomically with its metadata. if not attempt_is_current(webmention, revision): return - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.sent_revision = revision - - webmention.status = SentWebmentionStatus.SENT - webmention.failure_reason = None - + webmention.status = outcome + webmention.failure_reason = reason webmention.endpoint = endpoint webmention.response_status = response_status - webmention.status_url = ( - status_url if status_url is not None and is_http_url(status_url) else None - ) - webmention.last_sent_at = datetime.now(UTC) + if terminal: + webmention.processed_revision = revision + webmention.next_retry_at = None + + if outcome == SentWebmentionStatus.SENT: + webmention.sent_revision = revision + webmention.status_url = ( + status_url if status_url is not None and is_http_url(status_url) else None + ) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() + + if temporary_failure and not terminal: + # Huey does not retry: scanner requeues after next_retry_at. + current_app.logger.info( + "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt + ) + elif temporary_failure and terminal: + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + reason, + ) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py new file mode 100644 index 0000000..cb15ab6 --- /dev/null +++ b/webmentions_ssg/tasks/source_parser.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Extract canonical source URLs, content fingerprints, and outgoing links.""" + +import json +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from fnmatch import fnmatchcase +from itertools import chain +from pathlib import Path +from typing import Any +from urllib.parse import quote, urldefrag, urljoin, urlsplit + +import mf2py +import xxhash +from bs4 import BeautifulSoup, Tag +from bs4.element import NavigableString, PageElement + +from ..net import is_http_url + +REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") + +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + + +class SourceScanError(Exception): + """A source document cannot safely be processed.""" + + +@dataclass(frozen=True) +class ScannedSource: + path: str + url: str + content_hash: str + targets: frozenset[str] + + +def source_url_for_path(relative_path: Path, base_url: str) -> str: + """ + Derive the public source URL from a relative filesystem path. + + :param relative_path: Path of the source document relative to the source root. + :param base_url: Base URL under which source documents are published. + :return: Public URL corresponding to the source document. + """ + directory = relative_path.parent.as_posix() + + return urljoin(base_url, f"{quote(directory, safe='/')}/") + + +def canonical_url( + document: BeautifulSoup, *, relative_path: Path, base_url: str | None +) -> str | None: + """ + Return the canonical URL declared by a source document. + + Relative canonical URLs are resolved against the configured source base URL. + + :param document: Parsed HTML source document. + :param relative_path: Path of the source document relative to the source root. + :param base_url: Configured source base URL, if available. + :return: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, + or does not resolve to an HTTP or HTTPS URL. + """ + if (link := document.select_one('link[rel~="canonical"][href]')) is not None: + href = link.get("href") + + if not isinstance(href, str) or not (href := href.strip()): + raise SourceScanError("Document contains an empty canonical URL") + + if is_http_url(href): + return href + + if base_url is None: + raise SourceScanError( + "Document contains a relative canonical URL, but " + "SOURCE_BASE_URL is not configured" + ) + + resolved = urljoin(source_url_for_path(relative_path, base_url), href) + + if not is_http_url(resolved): + raise SourceScanError( + f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" + ) + + return resolved + + +def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: + """ + Yield URL values from a microformats property. + + :param entry: Parsed microformats entry. + :param property_name: Name of the property to inspect. + :return: Iterator over string values of the property. + """ + properties = entry.get("properties") + + if not isinstance(properties, dict): + return + + values = properties.get(property_name) + + if not isinstance(values, list): + return + + yield from (value for value in values if isinstance(value, str)) + + +def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: + """ + Parse and validate an h-entry with mf2py. + + :param element: HTML element containing the h-entry. + :param base_url: Base URL used when parsing the microformats data. + :return: Parsed h-entry. + :raises SourceScanError: If parsing does not produce exactly one valid h-entry. + """ + parsed = mf2py.parse(doc=str(element), url=base_url) + + if not isinstance(parsed, dict): + raise SourceScanError("Microformats parser did not return a document object") + + items = parsed.get("items") + + if not isinstance(items, list) or len(items) != 1: + raise SourceScanError("Expected exactly one parsed h-entry") + + entry = items[0] + + if not isinstance(entry, dict): + raise SourceScanError("Parsed h-entry is not an object") + + types = entry.get("type") + + if not isinstance(types, list) or "h-entry" not in types: + raise SourceScanError("Parsed microformats item is not an h-entry") + + return entry + + +def primary_entry( + document: BeautifulSoup, source_url: str +) -> tuple[Tag, dict[str, Any]]: + """ + Return and validate the primary h-entry of a source document. + + The document must contain exactly one h-entry whose ``u-url`` matches the + source URL. + + :param document: Parsed HTML source document. + :param source_url: Public URL of the source document. + :return: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry + or its ``u-url`` does not match the source URL. + """ + entries = document.find_all(class_="h-entry") + + if len(entries) != 1: + raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") + + entry_element = entries[0] + mf2_entry = parse_entry(entry_element, source_url) + + urls = tuple(property_urls(mf2_entry, "url")) + + if source_url not in urls: + raise SourceScanError( + f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" + ) + + return entry_element, mf2_entry + + +def content_element(entry: Tag) -> Tag: + """ + Return the e-content element of an h-entry. + + :param entry: HTML element containing the h-entry. + :return: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one + e-content element. + """ + contents = entry.find_all(class_="e-content") + + if len(contents) != 1: + raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") + + return contents[0] + + +def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: + """ + Resolve and validate a possible Webmention target URL. + + Empty values, non-HTTP URLs, and URLs referring to the source itself are + discarded. + + :param value: URL reference to normalize. + :param base_url: Base URL against which relative references are resolved. + :param source_url: URL of the source document. + :return: Normalized target URL, or ``None`` if the value is not a valid + Webmention target. + """ + value = value.strip() + + if not value: + return None + + target = urljoin(base_url, value) + + if not is_http_url(target): + return None + + if urldefrag(target)[0] == urldefrag(source_url)[0]: + return None + + return target + + +def iter_targets( + content: Tag, + mf2_entry: dict[str, Any], + *, + base_url: str, + source_url: str, + ignored_hostnames: Iterable[str] = (), +) -> Iterator[str]: + """ + Yield outgoing Webmention targets from an h-entry. + + Targets are collected from links in the entry content and supported + microformats reaction properties. Invalid, self-referencing, and ignored + targets are excluded. + + :param content: e-content element of the h-entry. + :param mf2_entry: Parsed microformats representation of the h-entry. + :param base_url: Base URL against which relative targets are resolved. + :param source_url: URL of the source document. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Iterator over outgoing Webmention target URLs. + """ + hrefs = ( + href + for element in content.find_all("a", href=True) + if isinstance(href := element.get("href"), str) + ) + + reactions = chain.from_iterable( + property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + ) + + for value in chain(hrefs, reactions): + target = normalize_target(value, base_url=base_url, source_url=source_url) + if target is not None: + hostname = (urlsplit(target).hostname or "").rstrip(".") + if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): + yield target + + +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + +def scan_source_file( + path: Path, + *, + root: Path, + base_url: str | None, + ignored_hostnames: Iterable[str] = (), +) -> ScannedSource: + """ + Scan one generated source document for outgoing Webmentions. + + :param path: Path of the generated HTML document. + :param root: Root directory containing generated source documents. + :param base_url: Configured source base URL, if available. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Scanned source metadata and discovered targets. + :raises SourceScanError: If the document cannot be safely interpreted as a + Webmention source. + :raises OSError: If the source document cannot be read. + """ + # Use the path as published for its URL, but read only files whose + # resolved target stays inside the configured source directory. Resolving + # before reading also protects against symlinked index.html files. + resolved_root = root.resolve() + resolved_path = path.resolve() + if not resolved_path.is_relative_to(resolved_root): + raise SourceScanError(f"Source document escapes source directory: {path}") + + relative_path = path.relative_to(root) + document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") + + source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + + if source_url is None: + if base_url is None: + raise SourceScanError( + "Document has no canonical URL and SOURCE_BASE_URL is not configured" + ) + + source_url = source_url_for_path(relative_path, base_url) + + entry_element, mf2_entry = primary_entry(document, source_url) + + content = content_element(entry_element) + + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + + return ScannedSource( + path=relative_path.as_posix(), + url=source_url, + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, + ) |
