aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/consumer.py2
-rw-r--r--webmentions_ssg/tasks/discovery.py231
-rw-r--r--webmentions_ssg/tasks/extension.py53
-rw-r--r--webmentions_ssg/tasks/json_serializer.py133
-rw-r--r--webmentions_ssg/tasks/mentions.py99
-rw-r--r--webmentions_ssg/tasks/notifications.py11
-rw-r--r--webmentions_ssg/tasks/receiver.py281
-rw-r--r--webmentions_ssg/tasks/retention.py11
-rw-r--r--webmentions_ssg/tasks/scanner.py414
-rw-r--r--webmentions_ssg/tasks/sender.py405
-rw-r--r--webmentions_ssg/tasks/source_parser.py390
11 files changed, 1001 insertions, 1029 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index fb1d46d..bf7189f 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,7 +21,7 @@ def create_consumer() -> Huey:
"""
app = create_app()
- from . import notifications, receiver, retention, scanner, sender # noqa: F401
+ from . import notifications, receiver, retention, scanner
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
new file mode 100644
index 0000000..98f3fd5
--- /dev/null
+++ b/webmentions_ssg/tasks/discovery.py
@@ -0,0 +1,231 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Discover a target Webmention endpoint and submit a POST."""
+
+import re
+from urllib.parse import urljoin
+
+import httpx
+from bs4 import BeautifulSoup
+from flask import current_app
+
+from ..net import is_http_url, is_temporary_status, read_limited_body
+
+LINK_SPLIT = re.compile(r",\s*(?=<)")
+DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
+
+
+class SenderError(Exception):
+ """A Webmention cannot be sent."""
+
+
+class TemporarySenderError(SenderError):
+ """A Webmention send failed for a potentially temporary reason."""
+
+
+class PermanentSenderError(SenderError):
+ """A Webmention send failed permanently for this source revision."""
+
+
+def resolve_endpoint(response: httpx.Response, href: str) -> str:
+ """
+ Resolve and validate a discovered Webmention endpoint.
+
+ :param response: Response from which the endpoint was discovered.
+ :param href: Endpoint reference to resolve.
+ :return: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
+ HTTPS URL.
+ """
+ try:
+ endpoint = urljoin(str(response.url), href.strip())
+ except ValueError as exc:
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
+
+ if not is_http_url(endpoint):
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
+
+ return endpoint
+
+
+def parse_link_value(value: str) -> tuple[str, set[str]] | None:
+ """
+ Parse a Link header value into its target and relations.
+
+ :param value: Link header value to parse.
+ :return: Link target and relation names, or ``None`` if the value is invalid.
+ """
+ value = value.strip()
+
+ if not value.startswith("<"):
+ return None
+
+ href, separator, parameters = value[1:].partition(">")
+
+ if not separator:
+ return None
+
+ relations: set[str] = set()
+
+ for parameter in parameters.split(";"):
+ name, separator, value = parameter.partition("=")
+
+ if separator and name.strip().lower() == "rel":
+ relations.update(
+ relation.lower() for relation in value.strip().strip("\"'").split()
+ )
+
+ return href, relations
+
+
+def endpoint_from_headers(response: httpx.Response) -> str | None:
+ """
+ Find a Webmention endpoint in the response Link headers.
+
+ :param response: HTTP response whose headers should be inspected.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ for header in response.headers.get_list("Link"):
+ for value in LINK_SPLIT.split(header):
+ if (link := parse_link_value(value)) is None:
+ continue
+
+ href, relations = link
+
+ if "webmention" in relations:
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
+ """
+ Find a Webmention endpoint in an HTML document.
+
+ ``link`` and ``a`` elements are inspected in document order.
+
+ :param response: HTTP response from which the document was retrieved.
+ :param body: HTML document body.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ document = BeautifulSoup(body, "html5lib")
+
+ for element in document.find_all(["link", "a"], href=True):
+ relations = element.get("rel")
+
+ if isinstance(relations, str):
+ relations = relations.split()
+
+ if not relations:
+ continue
+
+ if not any(
+ isinstance(relation, str) and relation.lower() == "webmention"
+ for relation in relations
+ ):
+ continue
+
+ href = element.get("href")
+
+ if not isinstance(href, str):
+ continue
+
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def read_target_body(response: httpx.Response) -> bytes:
+ """Return a bounded target body, using the shared HTTP size guard."""
+ try:
+ return read_limited_body(
+ response, current_app.config["MAX_TARGET_BYTES"], label="Target"
+ )
+ except ValueError as exc:
+ raise PermanentSenderError(str(exc)) from exc
+
+
+def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
+ """
+ Discover the Webmention endpoint advertised by a target URL.
+
+ Endpoint discovery first checks the response to a HEAD request and then falls
+ back to a GET request, inspecting both HTTP Link headers and supported HTML
+ documents.
+
+ :param client: HTTP client to use for discovery requests.
+ :param target: Target URL whose Webmention endpoint should be discovered.
+ :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :raises TemporarySenderError: If the target returns a temporary HTTP failure.
+ :raises PermanentSenderError: If GET returns a permanent HTTP failure.
+ """
+ try:
+ head_response = client.head(target, headers=DISCOVERY_HEADERS)
+ except httpx.RequestError, TemporarySenderError:
+ # HEAD is only an optimization. Servers and intermediaries may reject
+ # or drop it even when fetching the document with GET would succeed.
+ pass
+ else:
+ if endpoint := endpoint_from_headers(head_response):
+ return endpoint
+
+ with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
+ if endpoint := endpoint_from_headers(response):
+ return endpoint
+
+ if is_temporary_status(response.status_code):
+ raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
+
+ if not response.is_success:
+ raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
+
+ media_type = (
+ response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
+ )
+
+ if media_type not in {"text/html", "application/xhtml+xml"}:
+ return None
+
+ return endpoint_from_html(response, read_target_body(response))
+
+
+def post_webmention(
+ client: httpx.Client, *, endpoint: str, source: str, target: str
+) -> tuple[int, str | None]:
+ """
+ Send a Webmention to a discovered endpoint.
+
+ :param client: HTTP client to use for the request.
+ :param endpoint: Webmention endpoint URL.
+ :param source: Source URL of the Webmention.
+ :param target: Target URL of the Webmention.
+ :return: HTTP response status code and optional status URL.
+ """
+ with client.stream(
+ "POST", endpoint, data={"source": source, "target": target}
+ ) as response:
+ status_url = None
+
+ if response.status_code == 201 and (
+ location := response.headers.get("Location")
+ ):
+ try:
+ candidate = urljoin(str(response.url), location)
+ except ValueError:
+ # A malformed Location must not fail an otherwise successful send.
+ pass
+ else:
+ if is_http_url(candidate):
+ status_url = candidate
+
+ return (response.status_code, status_url)
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index df6cb21..9f53f3b 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -93,54 +93,37 @@ class Huey:
)
return self._huey
+ def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]:
+ """Return a task wrapper that executes with the configured Flask app."""
+
+ @wraps(func)
+ def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
+ if self.app is None:
+ raise RuntimeError("Flask app is not available.")
+ with self.app.app_context():
+ return func(*args, **kwargs)
+
+ return wrapped
+
def task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's task decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's task decorator.
- :return: Decorator that registers the wrapped function as a Huey task.
- """
+ """Return a Huey task decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func))
return decorator
def periodic_task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a periodic Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's periodic task
- decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's periodic task
- decorator.
- :return: Decorator that registers the wrapped function as a periodic Huey
- task.
- """
+ """Return a periodic Huey decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.periodic_task(*task_args, **task_kwargs)(
+ self._in_app_context(func)
+ )
return decorator
diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py
index 473a544..7e51358 100644
--- a/webmentions_ssg/tasks/json_serializer.py
+++ b/webmentions_ssg/tasks/json_serializer.py
@@ -11,7 +11,7 @@ from typing import Any
from huey.registry import Message
from huey.serializer import Serializer
-from huey.utils import Error, SKIPPED
+from huey.utils import SKIPPED, Error
class JSONSerializer(Serializer):
@@ -39,66 +39,87 @@ class JSONSerializer(Serializer):
@classmethod
def _encode(cls, value: Any) -> Any:
- if value is None or type(value) in {str, int, float, bool}:
- return value
- if isinstance(value, uuid.UUID):
- return {"type": "uuid", "value": str(value)}
- if isinstance(value, datetime):
- return {"type": "datetime", "value": value.isoformat()}
- if isinstance(value, Message):
- return {"type": "message", "value": [cls._encode(v) for v in value]}
- if isinstance(value, Error):
- return {"type": "error", "value": cls._encode(value.metadata)}
- if value is SKIPPED:
- return {"type": "skipped", "value": None}
- if isinstance(value, tuple):
- return {"type": "tuple", "value": [cls._encode(v) for v in value]}
- if isinstance(value, list):
- return {"type": "list", "value": [cls._encode(v) for v in value]}
- if isinstance(value, dict) and all(type(k) is str for k in value):
- return {
- "type": "dict",
- "value": [[k, cls._encode(v)] for k, v in value.items()],
- }
- raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
+ match value:
+ case _ if value is None or type(value) in {str, int, float, bool}:
+ return value
+
+ case uuid.UUID():
+ return {"type": "uuid", "value": str(value)}
+
+ case datetime():
+ return {"type": "datetime", "value": value.isoformat()}
+
+ case Message():
+ return {"type": "message", "value": [cls._encode(v) for v in value]}
+
+ case Error():
+ return {"type": "error", "value": cls._encode(value.metadata)}
+
+ case _ if value is SKIPPED:
+ return {"type": "skipped", "value": None}
+
+ case tuple():
+ return {"type": "tuple", "value": [cls._encode(v) for v in value]}
+
+ case list():
+ return {"type": "list", "value": [cls._encode(v) for v in value]}
+
+ case dict() if all(isinstance(k, str) for k in value):
+ return {
+ "type": "dict",
+ "value": [[k, cls._encode(v)] for k, v in value.items()],
+ }
+
+ case _:
+ raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
@classmethod
def _decode(cls, value: Any) -> Any:
if value is None or type(value) in {str, int, float, bool}:
return value
+
if not isinstance(value, dict) or set(value) != {"type", "value"}:
raise ValueError("Invalid Huey JSON data")
- kind, payload = value["type"], value["value"]
- if kind == "uuid" and isinstance(payload, str):
- return uuid.UUID(payload)
- if kind == "datetime" and isinstance(payload, str):
- return datetime.fromisoformat(payload)
- if kind == "message" and isinstance(payload, list):
- if len(payload) != len(Message._fields):
- raise ValueError("Invalid Huey JSON message field count")
- return Message(*(cls._decode(v) for v in payload))
- if kind == "error":
- metadata = cls._decode(payload)
- if not isinstance(metadata, dict):
- raise ValueError("Invalid Huey JSON error metadata")
- return Error(metadata)
- if kind == "skipped" and payload is None:
- return SKIPPED
- if kind in {"tuple", "list"} and isinstance(payload, list):
- items = [cls._decode(v) for v in payload]
- return tuple(items) if kind == "tuple" else items
- if kind == "dict" and isinstance(payload, list):
- result: dict[str, Any] = {}
- for pair in payload:
- if (
- not isinstance(pair, list)
- or len(pair) != 2
- or type(pair[0]) is not str
- ):
- raise ValueError("Invalid Huey JSON dictionary entry")
- if pair[0] in result:
- raise ValueError("Duplicate Huey JSON dictionary key")
- result[pair[0]] = cls._decode(pair[1])
- return result
- raise ValueError("Unsupported Huey JSON type")
+ match value["type"], value["value"]:
+ case "uuid", str() as payload:
+ return uuid.UUID(payload)
+
+ case "datetime", str() as payload:
+ return datetime.fromisoformat(payload)
+
+ case "message", list() as payload:
+ if len(payload) != len(Message._fields):
+ raise ValueError("Invalid Huey JSON message field count")
+ return Message(*(cls._decode(v) for v in payload))
+
+ case "error", payload:
+ metadata = cls._decode(payload)
+ if not isinstance(metadata, dict):
+ raise TypeError("Invalid Huey JSON error metadata")
+ return Error(metadata)
+
+ case "skipped", None:
+ return SKIPPED
+
+ case ("tuple" | "list") as kind, list() as payload:
+ items = [cls._decode(v) for v in payload]
+ return tuple(items) if kind == "tuple" else items
+
+ case "dict", list() as payload:
+ result: dict[str, Any] = {}
+
+ for pair in payload:
+ match pair:
+ case list([str() as key, encoded]):
+ if key in result:
+ raise ValueError("Duplicate Huey JSON dictionary key")
+ result[key] = cls._decode(encoded)
+
+ case _:
+ raise ValueError("Invalid Huey JSON dictionary entry")
+
+ return result
+
+ case _:
+ raise ValueError("Unsupported Huey JSON type")
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py
new file mode 100644
index 0000000..4e5a5ca
--- /dev/null
+++ b/webmentions_ssg/tasks/mentions.py
@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Pure HTML and text matching for received Webmention targets."""
+
+from urllib.parse import urljoin
+
+import rfc3987
+from bs4 import BeautifulSoup
+
+IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
+TRAILING_IRI_PUNCTUATION = ".,;:!?"
+
+HTML_URL_ATTRIBUTES = {
+ "href": {"a", "area", "link"},
+ "src": {
+ "audio",
+ "embed",
+ "iframe",
+ "img",
+ 'input[type="image" i]',
+ "script",
+ "audio source",
+ "video source",
+ "track",
+ "video",
+ },
+ "cite": {"blockquote", "del", "ins", "q"},
+}
+
+
+def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
+ """
+ Check whether an HTML document mentions the target URL.
+
+ URL references are resolved against the document's ``base`` element when
+ present, or against the source URL otherwise.
+
+ :param body: HTML document body.
+ :param source_url: URL from which the document was retrieved.
+ :param target_url: Exact URL to look for.
+ :return: Whether a supported HTML URL attribute references the target URL.
+ """
+
+ document = BeautifulSoup(body, "html.parser")
+ base_url = source_url
+
+ if (base_element := document.select_one("base[href]")) is not None and isinstance(
+ base_href := base_element.get("href"), str
+ ):
+ base_url = urljoin(source_url, base_href.strip())
+
+ for attribute, selectors in HTML_URL_ATTRIBUTES.items():
+ selector = ", ".join(
+ [f"{selector_string}[{attribute}]" for selector_string in selectors]
+ )
+
+ for element in document.select(selector):
+ if not isinstance(reference := element.get(attribute), str):
+ continue
+
+ if urljoin(base_url, reference.strip()) == target_url:
+ return True
+
+ return False
+
+
+def text_mentions_target(body: str, target_url: str) -> bool:
+ """
+ Check whether plain text contains the target IRI, ignoring trailing prose
+ punctuation but not extra URL path or query characters.
+
+ :param body: Plain-text document body.
+ :param target_url: Exact IRI to look for.
+ :return: Whether the target IRI occurs in the document.
+ """
+ for match in IRI_PATTERN.finditer(body):
+ candidate = match.group()
+
+ # A URL in prose can be followed by punctuation that the IRI pattern
+ # consumes as part of its match. Only remove trailing punctuation,
+ # leaving longer URLs (e.g. ``target/more``) unmatched.
+ if not candidate.startswith(target_url):
+ continue
+
+ while candidate != target_url:
+ if candidate[-1] in TRAILING_IRI_PUNCTUATION:
+ candidate = candidate[:-1]
+ elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
+ # Keep balanced parentheses that belong to the URL itself.
+ candidate = candidate[:-1]
+ else:
+ break
+
+ if candidate == target_url:
+ return True
+
+ return False
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index 475fad1..889030c 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -10,10 +10,10 @@ import sqlalchemy as sa
from flask import current_app
from flask_mail import Message
-from .. import DATABASE as db
-from .. import HUEY as huey
-from .. import MAIL as mail
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..extensions import MAIL as mail
+from ..models import ReceivedStatus, ReceivedWebmention
@huey.lock_task("send-webmention-digest")
@@ -27,7 +27,7 @@ def send_webmention_digest() -> None:
cutoff = datetime.now(UTC)
conditions = (
- ReceivedWebmention.status == "verified",
+ ReceivedWebmention.status == ReceivedStatus.VERIFIED,
ReceivedWebmention.notification_sent_at.is_(None),
ReceivedWebmention.updated_at <= cutoff,
)
@@ -53,7 +53,6 @@ def send_webmention_digest() -> None:
),
)
- # If SMTP fails, the records remain pending for the next digest.
mail.send(message)
db.session.execute(
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index acd66a1..bed923c 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -2,27 +2,27 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Fetch and verify incoming Webmentions against their source pages."""
+"""Fetch and verify incoming Webmentions with transactional state changes."""
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
-import rfc3987
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..models import ReceivedWebmention
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError
-
-
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..models import ReceivedStatus, ReceivedWebmention
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_temporary_status,
+ read_limited_body,
+)
+from .mentions import html_mentions_target, text_mentions_target
class VerificationError(Exception):
@@ -37,95 +37,7 @@ class TemporaryFetchError(Exception):
"""Fetching the source may succeed when retried later."""
-IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
-TRAILING_IRI_PUNCTUATION = ".,;:!?"
-
-
-HTML_URL_ATTRIBUTES = {
- "href": {"a", "area", "link"},
- "src": {
- "audio",
- "embed",
- "iframe",
- "img",
- 'input[type="image" i]',
- "script",
- "audio source",
- "video source",
- "track",
- "video",
- },
- "cite": {"blockquote", "del", "ins", "q"},
-}
-
-
-def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """
- Check whether an HTML document mentions the target URL.
-
- URL references are resolved against the document's ``base`` element when
- present, or against the source URL otherwise.
-
- :param body: HTML document body.
- :param source_url: URL from which the document was retrieved.
- :param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
- """
-
- document = BeautifulSoup(body, "html.parser")
- base_url = source_url
-
- if (base_element := document.select_one("base[href]")) is not None and isinstance(
- base_href := base_element.get("href"), str
- ):
- base_url = urljoin(source_url, base_href.strip())
-
- for attribute, selectors in HTML_URL_ATTRIBUTES.items():
- selector = ", ".join(
- [f"{selector_string}[{attribute}]" for selector_string in selectors]
- )
-
- for element in document.select(selector):
- if not isinstance(reference := element.get(attribute), str):
- continue
-
- if urljoin(base_url, reference.strip()) == target_url:
- return True
-
- return False
-
-
-def text_mentions_target(body: str, target_url: str) -> bool:
- """
- Check whether plain text contains the target IRI, ignoring trailing prose
- punctuation but not extra URL path or query characters.
-
- :param body: Plain-text document body.
- :param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
- """
- for match in IRI_PATTERN.finditer(body):
- candidate = match.group()
-
- # A URL in prose can be followed by punctuation that the IRI pattern
- # consumes as part of its match. Only remove trailing punctuation,
- # leaving longer URLs (e.g. ``target/more``) unmatched.
- if not candidate.startswith(target_url):
- continue
-
- while candidate != target_url:
- if candidate[-1] in TRAILING_IRI_PUNCTUATION:
- candidate = candidate[:-1]
- elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
- # Keep balanced parentheses that belong to the URL itself.
- candidate = candidate[:-1]
- else:
- break
-
- if candidate == target_url:
- return True
-
- return False
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
@@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
try:
- return _fetch_source(source_url)
+ with (
+ build_client(
+ f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if is_temporary_status(status):
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
+
+ try:
+ return response, read_limited_body(
+ response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
+ )
+ except ValueError as exc:
+ raise VerificationError(str(exc)) from exc
+
except UnsafeAddressError as exc:
raise VerificationError("Source resolves to a non-public address") from exc
except AddressResolutionError as exc:
raise TemporaryFetchError("Source hostname could not be resolved") from exc
-def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- with (
- httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client,
- client.stream("GET", source_url) as response,
- ):
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
-
- max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
-
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_source_bytes:
- raise VerificationError("Source document is too large")
-
- return response, bytes(body)
-
-
def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
Fetch a source and check whether it mentions the target URL.
@@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- # Failed tasks may be retried by Huey. Completed and already-running tasks
- # should not start another concurrent verification.
- if webmention.status not in {"received", "failed"}:
+ if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
return
- # A reaper-exhausted row must not be revived by a late Huey retry.
+
if (
- webmention.status == "failed"
+ webmention.status == ReceivedStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return
@@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
ReceivedWebmention.status == previous_status,
ReceivedWebmention.updated_at == previous_updated_at,
)
- .values(status="verifying", failure_reason=None, updated_at=started_at)
+ .values(
+ status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at
+ )
)
+
if claimed.rowcount != 1:
db.session.rollback()
return
- db.session.commit()
- def finish(status: str, failure_reason: str | None) -> None:
- # A timeout or a newer verification may have superseded this attempt.
- # Do not let a late worker overwrite its state.
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == "verifying",
- ReceivedWebmention.updated_at == started_at,
- )
- .values(
- status=status,
- failure_reason=failure_reason,
- updated_at=datetime.now(UTC),
- )
- )
- db.session.commit()
+ db.session.commit()
+ status = None
+ failure_reason = None
try:
mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- finish("deleted", str(exc))
+ status = ReceivedStatus.DELETED
+ failure_reason = str(exc)
except VerificationError as exc:
- finish("failed", str(exc))
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc)
except (TemporaryFetchError, httpx.RequestError) as exc:
- finish("failed", str(exc) or "Source could not be fetched")
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or "Source could not be fetched"
raise
except Exception as exc:
# InvalidURL is not a RequestError; other unexpected source errors must
@@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", webmention_uuid
)
- finish("failed", str(exc) or type(exc).__name__)
+
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or type(exc).__name__
else:
if mentions_target:
- finish("verified", None)
+ status = ReceivedStatus.VERIFIED
else:
- finish("deleted", "Source does not mention target")
+ status = ReceivedStatus.DELETED
+ failure_reason = "Source does not mention target"
+ finally:
+ if status is not None:
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
def recover_stale_webmentions() -> int:
@@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int:
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+
stale = db.session.execute(
sa.select(
ReceivedWebmention.uuid,
@@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.updated_at,
ReceivedWebmention.recovery_attempts,
).where(
- ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ ),
ReceivedWebmention.updated_at <= cutoff,
)
).all()
@@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int:
recovered = 0
exhausted = 0
max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+
for identifier, status, updated_at, recovery_attempts in stale:
at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
@@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.recovery_attempts == recovery_attempts,
)
.values(
- status="failed" if at_limit else "received",
- failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
- updated_at=now,
- recovery_attempts=(
- recovery_attempts if at_limit else recovery_attempts + 1
- ),
+ **(
+ {
+ "status": ReceivedStatus.FAILED,
+ "failure_reason": RECOVERY_EXHAUSTED_REASON,
+ "updated_at": now,
+ "recovery_attempts": recovery_attempts,
+ }
+ if at_limit
+ else ReceivedWebmention.verification_reset_values(
+ at=now, recoveries=recovery_attempts + 1
+ )
+ )
)
)
+
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+
if at_limit:
exhausted += 1
continue
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 727b186..bf79512 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -11,8 +11,8 @@ import sqlalchemy as sa
from flask import current_app
from sqlalchemy.engine import CursorResult
-from .. import DATABASE as db
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..models import ReceivedStatus, ReceivedWebmention
def purge_old_received_webmentions() -> int:
@@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int:
cutoff = datetime.now(UTC) - timedelta(
days=current_app.config["RECEIVED_RETENTION_DAYS"]
)
+
result = cast(
CursorResult[Any],
db.session.execute(
sa.delete(ReceivedWebmention).where(
- ReceivedWebmention.status.in_(("failed", "deleted")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ ),
ReceivedWebmention.updated_at < cutoff,
)
),
)
db.session.commit()
+
removed = result.rowcount
if removed:
current_app.logger.info("Purged %d old failed/deleted Webmentions", removed)
+
return removed
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 666eba7..b2d6def 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -2,404 +2,22 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Scan generated site pages to track outgoing Webmention targets."""
+"""Reconcile scanned static-site sources with the database."""
-import json
-from collections.abc import Iterator
-from dataclasses import dataclass
from datetime import UTC, datetime
-from fnmatch import fnmatchcase
-from itertools import chain
from pathlib import Path
-from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
-import mf2py
import sqlalchemy as sa
-import xxhash
-from bs4 import BeautifulSoup, NavigableString, Tag
-from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..config import normalize_ignored_hostnames
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, Source
-from ..url_security import is_http_url
+from ..net import is_http_url
from .sender import send_webmention
-
-REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-
-CONTENT_BLOCK_TAGS = frozenset(
- {
- "address",
- "article",
- "aside",
- "blockquote",
- "br",
- "dd",
- "div",
- "dl",
- "dt",
- "figcaption",
- "figure",
- "footer",
- "h1",
- "h2",
- "h3",
- "h4",
- "h5",
- "h6",
- "header",
- "hr",
- "li",
- "main",
- "ol",
- "p",
- "pre",
- "section",
- "table",
- "td",
- "th",
- "tr",
- "ul",
- }
-)
-
-
-class SourceScanError(Exception):
- """A source document cannot safely be processed."""
-
-
-@dataclass(frozen=True)
-class ScannedSource:
- path: str
- url: str
- content_hash: str
- targets: frozenset[str]
-
-
-def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """
- Derive the public source URL from a relative filesystem path.
-
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
- """
- directory = relative_path.parent.as_posix()
-
- return urljoin(base_url, f"{quote(directory, safe='/')}/")
-
-
-def canonical_url(
- document: BeautifulSoup, *, relative_path: Path, base_url: str | None
-) -> str | None:
- """
- Return the canonical URL declared by a source document.
-
- Relative canonical URLs are resolved against the configured source base URL.
-
- :param document: Parsed HTML source document.
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
- """
- if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
- href = link.get("href")
-
- if not isinstance(href, str) or not (href := href.strip()):
- raise SourceScanError("Document contains an empty canonical URL")
-
- if is_http_url(href):
- return href
-
- if base_url is None:
- raise SourceScanError(
- "Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
- )
-
- resolved = urljoin(source_url_for_path(relative_path, base_url), href)
-
- if not is_http_url(resolved):
- raise SourceScanError(
- f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
- )
-
- return resolved
-
-
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """
- Yield URL values from a microformats property.
-
- :param entry: Parsed microformats entry.
- :param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
- """
- properties = entry.get("properties")
-
- if not isinstance(properties, dict):
- return
-
- values = properties.get(property_name)
-
- if not isinstance(values, list):
- return
-
- yield from (value for value in values if isinstance(value, str))
-
-
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """
- Parse and validate an h-entry with mf2py.
-
- :param element: HTML element containing the h-entry.
- :param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
- :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
- """
- parsed = mf2py.parse(doc=str(element), url=base_url)
-
- if not isinstance(parsed, dict):
- raise SourceScanError("Microformats parser did not return a document object")
-
- items = parsed.get("items")
-
- if not isinstance(items, list) or len(items) != 1:
- raise SourceScanError("Expected exactly one parsed h-entry")
-
- entry = items[0]
-
- if not isinstance(entry, dict):
- raise SourceScanError("Parsed h-entry is not an object")
-
- types = entry.get("type")
-
- if not isinstance(types, list) or "h-entry" not in types:
- raise SourceScanError("Parsed microformats item is not an h-entry")
-
- return entry
-
-
-def primary_entry(
- document: BeautifulSoup, source_url: str
-) -> tuple[Tag, dict[str, Any]]:
- """
- Return and validate the primary h-entry of a source document.
-
- The document must contain exactly one h-entry whose ``u-url`` matches the
- source URL.
-
- :param document: Parsed HTML source document.
- :param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
- """
- entries = document.find_all(class_="h-entry")
-
- if len(entries) != 1:
- raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
-
- entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
-
- urls = tuple(property_urls(mf2_entry, "url"))
-
- if source_url not in urls:
- raise SourceScanError(
- f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
- )
-
- return entry_element, mf2_entry
-
-
-def content_element(entry: Tag) -> Tag:
- """
- Return the e-content element of an h-entry.
-
- :param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
- """
- contents = entry.find_all(class_="e-content")
-
- if len(contents) != 1:
- raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
-
- return contents[0]
-
-
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """
- Resolve and validate a possible Webmention target URL.
-
- Empty values, non-HTTP URLs, and URLs referring to the source itself are
- discarded.
-
- :param value: URL reference to normalize.
- :param base_url: Base URL against which relative references are resolved.
- :param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
- """
- value = value.strip()
-
- if not value:
- return None
-
- target = urljoin(base_url, value)
-
- if not is_http_url(target):
- return None
-
- if urldefrag(target)[0] == urldefrag(source_url)[0]:
- return None
-
- return target
-
-
-def iter_targets(
- content: Tag,
- mf2_entry: dict[str, Any],
- *,
- base_url: str,
- source_url: str,
- ignored_hostnames: tuple[str, ...] = (),
-) -> Iterator[str]:
- """
- Yield outgoing Webmention targets from an h-entry.
-
- Targets are collected from links in the entry content and supported
- microformats reaction properties. Invalid, self-referencing, and ignored
- targets are excluded.
-
- :param content: e-content element of the h-entry.
- :param mf2_entry: Parsed microformats representation of the h-entry.
- :param base_url: Base URL against which relative targets are resolved.
- :param source_url: URL of the source document.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
- """
- hrefs = (
- href
- for element in content.find_all("a", href=True)
- if isinstance(href := element.get("href"), str)
- )
-
- reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
- )
-
- for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
- if target is not None:
- hostname = urlsplit(target).hostname or ""
- if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
- yield target
-
-
-def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
-
- def extract(node: PageElement) -> str:
- if isinstance(node, NavigableString):
- return str(node)
- if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
- return ""
-
- # A relative-time label may change on every site build even though its
- # machine-readable date stays the same.
- if node.name == "time" and isinstance(value := node.get("datetime"), str):
- return value.strip()
-
- # Image-only posts also have meaningful content, despite having no text.
- if node.name == "img":
- return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
-
- text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
-
- return " ".join(extract(content).split())
-
-
-def scan_source_file(
- path: Path,
- *,
- root: Path,
- base_url: str | None,
- ignored_hostnames: tuple[str, ...] = (),
-) -> ScannedSource:
- """
- Scan one generated source document for outgoing Webmentions.
-
- :param path: Path of the generated HTML document.
- :param root: Root directory containing generated source documents.
- :param base_url: Configured source base URL, if available.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
- :raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
- :raises OSError: If the source document cannot be read.
- """
- # Use the path as published for its URL, but read only files whose
- # resolved target stays inside the configured source directory. Resolving
- # before reading also protects against symlinked index.html files.
- resolved_root = root.resolve()
- resolved_path = path.resolve()
- if not resolved_path.is_relative_to(resolved_root):
- raise SourceScanError(f"Source document escapes source directory: {path}")
-
- relative_path = path.relative_to(root)
- document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
-
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
-
- if source_url is None:
- if base_url is None:
- raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
- )
-
- source_url = source_url_for_path(relative_path, base_url)
-
- entry_element, mf2_entry = primary_entry(document, source_url)
-
- content = content_element(entry_element)
-
- targets = frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- )
-
- # Fingerprint the meaningful post text and its outgoing targets rather than
- # the full h-entry markup. Relative timestamps, classes, and other volatile
- # metadata outside e-content must not cause repeat Webmentions.
- stable_content = normalized_content(content)
- fingerprint = json.dumps(
- {"content": stable_content, "targets": sorted(targets)},
- ensure_ascii=False,
- separators=(",", ":"),
- )
-
- return ScannedSource(
- path=relative_path.as_posix(),
- url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
- targets=targets,
- )
+from .source_parser import ScannedSource, SourceScanError, scan_source_file
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
@@ -466,10 +84,9 @@ def update_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.active = webmention.target in scanned.targets
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(
+ source.revision, webmention.target in scanned.targets
+ )
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -523,10 +140,6 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = normalize_ignored_hostnames(
- current_app.config["IGNORED_HOSTNAMES"]
- )
-
sources_by_path = {
source.path: source
for source in db.session.scalars(
@@ -555,7 +168,7 @@ def scan_sources_task() -> None:
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=ignored_hostnames,
+ ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -592,10 +205,6 @@ def scan_sources_task() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- # A static-site generator can temporarily empty its output directory while
- # rebuilding. With no successfully scanned pages, absence is not evidence
- # of deletion; do not generate removals or send pending Webmentions that
- # might be verified while the site is unavailable.
if scanned_count == 0:
current_app.logger.warning(
"No source documents successfully scanned; "
@@ -612,10 +221,7 @@ def scan_sources_task() -> None:
source.deleted_at = scan_time
for webmention in source.sent_webmentions:
- webmention.active = False
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 9330247..3c4c717 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -2,272 +2,33 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Discover endpoints and deliver outgoing Webmentions."""
+"""Claim, send, and persist outgoing Webmention attempts."""
-import re
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from huey.exceptions import TaskLockedException
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
-
-LINK_SPLIT = re.compile(r",\s*(?=<)")
-
-DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
-
-
-class SenderError(Exception):
- """A Webmention cannot be sent."""
-
-
-class TemporarySenderError(SenderError):
- """A Webmention send failed for a potentially temporary reason."""
-
-
-class PermanentSenderError(SenderError):
- """A Webmention send failed permanently for this source revision."""
-
-
-def temporary_http_status(status: int) -> bool:
- """
- Return whether an HTTP status indicates a temporary failure.
-
- :param status: HTTP response status code.
- :return: Whether the request should be retried.
- """
- return status in {408, 425, 429} or 500 <= status <= 599
-
-
-def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """
- Resolve and validate a discovered Webmention endpoint.
-
- :param response: Response from which the endpoint was discovered.
- :param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
- """
- try:
- endpoint = urljoin(str(response.url), href.strip())
- except ValueError as exc:
- raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
-
- if not is_http_url(endpoint):
- raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
-
- return endpoint
-
-
-def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """
- Parse a Link header value into its target and relations.
-
- :param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
- """
- value = value.strip()
-
- if not value.startswith("<"):
- return None
-
- href, separator, parameters = value[1:].partition(">")
-
- if not separator:
- return None
-
- relations: set[str] = set()
-
- for parameter in parameters.split(";"):
- name, separator, value = parameter.partition("=")
-
- if separator and name.strip().lower() == "rel":
- relations.update(
- relation.lower() for relation in value.strip().strip("\"'").split()
- )
-
- return href, relations
-
-
-def endpoint_from_headers(response: httpx.Response) -> str | None:
- """
- Find a Webmention endpoint in the response Link headers.
-
- :param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- for header in response.headers.get_list("Link"):
- for value in LINK_SPLIT.split(header):
- if (link := parse_link_value(value)) is None:
- continue
-
- href, relations = link
-
- if "webmention" in relations:
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """
- Find a Webmention endpoint in an HTML document.
-
- ``link`` and ``a`` elements are inspected in document order.
-
- :param response: HTTP response from which the document was retrieved.
- :param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- document = BeautifulSoup(body, "html5lib")
-
- for element in document.find_all(["link", "a"], href=True):
- relations = element.get("rel")
-
- if isinstance(relations, str):
- relations = relations.split()
-
- if not relations:
- continue
-
- if not any(
- isinstance(relation, str) and relation.lower() == "webmention"
- for relation in relations
- ):
- continue
-
- href = element.get("href")
-
- if not isinstance(href, str):
- continue
-
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def read_target_body(response: httpx.Response) -> bytes:
- """
- Read a target document up to the configured size limit.
-
- :param response: Streaming HTTP response to read.
- :return: Response body.
- :raises PermanentSenderError: If the document exceeds the configured maximum
- size.
- """
- max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_bytes:
- raise PermanentSenderError("Target document is too large")
- except ValueError:
- pass
-
- body = bytearray()
-
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_bytes:
- raise PermanentSenderError("Target document is too large")
-
- return bytes(body)
-
-
-def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """
- Discover the Webmention endpoint advertised by a target URL.
-
- Endpoint discovery first checks the response to a HEAD request and then falls
- back to a GET request, inspecting both HTTP Link headers and supported HTML
- documents.
-
- :param client: HTTP client to use for discovery requests.
- :param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
- :raises TemporarySenderError: If the target returns a temporary HTTP failure.
- :raises PermanentSenderError: If GET returns a permanent HTTP failure.
- """
- try:
- head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
- # HEAD is only an optimization. Servers and intermediaries may reject
- # or drop it even when fetching the document with GET would succeed.
- pass
- else:
- if endpoint := endpoint_from_headers(head_response):
- return endpoint
-
- with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
- if endpoint := endpoint_from_headers(response):
- return endpoint
-
- if temporary_http_status(response.status_code):
- raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
-
- if not response.is_success:
- raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
-
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
-
- if media_type not in {"text/html", "application/xhtml+xml"}:
- return None
-
- return endpoint_from_html(response, read_target_body(response))
-
-
-def post_webmention(
- client: httpx.Client, *, endpoint: str, source: str, target: str
-) -> tuple[int, str | None]:
- """
- Send a Webmention to a discovered endpoint.
-
- :param client: HTTP client to use for the request.
- :param endpoint: Webmention endpoint URL.
- :param source: Source URL of the Webmention.
- :param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
- """
- with client.stream(
- "POST", endpoint, data={"source": source, "target": target}
- ) as response:
- status_url = None
-
- if response.status_code == 201 and (
- location := response.headers.get("Location")
- ):
- try:
- candidate = urljoin(str(response.url), location)
- except ValueError:
- # A malformed Location must not fail an otherwise successful send.
- pass
- else:
- if is_http_url(candidate):
- status_url = candidate
-
- return (response.status_code, status_url)
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_http_url,
+ is_temporary_status,
+)
+from .discovery import (
+ PermanentSenderError,
+ TemporarySenderError,
+ discover_webmention_endpoint,
+ post_webmention,
+)
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
now = datetime.now(UTC)
deadline = webmention.next_retry_at
- # SQLite returns timezone-naive datetimes even for timezone-aware columns.
if deadline is not None:
aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
if aware_deadline > now:
@@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
endpoint: str | None = None
response_status: int | None = None
+ status_url: str | None = None
+ reason: str | None = None
+ terminal = True
+ temporary_failure = False
try:
if not is_http_url(target):
raise PermanentSenderError(f"Invalid target URL: {target!r}")
- with httpx.Client(
- headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")},
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client:
+ with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.UNSUPPORTED
- webmention.failure_reason = "No Webmention endpoint discovered"
- webmention.endpoint = None
- webmention.response_status = None
-
- db.session.commit()
- return
-
- (response_status, status_url) = post_webmention(
- client, endpoint=endpoint, source=source, target=target
- )
-
- if not (200 <= response_status <= 299):
- message = f"Webmention endpoint returned HTTP {response_status}"
-
- if temporary_http_status(response_status):
- raise TemporarySenderError(message)
-
- raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.UNSUPPORTED
+ reason = "No Webmention endpoint discovered"
+ else:
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.SENT
except (
PermanentSenderError,
@@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
ValueError,
UnicodeError,
) as exc:
- # Malformed URLs and local request/protocol errors cannot be repaired
- # by retrying this source revision. InvalidURL is not a RequestError;
- # protocol errors are, so handle them before the retryable branch.
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or type(exc).__name__
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
- return
-
+ # Invalid URLs and local protocol errors cannot improve with retries.
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or type(exc).__name__
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or "Webmention request failed"
- if attempt >= max_attempts:
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- current_app.logger.warning(
- "Webmention %s exhausted %d send attempts: %s",
- webmention_uuid,
- attempt,
- webmention.failure_reason,
- )
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
-
- # No Huey retries: only the scanner may requeue after next_retry_at.
- return
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or "Webmention request failed"
+ terminal = attempt >= max_attempts
+ temporary_failure = True
+ # A scanner may have reopened or superseded the revision during I/O.
+ # Refresh only once, and persist one outcome atomically with its metadata.
if not attempt_is_current(webmention, revision):
return
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.sent_revision = revision
-
- webmention.status = SentWebmentionStatus.SENT
- webmention.failure_reason = None
-
+ webmention.status = outcome
+ webmention.failure_reason = reason
webmention.endpoint = endpoint
webmention.response_status = response_status
- webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
- )
- webmention.last_sent_at = datetime.now(UTC)
+ if terminal:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+
+ if outcome == SentWebmentionStatus.SENT:
+ webmention.sent_revision = revision
+ webmention.status_url = (
+ status_url if status_url is not None and is_http_url(status_url) else None
+ )
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+
+ if temporary_failure and not terminal:
+ # Huey does not retry: scanner requeues after next_retry_at.
+ current_app.logger.info(
+ "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ )
+ elif temporary_failure and terminal:
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ reason,
+ )
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
new file mode 100644
index 0000000..cb15ab6
--- /dev/null
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -0,0 +1,390 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Extract canonical source URLs, content fingerprints, and outgoing links."""
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from itertools import chain
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urldefrag, urljoin, urlsplit
+
+import mf2py
+import xxhash
+from bs4 import BeautifulSoup, Tag
+from bs4.element import NavigableString, PageElement
+
+from ..net import is_http_url
+
+REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
+
+class SourceScanError(Exception):
+ """A source document cannot safely be processed."""
+
+
+@dataclass(frozen=True)
+class ScannedSource:
+ path: str
+ url: str
+ content_hash: str
+ targets: frozenset[str]
+
+
+def source_url_for_path(relative_path: Path, base_url: str) -> str:
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
+ directory = relative_path.parent.as_posix()
+
+ return urljoin(base_url, f"{quote(directory, safe='/')}/")
+
+
+def canonical_url(
+ document: BeautifulSoup, *, relative_path: Path, base_url: str | None
+) -> str | None:
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
+ if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
+ href = link.get("href")
+
+ if not isinstance(href, str) or not (href := href.strip()):
+ raise SourceScanError("Document contains an empty canonical URL")
+
+ if is_http_url(href):
+ return href
+
+ if base_url is None:
+ raise SourceScanError(
+ "Document contains a relative canonical URL, but "
+ "SOURCE_BASE_URL is not configured"
+ )
+
+ resolved = urljoin(source_url_for_path(relative_path, base_url), href)
+
+ if not is_http_url(resolved):
+ raise SourceScanError(
+ f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
+ )
+
+ return resolved
+
+
+def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
+ properties = entry.get("properties")
+
+ if not isinstance(properties, dict):
+ return
+
+ values = properties.get(property_name)
+
+ if not isinstance(values, list):
+ return
+
+ yield from (value for value in values if isinstance(value, str))
+
+
+def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
+ parsed = mf2py.parse(doc=str(element), url=base_url)
+
+ if not isinstance(parsed, dict):
+ raise SourceScanError("Microformats parser did not return a document object")
+
+ items = parsed.get("items")
+
+ if not isinstance(items, list) or len(items) != 1:
+ raise SourceScanError("Expected exactly one parsed h-entry")
+
+ entry = items[0]
+
+ if not isinstance(entry, dict):
+ raise SourceScanError("Parsed h-entry is not an object")
+
+ types = entry.get("type")
+
+ if not isinstance(types, list) or "h-entry" not in types:
+ raise SourceScanError("Parsed microformats item is not an h-entry")
+
+ return entry
+
+
+def primary_entry(
+ document: BeautifulSoup, source_url: str
+) -> tuple[Tag, dict[str, Any]]:
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
+ entries = document.find_all(class_="h-entry")
+
+ if len(entries) != 1:
+ raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
+
+ entry_element = entries[0]
+ mf2_entry = parse_entry(entry_element, source_url)
+
+ urls = tuple(property_urls(mf2_entry, "url"))
+
+ if source_url not in urls:
+ raise SourceScanError(
+ f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
+ )
+
+ return entry_element, mf2_entry
+
+
+def content_element(entry: Tag) -> Tag:
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
+ contents = entry.find_all(class_="e-content")
+
+ if len(contents) != 1:
+ raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
+
+ return contents[0]
+
+
+def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
+ value = value.strip()
+
+ if not value:
+ return None
+
+ target = urljoin(base_url, value)
+
+ if not is_http_url(target):
+ return None
+
+ if urldefrag(target)[0] == urldefrag(source_url)[0]:
+ return None
+
+ return target
+
+
+def iter_targets(
+ content: Tag,
+ mf2_entry: dict[str, Any],
+ *,
+ base_url: str,
+ source_url: str,
+ ignored_hostnames: Iterable[str] = (),
+) -> Iterator[str]:
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
+ hrefs = (
+ href
+ for element in content.find_all("a", href=True)
+ if isinstance(href := element.get("href"), str)
+ )
+
+ reactions = chain.from_iterable(
+ property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ )
+
+ for value in chain(hrefs, reactions):
+ target = normalize_target(value, base_url=base_url, source_url=source_url)
+ if target is not None:
+ hostname = (urlsplit(target).hostname or "").rstrip(".")
+ if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
+ yield target
+
+
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
+def scan_source_file(
+ path: Path,
+ *,
+ root: Path,
+ base_url: str | None,
+ ignored_hostnames: Iterable[str] = (),
+) -> ScannedSource:
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
+
+ relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
+
+ source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+
+ if source_url is None:
+ if base_url is None:
+ raise SourceScanError(
+ "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ )
+
+ source_url = source_url_for_path(relative_path, base_url)
+
+ entry_element, mf2_entry = primary_entry(document, source_url)
+
+ content = content_element(entry_element)
+
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+ return ScannedSource(
+ path=relative_path.as_posix(),
+ url=source_url,
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
+ )