diff options
| author | Dennis Fink | 2026-08-19 19:49:55 +0200 |
|---|---|---|
| committer | Dennis Fink | 2026-08-19 19:49:55 +0200 |
| commit | d0e245d63e014d268310976fc45429f08d2cbbe9 (patch) | |
| tree | ef2e96570723190baf02f05c2c4644a19ff1af78 /webmentions_ssg/tasks | |
| parent | d4a03623544aac0ce911d8471ff4b033bc3255e1 (diff) | |
| download | webmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.tar.gz webmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.zip | |
refactor(core): improve typing and test coverage
Restructure database and user CLI commands, make passwords write-only
model properties, and use the UTC datetime constant throughout the
application.
Add PEP 287-style documentation and expand receiver, scanner, sender,
and URL security tests to cover error paths and edge cases.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 155 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 139 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 115 |
4 files changed, 343 insertions, 77 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index e53fce5..3bfe861 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -4,9 +4,18 @@ from .. import HUEY, create_app def create_consumer() -> Huey: + """ + Create and configure the Huey consumer. + + The application is initialized, task modules are imported to register their + tasks, and the source scanner is registered with the configured periodic + schedule. + + :return: Configured Huey instance. + """ app = create_app() - from . import receiver, scanner, sender # noqa: E402, F401 + from . import receiver, scanner, sender # noqa: F401 schedule = crontab( *app.config["WEBMENTIONS_SSG_SCANNER_SCHEDULE"].split(), strict=True diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 39def73..4aba3d8 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -47,7 +47,17 @@ HTML_URL_ATTRIBUTES = { def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """Check valid HTML URL attributes for the exact target URL.""" + """ + Check whether an HTML document mentions the target URL. + + URL references are resolved against the document's ``base`` element when + present, or against the source URL otherwise. + + :param body: HTML document body. + :param source_url: URL from which the document was retrieved. + :param target_url: Exact URL to look for. + :return: Whether a supported HTML URL attribute references the target URL. + """ document = BeautifulSoup(body, "html.parser") base_url = source_url @@ -59,12 +69,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: for attribute, selectors in HTML_URL_ATTRIBUTES.items(): selector = ", ".join( - [ - "{selector}[{attribute}]".format( - selector=selector_string, attribute=attribute - ) - for selector_string in selectors - ] + [f"{selector_string}[{attribute}]" for selector_string in selectors] ) for element in document.select(selector): @@ -78,12 +83,25 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: def text_mentions_target(body: str, target_url: str) -> bool: - """Check whether plain text contains the exact target IRI.""" + """ + Check whether plain text contains the exact target IRI. + + :param body: Plain-text document body. + :param target_url: Exact IRI to look for. + :return: Whether the target IRI occurs in the document. + """ return any(match.group() == target_url for match in IRI_PATTERN.finditer(body)) def ensure_public_request(request: httpx.Request) -> None: - """Prevent requests to non-public network addresses.""" + """ + Ensure that an HTTP request targets a public network address. + + :param request: HTTP request to validate. + :raises VerificationError: If the URL has no hostname or resolves to a + non-public address. + :raises TemporaryFetchError: If the hostname cannot be resolved. + """ try: if not is_public_url(str(request.url)): raise VerificationError("Source resolves to a non-public address") @@ -94,56 +112,83 @@ def ensure_public_request(request: httpx.Request) -> None: def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - """Fetch a source with limits on redirects, time, and response size.""" + """ + Fetch a webmention source document. - with httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout( - current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) - ), - follow_redirects=True, - max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), - trust_env=False, - event_hooks={"request": [ensure_public_request]}, - ) as client: - with client.stream("GET", source_url) as response: - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") + The request follows redirects while enforcing the configured timeout, + redirect limit, and maximum response size. - max_source_bytes = current_app.config.get( - "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000 - ) + :param source_url: URL of the source document. + :return: HTTP response and response body. + :raises SourceGoneError: If the source returns HTTP 410. + :raises TemporaryFetchError: If the source returns a temporary HTTP error. + :raises VerificationError: If the source returns another unsuccessful HTTP + status or exceeds the configured maximum response size. + """ - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass + with ( + httpx.Client( + headers={ + "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", + }, + timeout=httpx.Timeout( + current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) + ), + follow_redirects=True, + max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), + trust_env=False, + event_hooks={"request": [ensure_public_request]}, + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if status in {408, 425, 429} or 500 <= status <= 599: + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) + max_source_bytes = current_app.config.get( + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000 + ) - if len(body) > max_source_bytes: + if (content_length := response.headers.get("Content-Length")) is not None: + try: + if int(content_length) > max_source_bytes: raise VerificationError("Source document is too large") + except ValueError: + pass + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) - return response, bytes(body) + if len(body) > max_source_bytes: + raise VerificationError("Source document is too large") + + return response, bytes(body) def source_mentions_target(source_url: str, target_url: str) -> bool: - """Fetch the source and verify it according to its media type.""" + """ + Fetch a source and check whether it mentions the target URL. + + HTML and XHTML sources are inspected for supported URL attributes, while + plain-text sources are searched for the exact target IRI. + + :param source_url: URL of the source document. + :param target_url: Exact target URL to look for. + :return: Whether the source mentions the target URL. + :raises VerificationError: If the source has an unsupported or missing + content type. + :raises SourceGoneError: If the source explicitly reports that it is gone. + :raises TemporaryFetchError: If fetching the source fails temporarily. + """ response, body = fetch_source(source_url) @@ -170,7 +215,17 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: - """Verify a ReceivedWebmention and store the result.""" + """ + Verify a received webmention and store its verification status. + + Temporary fetch failures are stored before being re-raised so that Huey can + retry the task. + + :param webmention_uuid: Identifier of the received webmention to verify. + :raises TemporaryFetchError: If fetching the source fails temporarily. + :raises httpx.RequestError: If an HTTP request error occurs while fetching + the source. + """ webmention = db.session.get(ReceivedWebmention, webmention_uuid) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index ff75697..fa6a47b 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -36,7 +36,13 @@ class ScannedSource: def source_url_for_path(relative_path: Path, base_url: str) -> str: - """Derive the public source URL from its relative filesystem path.""" + """ + Derive the public source URL from a relative filesystem path. + + :param relative_path: Path of the source document relative to the source root. + :param base_url: Base URL under which source documents are published. + :return: Public URL corresponding to the source document. + """ directory = relative_path.parent.as_posix() return urljoin(base_url, f"{quote(directory, safe='/')}/") @@ -45,7 +51,18 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: def canonical_url( document: BeautifulSoup, *, relative_path: Path, base_url: str | None ) -> str | None: - """Return the canonical URL declared by the document.""" + """ + Return the canonical URL declared by a source document. + + Relative canonical URLs are resolved against the configured source base URL. + + :param document: Parsed HTML source document. + :param relative_path: Path of the source document relative to the source root. + :param base_url: Configured source base URL, if available. + :return: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, + or does not resolve to an HTTP or HTTPS URL. + """ if (link := document.select_one('link[rel~="canonical"][href]')) is not None: href = link.get("href") @@ -72,7 +89,13 @@ def canonical_url( def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """Yield URL values from a microformats property.""" + """ + Yield URL values from a microformats property. + + :param entry: Parsed microformats entry. + :param property_name: Name of the property to inspect. + :return: Iterator over string values of the property. + """ properties = entry.get("properties") if not isinstance(properties, dict): @@ -87,7 +110,14 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """Parse the source h-entry with mf2py.""" + """ + Parse and validate an h-entry with mf2py. + + :param element: HTML element containing the h-entry. + :param base_url: Base URL used when parsing the microformats data. + :return: Parsed h-entry. + :raises SourceScanError: If parsing does not produce exactly one valid h-entry. + """ parsed = mf2py.parse(doc=str(element), url=base_url) if not isinstance(parsed, dict): @@ -114,7 +144,18 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: def primary_entry( document: BeautifulSoup, source_url: str ) -> tuple[Tag, dict[str, Any]]: - """Return and validate the source h-entry.""" + """ + Return and validate the primary h-entry of a source document. + + The document must contain exactly one h-entry whose ``u-url`` matches the + source URL. + + :param document: Parsed HTML source document. + :param source_url: Public URL of the source document. + :return: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry + or its ``u-url`` does not match the source URL. + """ entries = document.find_all(class_="h-entry") if len(entries) != 1: @@ -134,7 +175,14 @@ def primary_entry( def content_element(entry: Tag) -> Tag: - """Return the source h-entry's e-content element.""" + """ + Return the e-content element of an h-entry. + + :param entry: HTML element containing the h-entry. + :return: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one + e-content element. + """ contents = entry.find_all(class_="e-content") if len(contents) != 1: @@ -144,7 +192,18 @@ def content_element(entry: Tag) -> Tag: def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """Resolve and validate a possible Webmention target URL.""" + """ + Resolve and validate a possible Webmention target URL. + + Empty values, non-HTTP URLs, and URLs referring to the source itself are + discarded. + + :param value: URL reference to normalize. + :param base_url: Base URL against which relative references are resolved. + :param source_url: URL of the source document. + :return: Normalized target URL, or ``None`` if the value is not a valid + Webmention target. + """ value = value.strip() if not value: @@ -169,7 +228,20 @@ def iter_targets( source_url: str, ignored_hostnames: tuple[str, ...] = (), ) -> Iterator[str]: - """Yield outgoing Webmention targets from an h-entry.""" + """ + Yield outgoing Webmention targets from an h-entry. + + Targets are collected from links in the entry content and supported + microformats reaction properties. Invalid, self-referencing, and ignored + targets are excluded. + + :param content: e-content element of the h-entry. + :param mf2_entry: Parsed microformats representation of the h-entry. + :param base_url: Base URL against which relative targets are resolved. + :param source_url: URL of the source document. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Iterator over outgoing Webmention target URLs. + """ hrefs = ( href for element in content.find_all("a", href=True) @@ -195,7 +267,18 @@ def scan_source_file( base_url: str | None, ignored_hostnames: tuple[str, ...] = (), ) -> ScannedSource: - """Parse one generated source document.""" + """ + Scan one generated source document for outgoing Webmentions. + + :param path: Path of the generated HTML document. + :param root: Root directory containing generated source documents. + :param base_url: Configured source base URL, if available. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Scanned source metadata and discovered targets. + :raises SourceScanError: If the document cannot be safely interpreted as a + Webmention source. + :raises OSError: If the source document cannot be read. + """ document = BeautifulSoup(path.read_bytes(), "html5lib") relative_path = path.relative_to(root) @@ -232,7 +315,16 @@ def scan_source_file( def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: - """Create a source from a newly discovered document.""" + """ + Create a source model from a newly discovered document. + + A sent Webmention is created for each discovered target at the initial source + revision. + + :param scanned: Scanned source data. + :param scan_time: Time at which the source was discovered. + :return: Newly created source model. + """ source = Source( path=scanned.path, url=scanned.url, @@ -254,7 +346,19 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: def update_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: - """Update a source from a newly scanned revision.""" + """ + Update an existing source from newly scanned data. + + A changed or previously deleted source receives a new revision. Existing sent + Webmentions are updated to reflect the current targets, and newly discovered + targets are added. + + :param source: Existing source model to update. + :param scanned: Newly scanned source data. + :param scan_time: Time at which the source was scanned. + :return: Updated source model. + :raises SourceScanError: If the public URL of the source has changed. + """ if source.url != scanned.url: raise SourceScanError( f"Source path {source.path!r} changed public URL " @@ -293,7 +397,15 @@ def update_source( @huey.lock_task("scan-webmention-sources") def scan_sources() -> None: - """Scan generated source documents and queue pending Webmentions.""" + """ + Scan generated source documents and queue pending Webmentions. + + New and changed sources are persisted, missing sources are marked as deleted, + and Webmentions requiring processing are queued for sending. + + :raises RuntimeError: If the configured source directory or base URL is + invalid. + """ directory = current_app.config.get("WEBMENTIONS_SSG_SOURCE_DIRECTORY") if directory is None: @@ -399,4 +511,7 @@ def scan_sources() -> None: @huey.task() def manual_scan_sources() -> None: + """ + Run a source scan as a Huey task. + """ return scan_sources() diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index fa9b624..629c538 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -1,6 +1,6 @@ import re import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from urllib.parse import urljoin import httpx @@ -31,12 +31,23 @@ class PermanentSenderError(SenderError): def temporary_http_status(status: int) -> bool: - """Return whether an HTTP status should be retried.""" + """ + Return whether an HTTP status indicates a temporary failure. + + :param status: HTTP response status code. + :return: Whether the request should be retried. + """ return status in {408, 425, 429} or 500 <= status <= 599 def ensure_public_request(request: httpx.Request) -> None: - """Prevent requests to non-public network addresses.""" + """ + Ensure that an HTTP request targets a public network address. + + :param request: HTTP request to validate. + :raises PermanentSenderError: If the request resolves to a non-public address. + :raises TemporarySenderError: If the request hostname cannot be resolved. + """ try: if not is_public_url(str(request.url)): raise PermanentSenderError("Request resolves to a non-public address") @@ -45,7 +56,15 @@ def ensure_public_request(request: httpx.Request) -> None: def resolve_endpoint(response: httpx.Response, href: str) -> str: - """Resolve and validate a discovered Webmention endpoint.""" + """ + Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :return: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or + HTTPS URL. + """ endpoint = urljoin(str(response.url), href.strip()) if not is_http_url(endpoint): @@ -55,7 +74,12 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str: def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """Parse a Link header value into its target and relations.""" + """ + Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :return: Link target and relation names, or ``None`` if the value is invalid. + """ value = value.strip() if not value.startswith("<"): @@ -80,7 +104,13 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None: def endpoint_from_headers(response: httpx.Response) -> str | None: - """Return the first Webmention endpoint advertised by HTTP Link.""" + """ + Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :return: First advertised Webmention endpoint, or ``None`` if none is found. + :raises PermanentSenderError: If a discovered endpoint is invalid. + """ for header in response.headers.get_list("Link"): for value in LINK_SPLIT.split(header): if (link := parse_link_value(value)) is None: @@ -95,7 +125,16 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """Return the first HTML Webmention endpoint in document order.""" + """ + Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :return: First advertised Webmention endpoint, or ``None`` if none is found. + :raises PermanentSenderError: If a discovered endpoint is invalid. + """ document = BeautifulSoup(body, "html5lib") for element in document.find_all(["link", "a"], href=True): @@ -124,7 +163,14 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: def read_target_body(response: httpx.Response) -> bytes: - """Read a target document up to the configured size limit.""" + """ + Read a target document up to the configured size limit. + + :param response: Streaming HTTP response to read. + :return: Response body. + :raises PermanentSenderError: If the document exceeds the configured maximum + size. + """ max_bytes = current_app.config.get("WEBMENTIONS_SSG_MAX_TARGET_BYTES", 1_000_000) if (content_length := response.headers.get("Content-Length")) is not None: @@ -146,7 +192,20 @@ def read_target_body(response: httpx.Response) -> bytes: def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """Discover the Webmention endpoint advertised by a target.""" + """ + Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises TemporarySenderError: If the target returns a temporary HTTP failure. + :raises PermanentSenderError: If the target returns a permanent HTTP failure + or advertises an invalid endpoint. + """ head_response = client.head(target, headers=DISCOVERY_HEADERS) if endpoint := endpoint_from_headers(head_response): @@ -175,7 +234,15 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non def post_webmention( client: httpx.Client, *, endpoint: str, source: str, target: str ) -> tuple[int, str | None]: - """POST a Webmention and return its status code and status URL.""" + """ + Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :return: HTTP response status code and optional status URL. + """ with client.stream( "POST", endpoint, data={"source": source, "target": target} ) as response: @@ -190,7 +257,16 @@ def post_webmention( def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: - """Return whether an attempt still represents the desired revision.""" + """ + Check whether a send attempt still represents the desired revision. + + The Webmention is refreshed from the database before comparing its desired + and processed revisions. + + :param webmention: Sent Webmention being processed. + :param revision: Source revision represented by the current attempt. + :return: Whether the attempt is still current and requires processing. + """ db.session.refresh(webmention) return webmention.desired_revision == revision and ( @@ -201,7 +277,18 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @huey.task(retries=2, retry_delay=50) def send_webmention(webmention_uuid: uuid.UUID) -> None: - """Discover a receiver endpoint and send one Webmention.""" + """ + Discover a receiver endpoint and send a Webmention. + + The result of the attempt is stored on the corresponding sent Webmention. + Temporary failures are re-raised so Huey can retry them, while permanent + failures mark the current revision as processed. + + :param webmention_uuid: Identifier of the sent Webmention to process. + :raises TemporarySenderError: If endpoint discovery or delivery fails for a + potentially temporary reason. + :raises httpx.RequestError: If an HTTP request fails. + """ webmention = db.session.get(SentWebmention, webmention_uuid) if webmention is None: @@ -220,7 +307,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: source = webmention.source.url target = webmention.target - webmention.last_attempted_at = datetime.now(timezone.utc) + webmention.last_attempted_at = datetime.now(UTC) webmention.endpoint = None webmention.response_status = None webmention.status_url = None @@ -310,6 +397,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.response_status = response_status webmention.status_url = status_url - webmention.last_sent_at = datetime.now(timezone.utc) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() |
