diff options
| author | Dennis Fink | 2026-11-12 19:57:08 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-12 19:57:08 +0100 |
| commit | 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch) | |
| tree | f7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/receiver.py | |
| parent | 7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff) | |
| download | webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip | |
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and
application extension setup so each module owns a focused concern.
Reorganize the test suite around the new boundaries while preserving
existing Webmention behavior and coverage.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 281 |
1 files changed, 95 insertions, 186 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index acd66a1..bed923c 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -2,27 +2,27 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Fetch and verify incoming Webmentions against their source pages.""" +"""Fetch and verify incoming Webmentions with transactional state changes.""" import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx -import rfc3987 import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey -from ..models import ReceivedWebmention -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..models import ReceivedStatus, ReceivedWebmention +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_temporary_status, + read_limited_body, +) +from .mentions import html_mentions_target, text_mentions_target class VerificationError(Exception): @@ -37,95 +37,7 @@ class TemporaryFetchError(Exception): """Fetching the source may succeed when retried later.""" -IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") -TRAILING_IRI_PUNCTUATION = ".,;:!?" - - -HTML_URL_ATTRIBUTES = { - "href": {"a", "area", "link"}, - "src": { - "audio", - "embed", - "iframe", - "img", - 'input[type="image" i]', - "script", - "audio source", - "video source", - "track", - "video", - }, - "cite": {"blockquote", "del", "ins", "q"}, -} - - -def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """ - Check whether an HTML document mentions the target URL. - - URL references are resolved against the document's ``base`` element when - present, or against the source URL otherwise. - - :param body: HTML document body. - :param source_url: URL from which the document was retrieved. - :param target_url: Exact URL to look for. - :return: Whether a supported HTML URL attribute references the target URL. - """ - - document = BeautifulSoup(body, "html.parser") - base_url = source_url - - if (base_element := document.select_one("base[href]")) is not None and isinstance( - base_href := base_element.get("href"), str - ): - base_url = urljoin(source_url, base_href.strip()) - - for attribute, selectors in HTML_URL_ATTRIBUTES.items(): - selector = ", ".join( - [f"{selector_string}[{attribute}]" for selector_string in selectors] - ) - - for element in document.select(selector): - if not isinstance(reference := element.get(attribute), str): - continue - - if urljoin(base_url, reference.strip()) == target_url: - return True - - return False - - -def text_mentions_target(body: str, target_url: str) -> bool: - """ - Check whether plain text contains the target IRI, ignoring trailing prose - punctuation but not extra URL path or query characters. - - :param body: Plain-text document body. - :param target_url: Exact IRI to look for. - :return: Whether the target IRI occurs in the document. - """ - for match in IRI_PATTERN.finditer(body): - candidate = match.group() - - # A URL in prose can be followed by punctuation that the IRI pattern - # consumes as part of its match. Only remove trailing punctuation, - # leaving longer URLs (e.g. ``target/more``) unmatched. - if not candidate.startswith(target_url): - continue - - while candidate != target_url: - if candidate[-1] in TRAILING_IRI_PUNCTUATION: - candidate = candidate[:-1] - elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): - # Keep balanced parentheses that belong to the URL itself. - candidate = candidate[:-1] - else: - break - - if candidate == target_url: - return True - - return False +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: @@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ try: - return _fetch_source(source_url) + with ( + build_client( + f"{APP_NAME}/{VERSION} ReceivedWebmention", + accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if is_temporary_status(status): + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") + + try: + return response, read_limited_body( + response, current_app.config["MAX_SOURCE_BYTES"], label="Source" + ) + except ValueError as exc: + raise VerificationError(str(exc)) from exc + except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc -def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - with ( - httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client, - client.stream("GET", source_url) as response, - ): - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") - - max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass - - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_source_bytes: - raise VerificationError("Source document is too large") - - return response, bytes(body) - - def source_mentions_target(source_url: str, target_url: str) -> bool: """ Fetch a source and check whether it mentions the target URL. @@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - # Failed tasks may be retried by Huey. Completed and already-running tasks - # should not start another concurrent verification. - if webmention.status not in {"received", "failed"}: + if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: return - # A reaper-exhausted row must not be revived by a late Huey retry. + if ( - webmention.status == "failed" + webmention.status == ReceivedStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return @@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ReceivedWebmention.status == previous_status, ReceivedWebmention.updated_at == previous_updated_at, ) - .values(status="verifying", failure_reason=None, updated_at=started_at) + .values( + status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at + ) ) + if claimed.rowcount != 1: db.session.rollback() return - db.session.commit() - def finish(status: str, failure_reason: str | None) -> None: - # A timeout or a newer verification may have superseded this attempt. - # Do not let a late worker overwrite its state. - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == "verifying", - ReceivedWebmention.updated_at == started_at, - ) - .values( - status=status, - failure_reason=failure_reason, - updated_at=datetime.now(UTC), - ) - ) - db.session.commit() + db.session.commit() + status = None + failure_reason = None try: mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - finish("deleted", str(exc)) + status = ReceivedStatus.DELETED + failure_reason = str(exc) except VerificationError as exc: - finish("failed", str(exc)) + status = ReceivedStatus.FAILED + failure_reason = str(exc) except (TemporaryFetchError, httpx.RequestError) as exc: - finish("failed", str(exc) or "Source could not be fetched") + status = ReceivedStatus.FAILED + failure_reason = str(exc) or "Source could not be fetched" raise except Exception as exc: # InvalidURL is not a RequestError; other unexpected source errors must @@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", webmention_uuid ) - finish("failed", str(exc) or type(exc).__name__) + + status = ReceivedStatus.FAILED + failure_reason = str(exc) or type(exc).__name__ else: if mentions_target: - finish("verified", None) + status = ReceivedStatus.VERIFIED else: - finish("deleted", "Source does not mention target") + status = ReceivedStatus.DELETED + failure_reason = "Source does not mention target" + finally: + if status is not None: + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() def recover_stale_webmentions() -> int: @@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int: """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( sa.select( ReceivedWebmention.uuid, @@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.updated_at, ReceivedWebmention.recovery_attempts, ).where( - ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.status.in_( + (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + ), ReceivedWebmention.updated_at <= cutoff, ) ).all() @@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int: recovered = 0 exhausted = 0 max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( @@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.recovery_attempts == recovery_attempts, ) .values( - status="failed" if at_limit else "received", - failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, - updated_at=now, - recovery_attempts=( - recovery_attempts if at_limit else recovery_attempts + 1 - ), + **( + { + "status": ReceivedStatus.FAILED, + "failure_reason": RECOVERY_EXHAUSTED_REASON, + "updated_at": now, + "recovery_attempts": recovery_attempts, + } + if at_limit + else ReceivedWebmention.verification_reset_values( + at=now, recoveries=recovery_attempts + 1 + ) + ) ) ) + if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: exhausted += 1 continue |
