diff options
Diffstat (limited to 'webmentions_ssg/tasks/receiver.py')
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 358 |
1 files changed, 0 insertions, 358 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py deleted file mode 100644 index 313a8e4..0000000 --- a/webmentions_ssg/tasks/receiver.py +++ /dev/null @@ -1,358 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Fetch and verify incoming Webmentions with transactional state changes.""" - -import uuid -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from http import HTTPStatus - -import httpx -import sqlalchemy as sa -from flask import current_app - -from ..db_helpers import rows_affected -from ..extensions import db, huey -from ..models import ReceivedStatus, ReceivedWebmention -from ..net import ( - AddressResolutionError, - ResponseTooLargeError, - UnsafeAddressError, - build_public_http_client, - is_retryable_status, - media_type_of, - read_limited_body, -) -from ..version import APP_NAME, VERSION -from .target_matching import html_mentions_target, text_mentions_target - - -class VerificationError(Exception): - """The source permanently failed ReceivedWebmention verification.""" - - -class SourceGoneError(VerificationError): - """The source explicitly reports that it has been removed.""" - - -class RetryableFetchError(Exception): - """Fetching the source may succeed when retried later.""" - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" - - -def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: - """Fetch a webmention source document. - - The request follows redirects while enforcing the configured timeout, - redirect limit, and maximum response size. - - :param source_url: URL of the source document. - :returns: HTTP response and response body. - :raises SourceGoneError: If the source returns HTTP 410. - :raises RetryableFetchError: If the source returns a temporary HTTP error. - :raises VerificationError: If the source returns another unsuccessful HTTP status or - exceeds the configured maximum response size. - """ - try: - with ( - build_public_http_client( - f"{APP_NAME}/{VERSION} ReceivedWebmention", - accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - ) as client, - client.stream("GET", source_url) as response, - ): - if response.status_code == HTTPStatus.OK: - pass - elif response.status_code == HTTPStatus.GONE: - raise SourceGoneError("Source returned HTTP 410") - elif is_retryable_status(response.status_code): - raise RetryableFetchError( - f"Source returned HTTP {response.status_code}" - ) - else: - raise VerificationError(f"Source returned HTTP {response.status_code}") - - try: - return response, read_limited_body( - response, - current_app.config["MAX_RECEIVED_SOURCE_BYTES"], - label="Source", - ) - except ResponseTooLargeError as exc: - raise VerificationError(str(exc)) from exc - - except UnsafeAddressError as exc: - raise VerificationError("Source resolves to a non-public address") from exc - except AddressResolutionError as exc: - raise RetryableFetchError("Source hostname could not be resolved") from exc - - -def source_mentions_target(source_url: str, target_url: str) -> bool: - """Fetch a source and check whether it mentions the target URL. - - HTML and XHTML sources are inspected for supported URL attributes, while - plain-text sources are searched for the exact target IRI. - - :param source_url: URL of the source document. - :param target_url: Exact target URL to look for. - :returns: Whether the source mentions the target URL. - :raises VerificationError: If the source has an unsupported or missing content type. - :raises SourceGoneError: If the source explicitly reports that it is gone. - :raises RetryableFetchError: If fetching the source fails temporarily. - """ - response, body = fetch_source_document(source_url) - - media_type = media_type_of(response.headers) - - match media_type: - case "text/html" | "application/xhtml+xml": - return html_mentions_target(body, str(response.url), target_url) - case "text/plain": - try: - decoded_body = body.decode( - response.encoding or "utf-8", errors="replace" - ) - except LookupError: - decoded_body = body.decode("utf-8", errors="replace") - return text_mentions_target(decoded_body, target_url) - case _: - raise VerificationError( - f"Unsupported source content type: {media_type or 'missing'}" - ) - - -@dataclass(frozen=True) -class VerificationClaim: - """Source and lease version for one atomically claimed verification.""" - - webmention_uuid: uuid.UUID - source: str - target: str - lease_started_at: datetime - - -def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: - """Claim an eligible row, returning its source, target, and lease version. - - :param webmention_uuid: Identifier of the received Webmention to claim. - :returns: Verification claim, or ``None`` if the row cannot be claimed. - """ - webmention = db.session.get(ReceivedWebmention, webmention_uuid) - - if webmention is None: - current_app.logger.warning( - "Cannot verify unknown ReceivedWebmention %s", webmention_uuid - ) - return None - - if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: - return None - - if ( - webmention.status == ReceivedStatus.FAILED - and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON - ): - return None - - claim = VerificationClaim( - webmention_uuid=webmention_uuid, - source=webmention.source, - target=webmention.target, - lease_started_at=datetime.now(UTC), - ) - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == webmention.status, - ReceivedWebmention.updated_at == webmention.updated_at, - ) - .values( - status=ReceivedStatus.VERIFYING, - failure_reason=None, - updated_at=claim.lease_started_at, - ) - ) - - if rows_affected(changed) != 1: - db.session.rollback() - return None - - db.session.commit() - return claim - - -def determine_verification_outcome( - claim: VerificationClaim, -) -> tuple[ReceivedStatus, str | None]: - """Map verification results and permanent errors to a final status/reason. - - :param claim: Claim containing the source and target URLs to verify. - :returns: Final verification status and optional reason. - :raises RetryableFetchError: If the source fetch fails transiently and should be - retried. - :raises httpx.RequestError: If a transient HTTP request error should be retried. - """ - try: - mentions_target = source_mentions_target(claim.source, claim.target) - except SourceGoneError as exc: - return ReceivedStatus.DELETED, str(exc) - except VerificationError as exc: - return ReceivedStatus.FAILED, str(exc) - except RetryableFetchError, httpx.RequestError: - raise # These errors must be persisted and then retried by Huey. - except Exception as exc: - # InvalidURL is not a RequestError; unexpected source errors are terminal. - current_app.logger.exception( - "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid - ) - return ReceivedStatus.FAILED, str(exc) or type(exc).__name__ - - if mentions_target: - return ReceivedStatus.VERIFIED, None - return ReceivedStatus.DELETED, "Source does not mention target" - - -def release_claim( - claim: VerificationClaim, status: ReceivedStatus, reason: str | None -) -> bool: - """Persist an outcome only if this worker still owns the verification lease. - - :param claim: Verification lease acquired by this worker. - :param status: Final status to persist for the claimed Webmention. - :param reason: Optional explanation for a failed or deleted Webmention. - :returns: ``True`` if the worker still owned the lease and persisted the outcome. - """ - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == claim.webmention_uuid, - ReceivedWebmention.status == ReceivedStatus.VERIFYING, - ReceivedWebmention.updated_at == claim.lease_started_at, - ) - .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) - ) - db.session.commit() - return rows_affected(changed) == 1 - - -@huey.task(retries=2, retry_delay=50) -def verify_webmention(webmention_uuid: uuid.UUID) -> None: - """Claim and verify a received Webmention, preserving its lease on release. - - Temporary fetch failures are recorded and re-raised for Huey's retries. - - Retry transient fetch errors; persist only results belonging to the currently held - lease. - - :param webmention_uuid: Identifier of the received webmention to verify. - :raises RetryableFetchError: If fetching the source fails temporarily. - :raises httpx.RequestError: If an HTTP request fails while fetching. - """ - if (claim := claim_for_verification(webmention_uuid)) is None: - return - - try: - status, reason = determine_verification_outcome(claim) - except (RetryableFetchError, httpx.RequestError) as exc: - release_claim( - claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched" - ) - raise - - release_claim(claim, status, reason) - - -def recover_stale_verifications() -> int: - """Requeue verification attempts that were abandoned or lost from the queue. - - Recover both stale ``verifying`` rows (workers that died) and stale - ``received`` rows (tasks that were never enqueued or were lost). Conditional - updates prevent overlapping reapers from enqueueing the same row. Each - reclaimed row is committed *before* enqueueing, so the worker can claim it. - A crash between commit and enqueue is recovered on a later sweep. - Persist a recovery count so repeatedly lost or crashing tasks eventually - terminate rather than being requeued forever. - - Eligible expired verification leases are returned to the queue, subject to recovery - limits. - - :returns: Number of webmentions requeued. - """ - now = datetime.now(UTC) - cutoff = now - timedelta( - seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"] - ) - - stale = db.session.execute( - sa.select( - ReceivedWebmention.uuid, - ReceivedWebmention.status, - ReceivedWebmention.updated_at, - ReceivedWebmention.recovery_attempts, - ).where( - ReceivedWebmention.status.in_( - (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) - ), - ReceivedWebmention.updated_at <= cutoff, - ) - ).all() - - recovered = 0 - exhausted = 0 - max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] - - for identifier, status, updated_at, recovery_attempts in stale: - at_limit = recovery_attempts >= max_recoveries - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == identifier, - ReceivedWebmention.status == status, - ReceivedWebmention.updated_at == updated_at, - ReceivedWebmention.recovery_attempts == recovery_attempts, - ) - .values( - **( - { - "status": ReceivedStatus.FAILED, - "failure_reason": RECOVERY_EXHAUSTED_REASON, - "updated_at": now, - "recovery_attempts": recovery_attempts, - } - if at_limit - else ReceivedWebmention.verification_reset_values( - at=now, recoveries=recovery_attempts + 1 - ) - ) - ) - ) - - if rows_affected(changed) != 1: - db.session.rollback() - continue - - db.session.commit() - - if at_limit: - exhausted += 1 - continue - verify_webmention(identifier) - recovered += 1 - - if exhausted: - current_app.logger.warning( - "Failed %d stale webmentions after maximum verification recoveries", - exhausted, - ) - - if recovered: - current_app.logger.warning( - "Requeued %d stale received webmention verifications", recovered - ) - - return recovered |
