# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause """Fetch and verify incoming Webmentions with transactional state changes.""" import uuid from datetime import UTC, datetime, timedelta import httpx import sqlalchemy as sa from flask import current_app from .. import APP_NAME, VERSION from ..extensions import DATABASE as db from ..extensions import HUEY as huey from ..models import ReceivedStatus, ReceivedWebmention from ..net import ( AddressResolutionError, UnsafeAddressError, build_client, is_temporary_status, read_limited_body, ) from .mentions import html_mentions_target, text_mentions_target class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" class SourceGoneError(VerificationError): """The source explicitly reports that it has been removed.""" class TemporaryFetchError(Exception): """Fetching the source may succeed when retried later.""" RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ Fetch a webmention source document. The request follows redirects while enforcing the configured timeout, redirect limit, and maximum response size. :param source_url: URL of the source document. :return: HTTP response and response body. :raises SourceGoneError: If the source returns HTTP 410. :raises TemporaryFetchError: If the source returns a temporary HTTP error. :raises VerificationError: If the source returns another unsuccessful HTTP status or exceeds the configured maximum response size. """ try: with ( build_client( f"{APP_NAME}/{VERSION} ReceivedWebmention", accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", ) as client, client.stream("GET", source_url) as response, ): if response.status_code == 200: pass elif response.status_code == 410: raise SourceGoneError("Source returned HTTP 410") elif is_temporary_status(response.status_code): raise TemporaryFetchError( f"Source returned HTTP {response.status_code}" ) else: raise VerificationError(f"Source returned HTTP {response.status_code}") try: return response, read_limited_body( response, current_app.config["MAX_SOURCE_BYTES"], label="Source" ) except ValueError as exc: raise VerificationError(str(exc)) from exc except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc def source_mentions_target(source_url: str, target_url: str) -> bool: """ Fetch a source and check whether it mentions the target URL. HTML and XHTML sources are inspected for supported URL attributes, while plain-text sources are searched for the exact target IRI. :param source_url: URL of the source document. :param target_url: Exact target URL to look for. :return: Whether the source mentions the target URL. :raises VerificationError: If the source has an unsupported or missing content type. :raises SourceGoneError: If the source explicitly reports that it is gone. :raises TemporaryFetchError: If fetching the source fails temporarily. """ response, body = fetch_source(source_url) media_type = ( response.headers.get("Content-Type", "").partition(";")[0].strip().lower() ) match media_type: case "text/html" | "application/xhtml+xml": return html_mentions_target(body, str(response.url), target_url) case "text/plain": try: decoded_body = body.decode( response.encoding or "utf-8", errors="replace" ) except LookupError: decoded_body = body.decode("utf-8", errors="replace") return text_mentions_target(decoded_body, target_url) case _: raise VerificationError( f"Unsupported source content type: {media_type or 'missing'}" ) @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: """ Verify a received webmention and persist the result. The transition into ``verifying`` is an atomic claim. Its ``updated_at`` timestamp acts as a lease version: if a reaper has reclaimed the row while the fetch was in progress, the old worker cannot overwrite the new result. Temporary fetch failures are persisted and re-raised for Huey's retries. :param webmention_uuid: Identifier of the received webmention to verify. :raises TemporaryFetchError: If fetching the source fails temporarily. :raises httpx.RequestError: If an HTTP request error occurs while fetching the source. """ webmention = db.session.get(ReceivedWebmention, webmention_uuid) if webmention is None: current_app.logger.warning( "Cannot verify unknown ReceivedWebmention %s", webmention_uuid ) return if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: return if ( webmention.status == ReceivedStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return previous_status = webmention.status previous_updated_at = webmention.updated_at source, target = webmention.source, webmention.target started_at = datetime.now(UTC) claimed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == webmention_uuid, ReceivedWebmention.status == previous_status, ReceivedWebmention.updated_at == previous_updated_at, ) .values( status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at ) ) if claimed.rowcount != 1: db.session.rollback() return db.session.commit() status = None failure_reason = None try: mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: status = ReceivedStatus.DELETED failure_reason = str(exc) except VerificationError as exc: status = ReceivedStatus.FAILED failure_reason = str(exc) except (TemporaryFetchError, httpx.RequestError) as exc: status = ReceivedStatus.FAILED failure_reason = str(exc) or "Source could not be fetched" raise except Exception as exc: # InvalidURL is not a RequestError; other unexpected source errors must # also leave the row in a terminal state, without an endless retry loop. current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", webmention_uuid ) status = ReceivedStatus.FAILED failure_reason = str(exc) or type(exc).__name__ else: if mentions_target: status = ReceivedStatus.VERIFIED else: status = ReceivedStatus.DELETED failure_reason = "Source does not mention target" finally: if status is not None: db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == webmention_uuid, ReceivedWebmention.status == ReceivedStatus.VERIFYING, ReceivedWebmention.updated_at == started_at, ) .values( status=status, failure_reason=failure_reason, updated_at=datetime.now(UTC), ) ) db.session.commit() def recover_stale_webmentions() -> int: """ Requeue verification attempts that were abandoned or lost from the queue. Recover both stale ``verifying`` rows (workers that died) and stale ``received`` rows (tasks that were never enqueued or were lost). Conditional updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. Persist a recovery count so repeatedly lost or crashing tasks eventually terminate rather than being requeued forever. :return: Number of webmentions requeued. """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) stale = db.session.execute( sa.select( ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_( (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) ), ReceivedWebmention.updated_at <= cutoff, ) ).all() recovered = 0 exhausted = 0 max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] for identifier, status, updated_at, recovery_attempts in stale: at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, ReceivedWebmention.recovery_attempts == recovery_attempts, ) .values( **( { "status": ReceivedStatus.FAILED, "failure_reason": RECOVERY_EXHAUSTED_REASON, "updated_at": now, "recovery_attempts": recovery_attempts, } if at_limit else ReceivedWebmention.verification_reset_values( at=now, recoveries=recovery_attempts + 1 ) ) ) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() if at_limit: exhausted += 1 continue verify_webmention(identifier) recovered += 1 if exhausted: current_app.logger.warning( "Failed %d stale webmentions after maximum verification recoveries", exhausted, ) if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered ) return recovered