aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/receiver.py
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks/receiver.py')
-rw-r--r--webmentions_ssg/tasks/receiver.py358
1 files changed, 0 insertions, 358 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
deleted file mode 100644
index 313a8e4..0000000
--- a/webmentions_ssg/tasks/receiver.py
+++ /dev/null
@@ -1,358 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Fetch and verify incoming Webmentions with transactional state changes."""
-
-import uuid
-from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
-from http import HTTPStatus
-
-import httpx
-import sqlalchemy as sa
-from flask import current_app
-
-from ..db_helpers import rows_affected
-from ..extensions import db, huey
-from ..models import ReceivedStatus, ReceivedWebmention
-from ..net import (
- AddressResolutionError,
- ResponseTooLargeError,
- UnsafeAddressError,
- build_public_http_client,
- is_retryable_status,
- media_type_of,
- read_limited_body,
-)
-from ..version import APP_NAME, VERSION
-from .target_matching import html_mentions_target, text_mentions_target
-
-
-class VerificationError(Exception):
- """The source permanently failed ReceivedWebmention verification."""
-
-
-class SourceGoneError(VerificationError):
- """The source explicitly reports that it has been removed."""
-
-
-class RetryableFetchError(Exception):
- """Fetching the source may succeed when retried later."""
-
-
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
-
-
-def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]:
- """Fetch a webmention source document.
-
- The request follows redirects while enforcing the configured timeout,
- redirect limit, and maximum response size.
-
- :param source_url: URL of the source document.
- :returns: HTTP response and response body.
- :raises SourceGoneError: If the source returns HTTP 410.
- :raises RetryableFetchError: If the source returns a temporary HTTP error.
- :raises VerificationError: If the source returns another unsuccessful HTTP status or
- exceeds the configured maximum response size.
- """
- try:
- with (
- build_public_http_client(
- f"{APP_NAME}/{VERSION} ReceivedWebmention",
- accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- ) as client,
- client.stream("GET", source_url) as response,
- ):
- if response.status_code == HTTPStatus.OK:
- pass
- elif response.status_code == HTTPStatus.GONE:
- raise SourceGoneError("Source returned HTTP 410")
- elif is_retryable_status(response.status_code):
- raise RetryableFetchError(
- f"Source returned HTTP {response.status_code}"
- )
- else:
- raise VerificationError(f"Source returned HTTP {response.status_code}")
-
- try:
- return response, read_limited_body(
- response,
- current_app.config["MAX_RECEIVED_SOURCE_BYTES"],
- label="Source",
- )
- except ResponseTooLargeError as exc:
- raise VerificationError(str(exc)) from exc
-
- except UnsafeAddressError as exc:
- raise VerificationError("Source resolves to a non-public address") from exc
- except AddressResolutionError as exc:
- raise RetryableFetchError("Source hostname could not be resolved") from exc
-
-
-def source_mentions_target(source_url: str, target_url: str) -> bool:
- """Fetch a source and check whether it mentions the target URL.
-
- HTML and XHTML sources are inspected for supported URL attributes, while
- plain-text sources are searched for the exact target IRI.
-
- :param source_url: URL of the source document.
- :param target_url: Exact target URL to look for.
- :returns: Whether the source mentions the target URL.
- :raises VerificationError: If the source has an unsupported or missing content type.
- :raises SourceGoneError: If the source explicitly reports that it is gone.
- :raises RetryableFetchError: If fetching the source fails temporarily.
- """
- response, body = fetch_source_document(source_url)
-
- media_type = media_type_of(response.headers)
-
- match media_type:
- case "text/html" | "application/xhtml+xml":
- return html_mentions_target(body, str(response.url), target_url)
- case "text/plain":
- try:
- decoded_body = body.decode(
- response.encoding or "utf-8", errors="replace"
- )
- except LookupError:
- decoded_body = body.decode("utf-8", errors="replace")
- return text_mentions_target(decoded_body, target_url)
- case _:
- raise VerificationError(
- f"Unsupported source content type: {media_type or 'missing'}"
- )
-
-
-@dataclass(frozen=True)
-class VerificationClaim:
- """Source and lease version for one atomically claimed verification."""
-
- webmention_uuid: uuid.UUID
- source: str
- target: str
- lease_started_at: datetime
-
-
-def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
- """Claim an eligible row, returning its source, target, and lease version.
-
- :param webmention_uuid: Identifier of the received Webmention to claim.
- :returns: Verification claim, or ``None`` if the row cannot be claimed.
- """
- webmention = db.session.get(ReceivedWebmention, webmention_uuid)
-
- if webmention is None:
- current_app.logger.warning(
- "Cannot verify unknown ReceivedWebmention %s", webmention_uuid
- )
- return None
-
- if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
- return None
-
- if (
- webmention.status == ReceivedStatus.FAILED
- and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
- ):
- return None
-
- claim = VerificationClaim(
- webmention_uuid=webmention_uuid,
- source=webmention.source,
- target=webmention.target,
- lease_started_at=datetime.now(UTC),
- )
- changed = db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == webmention.status,
- ReceivedWebmention.updated_at == webmention.updated_at,
- )
- .values(
- status=ReceivedStatus.VERIFYING,
- failure_reason=None,
- updated_at=claim.lease_started_at,
- )
- )
-
- if rows_affected(changed) != 1:
- db.session.rollback()
- return None
-
- db.session.commit()
- return claim
-
-
-def determine_verification_outcome(
- claim: VerificationClaim,
-) -> tuple[ReceivedStatus, str | None]:
- """Map verification results and permanent errors to a final status/reason.
-
- :param claim: Claim containing the source and target URLs to verify.
- :returns: Final verification status and optional reason.
- :raises RetryableFetchError: If the source fetch fails transiently and should be
- retried.
- :raises httpx.RequestError: If a transient HTTP request error should be retried.
- """
- try:
- mentions_target = source_mentions_target(claim.source, claim.target)
- except SourceGoneError as exc:
- return ReceivedStatus.DELETED, str(exc)
- except VerificationError as exc:
- return ReceivedStatus.FAILED, str(exc)
- except RetryableFetchError, httpx.RequestError:
- raise # These errors must be persisted and then retried by Huey.
- except Exception as exc:
- # InvalidURL is not a RequestError; unexpected source errors are terminal.
- current_app.logger.exception(
- "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid
- )
- return ReceivedStatus.FAILED, str(exc) or type(exc).__name__
-
- if mentions_target:
- return ReceivedStatus.VERIFIED, None
- return ReceivedStatus.DELETED, "Source does not mention target"
-
-
-def release_claim(
- claim: VerificationClaim, status: ReceivedStatus, reason: str | None
-) -> bool:
- """Persist an outcome only if this worker still owns the verification lease.
-
- :param claim: Verification lease acquired by this worker.
- :param status: Final status to persist for the claimed Webmention.
- :param reason: Optional explanation for a failed or deleted Webmention.
- :returns: ``True`` if the worker still owned the lease and persisted the outcome.
- """
- changed = db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == claim.webmention_uuid,
- ReceivedWebmention.status == ReceivedStatus.VERIFYING,
- ReceivedWebmention.updated_at == claim.lease_started_at,
- )
- .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC))
- )
- db.session.commit()
- return rows_affected(changed) == 1
-
-
-@huey.task(retries=2, retry_delay=50)
-def verify_webmention(webmention_uuid: uuid.UUID) -> None:
- """Claim and verify a received Webmention, preserving its lease on release.
-
- Temporary fetch failures are recorded and re-raised for Huey's retries.
-
- Retry transient fetch errors; persist only results belonging to the currently held
- lease.
-
- :param webmention_uuid: Identifier of the received webmention to verify.
- :raises RetryableFetchError: If fetching the source fails temporarily.
- :raises httpx.RequestError: If an HTTP request fails while fetching.
- """
- if (claim := claim_for_verification(webmention_uuid)) is None:
- return
-
- try:
- status, reason = determine_verification_outcome(claim)
- except (RetryableFetchError, httpx.RequestError) as exc:
- release_claim(
- claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched"
- )
- raise
-
- release_claim(claim, status, reason)
-
-
-def recover_stale_verifications() -> int:
- """Requeue verification attempts that were abandoned or lost from the queue.
-
- Recover both stale ``verifying`` rows (workers that died) and stale
- ``received`` rows (tasks that were never enqueued or were lost). Conditional
- updates prevent overlapping reapers from enqueueing the same row. Each
- reclaimed row is committed *before* enqueueing, so the worker can claim it.
- A crash between commit and enqueue is recovered on a later sweep.
- Persist a recovery count so repeatedly lost or crashing tasks eventually
- terminate rather than being requeued forever.
-
- Eligible expired verification leases are returned to the queue, subject to recovery
- limits.
-
- :returns: Number of webmentions requeued.
- """
- now = datetime.now(UTC)
- cutoff = now - timedelta(
- seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"]
- )
-
- stale = db.session.execute(
- sa.select(
- ReceivedWebmention.uuid,
- ReceivedWebmention.status,
- ReceivedWebmention.updated_at,
- ReceivedWebmention.recovery_attempts,
- ).where(
- ReceivedWebmention.status.in_(
- (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
- ),
- ReceivedWebmention.updated_at <= cutoff,
- )
- ).all()
-
- recovered = 0
- exhausted = 0
- max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
-
- for identifier, status, updated_at, recovery_attempts in stale:
- at_limit = recovery_attempts >= max_recoveries
- changed = db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == identifier,
- ReceivedWebmention.status == status,
- ReceivedWebmention.updated_at == updated_at,
- ReceivedWebmention.recovery_attempts == recovery_attempts,
- )
- .values(
- **(
- {
- "status": ReceivedStatus.FAILED,
- "failure_reason": RECOVERY_EXHAUSTED_REASON,
- "updated_at": now,
- "recovery_attempts": recovery_attempts,
- }
- if at_limit
- else ReceivedWebmention.verification_reset_values(
- at=now, recoveries=recovery_attempts + 1
- )
- )
- )
- )
-
- if rows_affected(changed) != 1:
- db.session.rollback()
- continue
-
- db.session.commit()
-
- if at_limit:
- exhausted += 1
- continue
- verify_webmention(identifier)
- recovered += 1
-
- if exhausted:
- current_app.logger.warning(
- "Failed %d stale webmentions after maximum verification recoveries",
- exhausted,
- )
-
- if recovered:
- current_app.logger.warning(
- "Requeued %d stale received webmention verifications", recovered
- )
-
- return recovered