diff options
Diffstat (limited to 'webmentions_ssg/tasks')
| -rw-r--r-- | webmentions_ssg/tasks/__init__.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 12 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/endpoint_discovery.py (renamed from webmentions_ssg/tasks/discovery.py) | 0 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/notifications.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 38 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 38 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/verification.py (renamed from webmentions_ssg/tasks/receiver.py) | 47 |
9 files changed, 84 insertions, 74 deletions
diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py index c744b49..37ab4d2 100644 --- a/webmentions_ssg/tasks/__init__.py +++ b/webmentions_ssg/tasks/__init__.py @@ -4,6 +4,6 @@ """Expose the background task queue integration.""" -from .extension import Huey +from .extension import FlaskHuey -__all__ = ["Huey"] +__all__ = ["FlaskHuey"] diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index c24a883..7991b6a 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -8,11 +8,11 @@ from huey import Huey, crontab from .. import create_app from ..extensions import huey as queue -from . import notifications, receiver, retention, scanner +from . import notifications, retention, scanner, verification -def create_consumer() -> Huey: - """Create and configure the Huey consumer. +def create_huey() -> Huey: + """Create and configure the Huey instance for the task worker. The application is initialized and the periodic scanner, notification, verification recovery and retention tasks are registered. @@ -23,8 +23,8 @@ def create_consumer() -> Huey: for config_key, task in ( ("SOURCE_SCAN_SCHEDULE", scanner.scan_sources), - ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications), - ("DIGEST_SCHEDULE", notifications.send_webmention_digest), + ("VERIFICATION_RECOVERY_SCHEDULE", verification.recover_stale_verifications), + ("DIGEST_SCHEDULE", notifications.email_verified_digest), ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), ): schedule = crontab(*app.config[config_key].split(), strict=True) @@ -33,4 +33,4 @@ def create_consumer() -> Huey: return queue.huey -huey = create_consumer() +huey = create_huey() diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/endpoint_discovery.py index c683cb0..c683cb0 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/endpoint_discovery.py diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index faed364..3f172f4 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -4,8 +4,6 @@ """Integrate Huey tasks with Flask application contexts.""" -from __future__ import annotations - from collections.abc import Callable from functools import update_wrapper, wraps from typing import Any @@ -100,7 +98,7 @@ class _DeferredTask: def __init__( self, - extension: Huey, + extension: FlaskHuey, func: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any], @@ -152,7 +150,7 @@ class _DeferredTask: return getattr(self._bound_task(), name) -class Huey: +class FlaskHuey: """Provide Flask integration for a Huey instance. The extension initializes a Huey backend from the Flask configuration and diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index eb540de..c503e1f 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,10 +11,10 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus -def send_webmention_digest() -> None: +def email_verified_digest() -> None: """Send one digest of verified, previously unreported Webmentions. Acquire the digest lock before notifying recipients about previously unreported @@ -38,7 +38,7 @@ def _send_digest_locked() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == ReceivedStatus.VERIFIED, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 2c35cc7..b882493 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: @@ -33,7 +33,10 @@ def purge_old_received_webmentions() -> int: result = db.session.execute( sa.delete(ReceivedWebmention).where( ReceivedWebmention.status.in_( - (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ( + ReceivedWebmentionStatus.FAILED, + ReceivedWebmentionStatus.SOURCE_REMOVED, + ) ), ReceivedWebmention.updated_at < cutoff, ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 034b2cf..fb0c770 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,13 +14,13 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, Source +from ..models import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file -def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: +def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. A sent Webmention is built for each discovered target at the initial source @@ -30,7 +30,7 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: :param scan_time: Time at which the source was discovered. :returns: Newly built source model. """ - source = Source( + source = SourceDocument( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, @@ -42,15 +42,15 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: for target in scanned.targets: source.sent_webmentions.append( - SentWebmention(target=target, active=True, desired_revision=1) + SentWebmention(target=target, linked=True, desired_revision=1) ) return source def apply_scan_to_source( - source: Source, scanned: ScannedSource, scan_time: datetime -) -> Source: + source: SourceDocument, scanned: ScannedSource, scan_time: datetime +) -> SourceDocument: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent @@ -82,16 +82,18 @@ def apply_scan_to_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.reopen(source.revision, webmention.target in scanned.targets) + webmention.reset_for_revision( + source.revision, webmention.target in scanned.targets + ) - if not webmention.active and webmention.sent_revision is None: + if not webmention.linked and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( - target=target, active=True, desired_revision=source.revision + target=target, linked=True, desired_revision=source.revision ) ) @@ -127,7 +129,7 @@ class TreeScanResult: def reconcile_source_tree( root: Path, base_url: str | None, - sources_by_path: dict[str, Source], + sources_by_path: dict[str, SourceDocument], paths_by_url: dict[str, str], scan_time: datetime, ) -> TreeScanResult: @@ -197,14 +199,16 @@ def reconcile_source_tree( def mark_missing_sources_deleted( - sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime + sources_by_path: dict[str, SourceDocument], + seen_paths: frozenset[str], + scan_time: datetime, ) -> int: - """Mark absent, previously active sources deleted and return their count. + """Mark absent, previously present sources deleted and return their count. :param sources_by_path: Known source records indexed by relative path. :param seen_paths: Paths found during this scan, including failed parses. :param scan_time: Timestamp to record for sources newly marked missing. - :returns: Number of previously active sources marked as deleted. + :returns: Number of previously linked sources marked as deleted. """ marked = 0 for path, source in sources_by_path.items(): @@ -217,7 +221,7 @@ def mark_missing_sources_deleted( marked += 1 for webmention in source.sent_webmentions: - webmention.reopen(source.revision, False) + webmention.reset_for_revision(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -235,7 +239,7 @@ def enqueue_due_sends(scan_time: datetime) -> int: for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, @@ -273,7 +277,9 @@ def _scan_and_enqueue() -> None: sources_by_path = { source.path: source for source in db.session.scalars( - sa.select(Source).options(selectinload(Source.sent_webmentions)) + sa.select(SourceDocument).options( + selectinload(SourceDocument.sent_webmentions) + ) ) } diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index c33cf06..3f64bed 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -24,7 +24,7 @@ from ..net import ( is_retryable_status, ) from ..version import APP_NAME, VERSION -from .discovery import ( +from .endpoint_discovery import ( PermanentDeliveryError, RetryableDeliveryError, discover_webmention_endpoint, @@ -44,7 +44,7 @@ def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """ db.session.refresh(webmention) - return webmention.desired_revision == revision and webmention.pending + return webmention.desired_revision == revision and webmention.awaiting_processing def backoff_delay(attempt: int) -> timedelta: @@ -99,8 +99,8 @@ class ClaimedSendAttempt: class DeliveryResult: """HTTP outcome, with whether the failure is eligible for another attempt. - ``terminal`` describes the failure type. The retry budget can also make a - temporary failure terminal when its last allowed attempt has been used. + ``retryable`` describes the failure type; it does not account for the + remaining retry budget, which may already have been exhausted. """ status: SentWebmentionStatus @@ -108,8 +108,7 @@ class DeliveryResult: endpoint: str | None = None response_status: int | None = None status_url: str | None = None - terminal: bool = True - temporary_failure: bool = False + retryable: bool = False def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: @@ -119,7 +118,7 @@ def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: :param now: Current timestamp against which the retry deadline is compared. :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed. """ - if not webmention.pending: + if not webmention.awaiting_processing: return False deadline = webmention.next_retry_at @@ -154,7 +153,7 @@ def reserve_send_attempt( SentWebmention.uuid == webmention.uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count >= max_attempts, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -188,7 +187,7 @@ def reserve_send_attempt( SentWebmention.uuid == attempt.webmention_uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count == attempt.attempt_number - 1, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -198,8 +197,8 @@ def reserve_send_attempt( attempt_count=attempt.attempt_number, last_attempted_at=now, next_retry_at=now + backoff_delay(attempt.attempt_number), - endpoint=None, - response_status=None, + webmention_endpoint=None, + endpoint_response_status=None, status_url=None, ) .execution_options(synchronize_session=False) @@ -236,7 +235,7 @@ def attempt_delivery( if endpoint is None: return DeliveryResult( - status=SentWebmentionStatus.UNSUPPORTED, + status=SentWebmentionStatus.NO_ENDPOINT, reason="No Webmention endpoint discovered", ) @@ -275,11 +274,10 @@ def attempt_delivery( reason=str(exc) or "Webmention request failed", endpoint=endpoint, response_status=response_status, - terminal=False, - temporary_failure=True, + retryable=True, ) except Exception as exc: - # Unexpected errors should be terminal and keep their traceback. + # Unexpected errors should not be retried; keep their traceback. current_app.logger.exception( "Unexpected error sending SentWebmention %s", webmention_uuid ) @@ -312,8 +310,8 @@ def record_delivery_outcome( webmention.status = result.status webmention.failure_reason = result.reason - webmention.endpoint = result.endpoint - webmention.response_status = result.response_status + webmention.webmention_endpoint = result.endpoint + webmention.endpoint_response_status = result.response_status if terminal: webmention.processed_revision = attempt.revision @@ -355,19 +353,19 @@ def _process_send_locked(webmention_uuid: uuid.UUID) -> bool: result = attempt_delivery( attempt.source, attempt.target, webmention_uuid=webmention_uuid ) - terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts + terminal = not result.retryable or attempt.attempt_number >= attempt.max_attempts if not record_delivery_outcome(webmention, attempt, result, terminal=terminal): return False - if result.temporary_failure and not terminal: + if result.retryable and not terminal: # Huey does not retry: the scanner requeues after next_retry_at. current_app.logger.info( "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt.attempt_number, ) - elif result.temporary_failure: + elif result.retryable: current_app.logger.warning( "Webmention %s exhausted %d send attempts: %s", webmention_uuid, diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/verification.py index 313a8e4..a614815 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/verification.py @@ -15,7 +15,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus from ..net import ( AddressResolutionError, ResponseTooLargeError, @@ -149,11 +149,14 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No ) return None - if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: + if webmention.status not in { + ReceivedWebmentionStatus.RECEIVED, + ReceivedWebmentionStatus.FAILED, + }: return None if ( - webmention.status == ReceivedStatus.FAILED + webmention.status == ReceivedWebmentionStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return None @@ -172,7 +175,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No ReceivedWebmention.updated_at == webmention.updated_at, ) .values( - status=ReceivedStatus.VERIFYING, + status=ReceivedWebmentionStatus.VERIFYING, failure_reason=None, updated_at=claim.lease_started_at, ) @@ -188,7 +191,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No def determine_verification_outcome( claim: VerificationClaim, -) -> tuple[ReceivedStatus, str | None]: +) -> tuple[ReceivedWebmentionStatus, str | None]: """Map verification results and permanent errors to a final status/reason. :param claim: Claim containing the source and target URLs to verify. @@ -200,9 +203,9 @@ def determine_verification_outcome( try: mentions_target = source_mentions_target(claim.source, claim.target) except SourceGoneError as exc: - return ReceivedStatus.DELETED, str(exc) + return ReceivedWebmentionStatus.SOURCE_REMOVED, str(exc) except VerificationError as exc: - return ReceivedStatus.FAILED, str(exc) + return ReceivedWebmentionStatus.FAILED, str(exc) except RetryableFetchError, httpx.RequestError: raise # These errors must be persisted and then retried by Huey. except Exception as exc: @@ -210,28 +213,28 @@ def determine_verification_outcome( current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid ) - return ReceivedStatus.FAILED, str(exc) or type(exc).__name__ + return ReceivedWebmentionStatus.FAILED, str(exc) or type(exc).__name__ if mentions_target: - return ReceivedStatus.VERIFIED, None - return ReceivedStatus.DELETED, "Source does not mention target" + return ReceivedWebmentionStatus.VERIFIED, None + return ReceivedWebmentionStatus.SOURCE_REMOVED, "Source does not mention target" -def release_claim( - claim: VerificationClaim, status: ReceivedStatus, reason: str | None +def store_verification_outcome( + claim: VerificationClaim, status: ReceivedWebmentionStatus, reason: str | None ) -> bool: """Persist an outcome only if this worker still owns the verification lease. :param claim: Verification lease acquired by this worker. :param status: Final status to persist for the claimed Webmention. - :param reason: Optional explanation for a failed or deleted Webmention. + :param reason: Optional explanation for a failed or source-removed Webmention. :returns: ``True`` if the worker still owned the lease and persisted the outcome. """ changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == claim.webmention_uuid, - ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFYING, ReceivedWebmention.updated_at == claim.lease_started_at, ) .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) @@ -259,12 +262,14 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: try: status, reason = determine_verification_outcome(claim) except (RetryableFetchError, httpx.RequestError) as exc: - release_claim( - claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched" + store_verification_outcome( + claim, + ReceivedWebmentionStatus.FAILED, + str(exc) or "Source could not be fetched", ) raise - release_claim(claim, status, reason) + store_verification_outcome(claim, status, reason) def recover_stale_verifications() -> int: @@ -296,7 +301,7 @@ def recover_stale_verifications() -> int: ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_( - (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + (ReceivedWebmentionStatus.VERIFYING, ReceivedWebmentionStatus.RECEIVED) ), ReceivedWebmention.updated_at <= cutoff, ) @@ -319,14 +324,14 @@ def recover_stale_verifications() -> int: .values( **( { - "status": ReceivedStatus.FAILED, + "status": ReceivedWebmentionStatus.FAILED, "failure_reason": RECOVERY_EXHAUSTED_REASON, "updated_at": now, "recovery_attempts": recovery_attempts, } if at_limit - else ReceivedWebmention.verification_reset_values( - at=now, recoveries=recovery_attempts + 1 + else ReceivedWebmention.reopen_verification_values( + at=now, recovery_attempts=recovery_attempts + 1 ) ) ) |
