aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/__init__.py4
-rw-r--r--webmentions_ssg/tasks/consumer.py12
-rw-r--r--webmentions_ssg/tasks/endpoint_discovery.py (renamed from webmentions_ssg/tasks/discovery.py)0
-rw-r--r--webmentions_ssg/tasks/extension.py6
-rw-r--r--webmentions_ssg/tasks/notifications.py6
-rw-r--r--webmentions_ssg/tasks/retention.py7
-rw-r--r--webmentions_ssg/tasks/scanner.py38
-rw-r--r--webmentions_ssg/tasks/sender.py38
-rw-r--r--webmentions_ssg/tasks/verification.py (renamed from webmentions_ssg/tasks/receiver.py)47
9 files changed, 84 insertions, 74 deletions
diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py
index c744b49..37ab4d2 100644
--- a/webmentions_ssg/tasks/__init__.py
+++ b/webmentions_ssg/tasks/__init__.py
@@ -4,6 +4,6 @@
"""Expose the background task queue integration."""
-from .extension import Huey
+from .extension import FlaskHuey
-__all__ = ["Huey"]
+__all__ = ["FlaskHuey"]
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index c24a883..7991b6a 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -8,11 +8,11 @@ from huey import Huey, crontab
from .. import create_app
from ..extensions import huey as queue
-from . import notifications, receiver, retention, scanner
+from . import notifications, retention, scanner, verification
-def create_consumer() -> Huey:
- """Create and configure the Huey consumer.
+def create_huey() -> Huey:
+ """Create and configure the Huey instance for the task worker.
The application is initialized and the periodic scanner, notification,
verification recovery and retention tasks are registered.
@@ -23,8 +23,8 @@ def create_consumer() -> Huey:
for config_key, task in (
("SOURCE_SCAN_SCHEDULE", scanner.scan_sources),
- ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications),
- ("DIGEST_SCHEDULE", notifications.send_webmention_digest),
+ ("VERIFICATION_RECOVERY_SCHEDULE", verification.recover_stale_verifications),
+ ("DIGEST_SCHEDULE", notifications.email_verified_digest),
("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions),
):
schedule = crontab(*app.config[config_key].split(), strict=True)
@@ -33,4 +33,4 @@ def create_consumer() -> Huey:
return queue.huey
-huey = create_consumer()
+huey = create_huey()
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/endpoint_discovery.py
index c683cb0..c683cb0 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/endpoint_discovery.py
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index faed364..3f172f4 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -4,8 +4,6 @@
"""Integrate Huey tasks with Flask application contexts."""
-from __future__ import annotations
-
from collections.abc import Callable
from functools import update_wrapper, wraps
from typing import Any
@@ -100,7 +98,7 @@ class _DeferredTask:
def __init__(
self,
- extension: Huey,
+ extension: FlaskHuey,
func: Callable[..., Any],
args: tuple[Any, ...],
kwargs: dict[str, Any],
@@ -152,7 +150,7 @@ class _DeferredTask:
return getattr(self._bound_task(), name)
-class Huey:
+class FlaskHuey:
"""Provide Flask integration for a Huey instance.
The extension initializes a Huey backend from the Flask configuration and
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index eb540de..c503e1f 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -11,10 +11,10 @@ from flask import current_app
from flask_mail import Message
from ..extensions import db, huey, mail
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
-def send_webmention_digest() -> None:
+def email_verified_digest() -> None:
"""Send one digest of verified, previously unreported Webmentions.
Acquire the digest lock before notifying recipients about previously unreported
@@ -38,7 +38,7 @@ def _send_digest_locked() -> None:
cutoff = datetime.now(UTC)
conditions = (
- ReceivedWebmention.status == ReceivedStatus.VERIFIED,
+ ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFIED,
ReceivedWebmention.notification_sent_at.is_(None),
ReceivedWebmention.updated_at <= cutoff,
)
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 2c35cc7..b882493 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -11,7 +11,7 @@ from flask import current_app
from ..db_helpers import rows_affected
from ..extensions import db
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
def purge_old_received_webmentions() -> int:
@@ -33,7 +33,10 @@ def purge_old_received_webmentions() -> int:
result = db.session.execute(
sa.delete(ReceivedWebmention).where(
ReceivedWebmention.status.in_(
- (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ (
+ ReceivedWebmentionStatus.FAILED,
+ ReceivedWebmentionStatus.SOURCE_REMOVED,
+ )
),
ReceivedWebmention.updated_at < cutoff,
)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 034b2cf..fb0c770 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -14,13 +14,13 @@ from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
from ..extensions import db, huey
-from ..models import SentWebmention, Source
+from ..models import SentWebmention, SourceDocument
from ..net import is_http_url
from .sender import send_webmention
from .source_parser import ScannedSource, SourceScanError, scan_source_file
-def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
+def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument:
"""Build a source model from a newly discovered document.
A sent Webmention is built for each discovered target at the initial source
@@ -30,7 +30,7 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
:param scan_time: Time at which the source was discovered.
:returns: Newly built source model.
"""
- source = Source(
+ source = SourceDocument(
path=scanned.path,
url=scanned.url,
content_hash=scanned.content_hash,
@@ -42,15 +42,15 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
for target in scanned.targets:
source.sent_webmentions.append(
- SentWebmention(target=target, active=True, desired_revision=1)
+ SentWebmention(target=target, linked=True, desired_revision=1)
)
return source
def apply_scan_to_source(
- source: Source, scanned: ScannedSource, scan_time: datetime
-) -> Source:
+ source: SourceDocument, scanned: ScannedSource, scan_time: datetime
+) -> SourceDocument:
"""Update an existing source from newly scanned data.
A changed or previously deleted source receives a new revision. Existing sent
@@ -82,16 +82,18 @@ def apply_scan_to_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.reopen(source.revision, webmention.target in scanned.targets)
+ webmention.reset_for_revision(
+ source.revision, webmention.target in scanned.targets
+ )
- if not webmention.active and webmention.sent_revision is None:
+ if not webmention.linked and webmention.sent_revision is None:
webmention.processed_revision = source.revision
for target in scanned.targets:
if target not in existing:
source.sent_webmentions.append(
SentWebmention(
- target=target, active=True, desired_revision=source.revision
+ target=target, linked=True, desired_revision=source.revision
)
)
@@ -127,7 +129,7 @@ class TreeScanResult:
def reconcile_source_tree(
root: Path,
base_url: str | None,
- sources_by_path: dict[str, Source],
+ sources_by_path: dict[str, SourceDocument],
paths_by_url: dict[str, str],
scan_time: datetime,
) -> TreeScanResult:
@@ -197,14 +199,16 @@ def reconcile_source_tree(
def mark_missing_sources_deleted(
- sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
+ sources_by_path: dict[str, SourceDocument],
+ seen_paths: frozenset[str],
+ scan_time: datetime,
) -> int:
- """Mark absent, previously active sources deleted and return their count.
+ """Mark absent, previously present sources deleted and return their count.
:param sources_by_path: Known source records indexed by relative path.
:param seen_paths: Paths found during this scan, including failed parses.
:param scan_time: Timestamp to record for sources newly marked missing.
- :returns: Number of previously active sources marked as deleted.
+ :returns: Number of previously linked sources marked as deleted.
"""
marked = 0
for path, source in sources_by_path.items():
@@ -217,7 +221,7 @@ def mark_missing_sources_deleted(
marked += 1
for webmention in source.sent_webmentions:
- webmention.reopen(source.revision, False)
+ webmention.reset_for_revision(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -235,7 +239,7 @@ def enqueue_due_sends(scan_time: datetime) -> int:
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
.where(
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= scan_time,
@@ -273,7 +277,9 @@ def _scan_and_enqueue() -> None:
sources_by_path = {
source.path: source
for source in db.session.scalars(
- sa.select(Source).options(selectinload(Source.sent_webmentions))
+ sa.select(SourceDocument).options(
+ selectinload(SourceDocument.sent_webmentions)
+ )
)
}
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index c33cf06..3f64bed 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -24,7 +24,7 @@ from ..net import (
is_retryable_status,
)
from ..version import APP_NAME, VERSION
-from .discovery import (
+from .endpoint_discovery import (
PermanentDeliveryError,
RetryableDeliveryError,
discover_webmention_endpoint,
@@ -44,7 +44,7 @@ def revision_still_pending(webmention: SentWebmention, revision: int) -> bool:
"""
db.session.refresh(webmention)
- return webmention.desired_revision == revision and webmention.pending
+ return webmention.desired_revision == revision and webmention.awaiting_processing
def backoff_delay(attempt: int) -> timedelta:
@@ -99,8 +99,8 @@ class ClaimedSendAttempt:
class DeliveryResult:
"""HTTP outcome, with whether the failure is eligible for another attempt.
- ``terminal`` describes the failure type. The retry budget can also make a
- temporary failure terminal when its last allowed attempt has been used.
+ ``retryable`` describes the failure type; it does not account for the
+ remaining retry budget, which may already have been exhausted.
"""
status: SentWebmentionStatus
@@ -108,8 +108,7 @@ class DeliveryResult:
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
- terminal: bool = True
- temporary_failure: bool = False
+ retryable: bool = False
def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
@@ -119,7 +118,7 @@ def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
:param now: Current timestamp against which the retry deadline is compared.
:returns: ``True`` if the Webmention is pending and its retry deadline has elapsed.
"""
- if not webmention.pending:
+ if not webmention.awaiting_processing:
return False
deadline = webmention.next_retry_at
@@ -154,7 +153,7 @@ def reserve_send_attempt(
SentWebmention.uuid == webmention.uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count >= max_attempts,
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= now,
@@ -188,7 +187,7 @@ def reserve_send_attempt(
SentWebmention.uuid == attempt.webmention_uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count == attempt.attempt_number - 1,
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= now,
@@ -198,8 +197,8 @@ def reserve_send_attempt(
attempt_count=attempt.attempt_number,
last_attempted_at=now,
next_retry_at=now + backoff_delay(attempt.attempt_number),
- endpoint=None,
- response_status=None,
+ webmention_endpoint=None,
+ endpoint_response_status=None,
status_url=None,
)
.execution_options(synchronize_session=False)
@@ -236,7 +235,7 @@ def attempt_delivery(
if endpoint is None:
return DeliveryResult(
- status=SentWebmentionStatus.UNSUPPORTED,
+ status=SentWebmentionStatus.NO_ENDPOINT,
reason="No Webmention endpoint discovered",
)
@@ -275,11 +274,10 @@ def attempt_delivery(
reason=str(exc) or "Webmention request failed",
endpoint=endpoint,
response_status=response_status,
- terminal=False,
- temporary_failure=True,
+ retryable=True,
)
except Exception as exc:
- # Unexpected errors should be terminal and keep their traceback.
+ # Unexpected errors should not be retried; keep their traceback.
current_app.logger.exception(
"Unexpected error sending SentWebmention %s", webmention_uuid
)
@@ -312,8 +310,8 @@ def record_delivery_outcome(
webmention.status = result.status
webmention.failure_reason = result.reason
- webmention.endpoint = result.endpoint
- webmention.response_status = result.response_status
+ webmention.webmention_endpoint = result.endpoint
+ webmention.endpoint_response_status = result.response_status
if terminal:
webmention.processed_revision = attempt.revision
@@ -355,19 +353,19 @@ def _process_send_locked(webmention_uuid: uuid.UUID) -> bool:
result = attempt_delivery(
attempt.source, attempt.target, webmention_uuid=webmention_uuid
)
- terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts
+ terminal = not result.retryable or attempt.attempt_number >= attempt.max_attempts
if not record_delivery_outcome(webmention, attempt, result, terminal=terminal):
return False
- if result.temporary_failure and not terminal:
+ if result.retryable and not terminal:
# Huey does not retry: the scanner requeues after next_retry_at.
current_app.logger.info(
"Webmention %s send attempt %d failed; will retry",
webmention_uuid,
attempt.attempt_number,
)
- elif result.temporary_failure:
+ elif result.retryable:
current_app.logger.warning(
"Webmention %s exhausted %d send attempts: %s",
webmention_uuid,
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/verification.py
index 313a8e4..a614815 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/verification.py
@@ -15,7 +15,7 @@ from flask import current_app
from ..db_helpers import rows_affected
from ..extensions import db, huey
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
from ..net import (
AddressResolutionError,
ResponseTooLargeError,
@@ -149,11 +149,14 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
)
return None
- if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
+ if webmention.status not in {
+ ReceivedWebmentionStatus.RECEIVED,
+ ReceivedWebmentionStatus.FAILED,
+ }:
return None
if (
- webmention.status == ReceivedStatus.FAILED
+ webmention.status == ReceivedWebmentionStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return None
@@ -172,7 +175,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
ReceivedWebmention.updated_at == webmention.updated_at,
)
.values(
- status=ReceivedStatus.VERIFYING,
+ status=ReceivedWebmentionStatus.VERIFYING,
failure_reason=None,
updated_at=claim.lease_started_at,
)
@@ -188,7 +191,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
def determine_verification_outcome(
claim: VerificationClaim,
-) -> tuple[ReceivedStatus, str | None]:
+) -> tuple[ReceivedWebmentionStatus, str | None]:
"""Map verification results and permanent errors to a final status/reason.
:param claim: Claim containing the source and target URLs to verify.
@@ -200,9 +203,9 @@ def determine_verification_outcome(
try:
mentions_target = source_mentions_target(claim.source, claim.target)
except SourceGoneError as exc:
- return ReceivedStatus.DELETED, str(exc)
+ return ReceivedWebmentionStatus.SOURCE_REMOVED, str(exc)
except VerificationError as exc:
- return ReceivedStatus.FAILED, str(exc)
+ return ReceivedWebmentionStatus.FAILED, str(exc)
except RetryableFetchError, httpx.RequestError:
raise # These errors must be persisted and then retried by Huey.
except Exception as exc:
@@ -210,28 +213,28 @@ def determine_verification_outcome(
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid
)
- return ReceivedStatus.FAILED, str(exc) or type(exc).__name__
+ return ReceivedWebmentionStatus.FAILED, str(exc) or type(exc).__name__
if mentions_target:
- return ReceivedStatus.VERIFIED, None
- return ReceivedStatus.DELETED, "Source does not mention target"
+ return ReceivedWebmentionStatus.VERIFIED, None
+ return ReceivedWebmentionStatus.SOURCE_REMOVED, "Source does not mention target"
-def release_claim(
- claim: VerificationClaim, status: ReceivedStatus, reason: str | None
+def store_verification_outcome(
+ claim: VerificationClaim, status: ReceivedWebmentionStatus, reason: str | None
) -> bool:
"""Persist an outcome only if this worker still owns the verification lease.
:param claim: Verification lease acquired by this worker.
:param status: Final status to persist for the claimed Webmention.
- :param reason: Optional explanation for a failed or deleted Webmention.
+ :param reason: Optional explanation for a failed or source-removed Webmention.
:returns: ``True`` if the worker still owned the lease and persisted the outcome.
"""
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == claim.webmention_uuid,
- ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFYING,
ReceivedWebmention.updated_at == claim.lease_started_at,
)
.values(status=status, failure_reason=reason, updated_at=datetime.now(UTC))
@@ -259,12 +262,14 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
try:
status, reason = determine_verification_outcome(claim)
except (RetryableFetchError, httpx.RequestError) as exc:
- release_claim(
- claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched"
+ store_verification_outcome(
+ claim,
+ ReceivedWebmentionStatus.FAILED,
+ str(exc) or "Source could not be fetched",
)
raise
- release_claim(claim, status, reason)
+ store_verification_outcome(claim, status, reason)
def recover_stale_verifications() -> int:
@@ -296,7 +301,7 @@ def recover_stale_verifications() -> int:
ReceivedWebmention.recovery_attempts,
).where(
ReceivedWebmention.status.in_(
- (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ (ReceivedWebmentionStatus.VERIFYING, ReceivedWebmentionStatus.RECEIVED)
),
ReceivedWebmention.updated_at <= cutoff,
)
@@ -319,14 +324,14 @@ def recover_stale_verifications() -> int:
.values(
**(
{
- "status": ReceivedStatus.FAILED,
+ "status": ReceivedWebmentionStatus.FAILED,
"failure_reason": RECOVERY_EXHAUSTED_REASON,
"updated_at": now,
"recovery_attempts": recovery_attempts,
}
if at_limit
- else ReceivedWebmention.verification_reset_values(
- at=now, recoveries=recovery_attempts + 1
+ else ReceivedWebmention.reopen_verification_values(
+ at=now, recovery_attempts=recovery_attempts + 1
)
)
)