aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
authorDennis Fink2026-11-15 21:18:19 +0100
committerDennis Fink2026-11-15 21:18:19 +0100
commit9b00526ab79257b65cedd71595848f09ecf84279 (patch)
treef9dfc5c0934499d47499bf083bd864f72947958c /webmentions_ssg/tasks
parent509d93b2976e353d29441a7a7b7e78ca466dc4fd (diff)
downloadwebmentions-ssg-9b00526ab79257b65cedd71595848f09ecf84279.tar.gz
webmentions-ssg-9b00526ab79257b65cedd71595848f09ecf84279.zip
refactor(core)!: clarify Webmention domain terminology
Rename ambiguous models, statuses, task modules, fields, and helpers so names reflect source documents, verification, endpoint discovery, and delivery state. Migrate persisted status values and database identifiers without losing existing Webmention data. BREAKING CHANGE: persisted status values and database schema names have changed, and several public Python names have been renamed.
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/__init__.py4
-rw-r--r--webmentions_ssg/tasks/consumer.py12
-rw-r--r--webmentions_ssg/tasks/endpoint_discovery.py (renamed from webmentions_ssg/tasks/discovery.py)0
-rw-r--r--webmentions_ssg/tasks/extension.py6
-rw-r--r--webmentions_ssg/tasks/notifications.py6
-rw-r--r--webmentions_ssg/tasks/retention.py7
-rw-r--r--webmentions_ssg/tasks/scanner.py38
-rw-r--r--webmentions_ssg/tasks/sender.py38
-rw-r--r--webmentions_ssg/tasks/verification.py (renamed from webmentions_ssg/tasks/receiver.py)47
9 files changed, 84 insertions, 74 deletions
diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py
index c744b49..37ab4d2 100644
--- a/webmentions_ssg/tasks/__init__.py
+++ b/webmentions_ssg/tasks/__init__.py
@@ -4,6 +4,6 @@
"""Expose the background task queue integration."""
-from .extension import Huey
+from .extension import FlaskHuey
-__all__ = ["Huey"]
+__all__ = ["FlaskHuey"]
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index c24a883..7991b6a 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -8,11 +8,11 @@ from huey import Huey, crontab
from .. import create_app
from ..extensions import huey as queue
-from . import notifications, receiver, retention, scanner
+from . import notifications, retention, scanner, verification
-def create_consumer() -> Huey:
- """Create and configure the Huey consumer.
+def create_huey() -> Huey:
+ """Create and configure the Huey instance for the task worker.
The application is initialized and the periodic scanner, notification,
verification recovery and retention tasks are registered.
@@ -23,8 +23,8 @@ def create_consumer() -> Huey:
for config_key, task in (
("SOURCE_SCAN_SCHEDULE", scanner.scan_sources),
- ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications),
- ("DIGEST_SCHEDULE", notifications.send_webmention_digest),
+ ("VERIFICATION_RECOVERY_SCHEDULE", verification.recover_stale_verifications),
+ ("DIGEST_SCHEDULE", notifications.email_verified_digest),
("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions),
):
schedule = crontab(*app.config[config_key].split(), strict=True)
@@ -33,4 +33,4 @@ def create_consumer() -> Huey:
return queue.huey
-huey = create_consumer()
+huey = create_huey()
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/endpoint_discovery.py
index c683cb0..c683cb0 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/endpoint_discovery.py
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index faed364..3f172f4 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -4,8 +4,6 @@
"""Integrate Huey tasks with Flask application contexts."""
-from __future__ import annotations
-
from collections.abc import Callable
from functools import update_wrapper, wraps
from typing import Any
@@ -100,7 +98,7 @@ class _DeferredTask:
def __init__(
self,
- extension: Huey,
+ extension: FlaskHuey,
func: Callable[..., Any],
args: tuple[Any, ...],
kwargs: dict[str, Any],
@@ -152,7 +150,7 @@ class _DeferredTask:
return getattr(self._bound_task(), name)
-class Huey:
+class FlaskHuey:
"""Provide Flask integration for a Huey instance.
The extension initializes a Huey backend from the Flask configuration and
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index eb540de..c503e1f 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -11,10 +11,10 @@ from flask import current_app
from flask_mail import Message
from ..extensions import db, huey, mail
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
-def send_webmention_digest() -> None:
+def email_verified_digest() -> None:
"""Send one digest of verified, previously unreported Webmentions.
Acquire the digest lock before notifying recipients about previously unreported
@@ -38,7 +38,7 @@ def _send_digest_locked() -> None:
cutoff = datetime.now(UTC)
conditions = (
- ReceivedWebmention.status == ReceivedStatus.VERIFIED,
+ ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFIED,
ReceivedWebmention.notification_sent_at.is_(None),
ReceivedWebmention.updated_at <= cutoff,
)
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 2c35cc7..b882493 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -11,7 +11,7 @@ from flask import current_app
from ..db_helpers import rows_affected
from ..extensions import db
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
def purge_old_received_webmentions() -> int:
@@ -33,7 +33,10 @@ def purge_old_received_webmentions() -> int:
result = db.session.execute(
sa.delete(ReceivedWebmention).where(
ReceivedWebmention.status.in_(
- (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ (
+ ReceivedWebmentionStatus.FAILED,
+ ReceivedWebmentionStatus.SOURCE_REMOVED,
+ )
),
ReceivedWebmention.updated_at < cutoff,
)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 034b2cf..fb0c770 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -14,13 +14,13 @@ from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
from ..extensions import db, huey
-from ..models import SentWebmention, Source
+from ..models import SentWebmention, SourceDocument
from ..net import is_http_url
from .sender import send_webmention
from .source_parser import ScannedSource, SourceScanError, scan_source_file
-def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
+def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument:
"""Build a source model from a newly discovered document.
A sent Webmention is built for each discovered target at the initial source
@@ -30,7 +30,7 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
:param scan_time: Time at which the source was discovered.
:returns: Newly built source model.
"""
- source = Source(
+ source = SourceDocument(
path=scanned.path,
url=scanned.url,
content_hash=scanned.content_hash,
@@ -42,15 +42,15 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
for target in scanned.targets:
source.sent_webmentions.append(
- SentWebmention(target=target, active=True, desired_revision=1)
+ SentWebmention(target=target, linked=True, desired_revision=1)
)
return source
def apply_scan_to_source(
- source: Source, scanned: ScannedSource, scan_time: datetime
-) -> Source:
+ source: SourceDocument, scanned: ScannedSource, scan_time: datetime
+) -> SourceDocument:
"""Update an existing source from newly scanned data.
A changed or previously deleted source receives a new revision. Existing sent
@@ -82,16 +82,18 @@ def apply_scan_to_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.reopen(source.revision, webmention.target in scanned.targets)
+ webmention.reset_for_revision(
+ source.revision, webmention.target in scanned.targets
+ )
- if not webmention.active and webmention.sent_revision is None:
+ if not webmention.linked and webmention.sent_revision is None:
webmention.processed_revision = source.revision
for target in scanned.targets:
if target not in existing:
source.sent_webmentions.append(
SentWebmention(
- target=target, active=True, desired_revision=source.revision
+ target=target, linked=True, desired_revision=source.revision
)
)
@@ -127,7 +129,7 @@ class TreeScanResult:
def reconcile_source_tree(
root: Path,
base_url: str | None,
- sources_by_path: dict[str, Source],
+ sources_by_path: dict[str, SourceDocument],
paths_by_url: dict[str, str],
scan_time: datetime,
) -> TreeScanResult:
@@ -197,14 +199,16 @@ def reconcile_source_tree(
def mark_missing_sources_deleted(
- sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
+ sources_by_path: dict[str, SourceDocument],
+ seen_paths: frozenset[str],
+ scan_time: datetime,
) -> int:
- """Mark absent, previously active sources deleted and return their count.
+ """Mark absent, previously present sources deleted and return their count.
:param sources_by_path: Known source records indexed by relative path.
:param seen_paths: Paths found during this scan, including failed parses.
:param scan_time: Timestamp to record for sources newly marked missing.
- :returns: Number of previously active sources marked as deleted.
+ :returns: Number of previously linked sources marked as deleted.
"""
marked = 0
for path, source in sources_by_path.items():
@@ -217,7 +221,7 @@ def mark_missing_sources_deleted(
marked += 1
for webmention in source.sent_webmentions:
- webmention.reopen(source.revision, False)
+ webmention.reset_for_revision(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -235,7 +239,7 @@ def enqueue_due_sends(scan_time: datetime) -> int:
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
.where(
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= scan_time,
@@ -273,7 +277,9 @@ def _scan_and_enqueue() -> None:
sources_by_path = {
source.path: source
for source in db.session.scalars(
- sa.select(Source).options(selectinload(Source.sent_webmentions))
+ sa.select(SourceDocument).options(
+ selectinload(SourceDocument.sent_webmentions)
+ )
)
}
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index c33cf06..3f64bed 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -24,7 +24,7 @@ from ..net import (
is_retryable_status,
)
from ..version import APP_NAME, VERSION
-from .discovery import (
+from .endpoint_discovery import (
PermanentDeliveryError,
RetryableDeliveryError,
discover_webmention_endpoint,
@@ -44,7 +44,7 @@ def revision_still_pending(webmention: SentWebmention, revision: int) -> bool:
"""
db.session.refresh(webmention)
- return webmention.desired_revision == revision and webmention.pending
+ return webmention.desired_revision == revision and webmention.awaiting_processing
def backoff_delay(attempt: int) -> timedelta:
@@ -99,8 +99,8 @@ class ClaimedSendAttempt:
class DeliveryResult:
"""HTTP outcome, with whether the failure is eligible for another attempt.
- ``terminal`` describes the failure type. The retry budget can also make a
- temporary failure terminal when its last allowed attempt has been used.
+ ``retryable`` describes the failure type; it does not account for the
+ remaining retry budget, which may already have been exhausted.
"""
status: SentWebmentionStatus
@@ -108,8 +108,7 @@ class DeliveryResult:
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
- terminal: bool = True
- temporary_failure: bool = False
+ retryable: bool = False
def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
@@ -119,7 +118,7 @@ def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
:param now: Current timestamp against which the retry deadline is compared.
:returns: ``True`` if the Webmention is pending and its retry deadline has elapsed.
"""
- if not webmention.pending:
+ if not webmention.awaiting_processing:
return False
deadline = webmention.next_retry_at
@@ -154,7 +153,7 @@ def reserve_send_attempt(
SentWebmention.uuid == webmention.uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count >= max_attempts,
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= now,
@@ -188,7 +187,7 @@ def reserve_send_attempt(
SentWebmention.uuid == attempt.webmention_uuid,
SentWebmention.desired_revision == revision,
SentWebmention.attempt_count == attempt.attempt_number - 1,
- SentWebmention.pending,
+ SentWebmention.awaiting_processing,
sa.or_(
SentWebmention.next_retry_at.is_(None),
SentWebmention.next_retry_at <= now,
@@ -198,8 +197,8 @@ def reserve_send_attempt(
attempt_count=attempt.attempt_number,
last_attempted_at=now,
next_retry_at=now + backoff_delay(attempt.attempt_number),
- endpoint=None,
- response_status=None,
+ webmention_endpoint=None,
+ endpoint_response_status=None,
status_url=None,
)
.execution_options(synchronize_session=False)
@@ -236,7 +235,7 @@ def attempt_delivery(
if endpoint is None:
return DeliveryResult(
- status=SentWebmentionStatus.UNSUPPORTED,
+ status=SentWebmentionStatus.NO_ENDPOINT,
reason="No Webmention endpoint discovered",
)
@@ -275,11 +274,10 @@ def attempt_delivery(
reason=str(exc) or "Webmention request failed",
endpoint=endpoint,
response_status=response_status,
- terminal=False,
- temporary_failure=True,
+ retryable=True,
)
except Exception as exc:
- # Unexpected errors should be terminal and keep their traceback.
+ # Unexpected errors should not be retried; keep their traceback.
current_app.logger.exception(
"Unexpected error sending SentWebmention %s", webmention_uuid
)
@@ -312,8 +310,8 @@ def record_delivery_outcome(
webmention.status = result.status
webmention.failure_reason = result.reason
- webmention.endpoint = result.endpoint
- webmention.response_status = result.response_status
+ webmention.webmention_endpoint = result.endpoint
+ webmention.endpoint_response_status = result.response_status
if terminal:
webmention.processed_revision = attempt.revision
@@ -355,19 +353,19 @@ def _process_send_locked(webmention_uuid: uuid.UUID) -> bool:
result = attempt_delivery(
attempt.source, attempt.target, webmention_uuid=webmention_uuid
)
- terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts
+ terminal = not result.retryable or attempt.attempt_number >= attempt.max_attempts
if not record_delivery_outcome(webmention, attempt, result, terminal=terminal):
return False
- if result.temporary_failure and not terminal:
+ if result.retryable and not terminal:
# Huey does not retry: the scanner requeues after next_retry_at.
current_app.logger.info(
"Webmention %s send attempt %d failed; will retry",
webmention_uuid,
attempt.attempt_number,
)
- elif result.temporary_failure:
+ elif result.retryable:
current_app.logger.warning(
"Webmention %s exhausted %d send attempts: %s",
webmention_uuid,
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/verification.py
index 313a8e4..a614815 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/verification.py
@@ -15,7 +15,7 @@ from flask import current_app
from ..db_helpers import rows_affected
from ..extensions import db, huey
-from ..models import ReceivedStatus, ReceivedWebmention
+from ..models import ReceivedWebmention, ReceivedWebmentionStatus
from ..net import (
AddressResolutionError,
ResponseTooLargeError,
@@ -149,11 +149,14 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
)
return None
- if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
+ if webmention.status not in {
+ ReceivedWebmentionStatus.RECEIVED,
+ ReceivedWebmentionStatus.FAILED,
+ }:
return None
if (
- webmention.status == ReceivedStatus.FAILED
+ webmention.status == ReceivedWebmentionStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return None
@@ -172,7 +175,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
ReceivedWebmention.updated_at == webmention.updated_at,
)
.values(
- status=ReceivedStatus.VERIFYING,
+ status=ReceivedWebmentionStatus.VERIFYING,
failure_reason=None,
updated_at=claim.lease_started_at,
)
@@ -188,7 +191,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No
def determine_verification_outcome(
claim: VerificationClaim,
-) -> tuple[ReceivedStatus, str | None]:
+) -> tuple[ReceivedWebmentionStatus, str | None]:
"""Map verification results and permanent errors to a final status/reason.
:param claim: Claim containing the source and target URLs to verify.
@@ -200,9 +203,9 @@ def determine_verification_outcome(
try:
mentions_target = source_mentions_target(claim.source, claim.target)
except SourceGoneError as exc:
- return ReceivedStatus.DELETED, str(exc)
+ return ReceivedWebmentionStatus.SOURCE_REMOVED, str(exc)
except VerificationError as exc:
- return ReceivedStatus.FAILED, str(exc)
+ return ReceivedWebmentionStatus.FAILED, str(exc)
except RetryableFetchError, httpx.RequestError:
raise # These errors must be persisted and then retried by Huey.
except Exception as exc:
@@ -210,28 +213,28 @@ def determine_verification_outcome(
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid
)
- return ReceivedStatus.FAILED, str(exc) or type(exc).__name__
+ return ReceivedWebmentionStatus.FAILED, str(exc) or type(exc).__name__
if mentions_target:
- return ReceivedStatus.VERIFIED, None
- return ReceivedStatus.DELETED, "Source does not mention target"
+ return ReceivedWebmentionStatus.VERIFIED, None
+ return ReceivedWebmentionStatus.SOURCE_REMOVED, "Source does not mention target"
-def release_claim(
- claim: VerificationClaim, status: ReceivedStatus, reason: str | None
+def store_verification_outcome(
+ claim: VerificationClaim, status: ReceivedWebmentionStatus, reason: str | None
) -> bool:
"""Persist an outcome only if this worker still owns the verification lease.
:param claim: Verification lease acquired by this worker.
:param status: Final status to persist for the claimed Webmention.
- :param reason: Optional explanation for a failed or deleted Webmention.
+ :param reason: Optional explanation for a failed or source-removed Webmention.
:returns: ``True`` if the worker still owned the lease and persisted the outcome.
"""
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == claim.webmention_uuid,
- ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFYING,
ReceivedWebmention.updated_at == claim.lease_started_at,
)
.values(status=status, failure_reason=reason, updated_at=datetime.now(UTC))
@@ -259,12 +262,14 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
try:
status, reason = determine_verification_outcome(claim)
except (RetryableFetchError, httpx.RequestError) as exc:
- release_claim(
- claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched"
+ store_verification_outcome(
+ claim,
+ ReceivedWebmentionStatus.FAILED,
+ str(exc) or "Source could not be fetched",
)
raise
- release_claim(claim, status, reason)
+ store_verification_outcome(claim, status, reason)
def recover_stale_verifications() -> int:
@@ -296,7 +301,7 @@ def recover_stale_verifications() -> int:
ReceivedWebmention.recovery_attempts,
).where(
ReceivedWebmention.status.in_(
- (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ (ReceivedWebmentionStatus.VERIFYING, ReceivedWebmentionStatus.RECEIVED)
),
ReceivedWebmention.updated_at <= cutoff,
)
@@ -319,14 +324,14 @@ def recover_stale_verifications() -> int:
.values(
**(
{
- "status": ReceivedStatus.FAILED,
+ "status": ReceivedWebmentionStatus.FAILED,
"failure_reason": RECOVERY_EXHAUSTED_REASON,
"updated_at": now,
"recovery_attempts": recovery_attempts,
}
if at_limit
- else ReceivedWebmention.verification_reset_values(
- at=now, recoveries=recovery_attempts + 1
+ else ReceivedWebmention.reopen_verification_values(
+ at=now, recovery_attempts=recovery_attempts + 1
)
)
)