aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/consumer.py6
-rw-r--r--webmentions_ssg/tasks/discovery.py62
-rw-r--r--webmentions_ssg/tasks/extension.py58
-rw-r--r--webmentions_ssg/tasks/json_serializer.py2
-rw-r--r--webmentions_ssg/tasks/notifications.py6
-rw-r--r--webmentions_ssg/tasks/receiver.py72
-rw-r--r--webmentions_ssg/tasks/scanner.py48
-rw-r--r--webmentions_ssg/tasks/sender.py78
-rw-r--r--webmentions_ssg/tasks/source_parser.py53
-rw-r--r--webmentions_ssg/tasks/target_matching.py (renamed from webmentions_ssg/tasks/mentions.py)0
10 files changed, 202 insertions, 183 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 13334bc..c24a883 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -22,9 +22,9 @@ def create_consumer() -> Huey:
app = create_app()
for config_key, task in (
- ("SCANNER_SCHEDULE", scanner.scan_sources),
- ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions),
- ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest),
+ ("SOURCE_SCAN_SCHEDULE", scanner.scan_sources),
+ ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications),
+ ("DIGEST_SCHEDULE", notifications.send_webmention_digest),
("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions),
):
schedule = crontab(*app.config[config_key].split(), strict=True)
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index 6d01e53..c683cb0 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -14,48 +14,48 @@ from flask import current_app
from ..net import (
ResponseTooLargeError,
- content_type_of,
is_http_url,
- is_temporary_status,
+ is_retryable_status,
+ media_type_of,
read_limited_body,
)
-LINK_SPLIT = re.compile(r",\s*(?=<)")
+LINK_HEADER_SPLIT = re.compile(r",\s*(?=<)")
DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
-class SenderError(Exception):
+class DeliveryError(Exception):
"""A Webmention cannot be sent."""
-class TemporarySenderError(SenderError):
+class RetryableDeliveryError(DeliveryError):
"""A Webmention send failed for a potentially temporary reason."""
-class PermanentSenderError(SenderError):
+class PermanentDeliveryError(DeliveryError):
"""A Webmention send failed permanently for this source revision."""
-def resolve_endpoint(response: httpx.Response, href: str) -> str:
+def resolve_endpoint_url(response: httpx.Response, href: str) -> str:
"""Resolve and validate a discovered Webmention endpoint.
:param response: Response from which the endpoint was discovered.
:param href: Endpoint reference to resolve.
:returns: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL.
+ :raises PermanentDeliveryError: If the resolved endpoint is not an HTTP or HTTPS URL.
"""
try:
endpoint = urljoin(str(response.url), href.strip())
except ValueError as exc:
- raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
+ raise PermanentDeliveryError(f"Invalid Webmention endpoint: {href!r}") from exc
if not is_http_url(endpoint):
- raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
+ raise PermanentDeliveryError(f"Invalid Webmention endpoint: {endpoint!r}")
return endpoint
-def parse_link_value(value: str) -> tuple[str, set[str]] | None:
+def parse_link_header_value(value: str) -> tuple[str, set[str]] | None:
"""Parse a Link header value into its target and relations.
:param value: Link header value to parse.
@@ -84,7 +84,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None:
return href, relations
-def endpoint_from_headers(response: httpx.Response) -> str | None:
+def find_endpoint_in_headers(response: httpx.Response) -> str | None:
"""Find a Webmention endpoint in the response Link headers.
:param response: HTTP response whose headers should be inspected.
@@ -92,22 +92,22 @@ def endpoint_from_headers(response: httpx.Response) -> str | None:
Invalid candidates are ignored.
"""
for header in response.headers.get_list("Link"):
- for value in LINK_SPLIT.split(header):
- if (link := parse_link_value(value)) is None:
+ for value in LINK_HEADER_SPLIT.split(header):
+ if (link := parse_link_header_value(value)) is None:
continue
href, relations = link
if "webmention" in relations:
try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
+ return resolve_endpoint_url(response, href)
+ except PermanentDeliveryError:
continue
return None
-def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
+def find_endpoint_in_html(response: httpx.Response, body: bytes) -> str | None:
"""Find a Webmention endpoint in an HTML document.
``link`` and ``a`` elements are inspected in document order.
@@ -123,8 +123,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
'link[rel~="webmention" i][href], a[rel~="webmention" i][href]'
):
try:
- return resolve_endpoint(response, str(element["href"]))
- except PermanentSenderError:
+ return resolve_endpoint_url(response, str(element["href"]))
+ except PermanentDeliveryError:
continue
return None
@@ -135,14 +135,14 @@ def read_target_body(response: httpx.Response) -> bytes:
:param response: HTTP response to read with the configured target-size limit.
:returns: Response body, provided it is within the target-size limit.
- :raises PermanentSenderError: If the target document is too large.
+ :raises PermanentDeliveryError: If the target document is too large.
"""
try:
return read_limited_body(
- response, current_app.config["MAX_TARGET_BYTES"], label="Target"
+ response, current_app.config["MAX_DISCOVERY_TARGET_BYTES"], label="Target"
)
except ResponseTooLargeError as exc:
- raise PermanentSenderError(str(exc)) from exc
+ raise PermanentDeliveryError(str(exc)) from exc
def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
@@ -155,8 +155,8 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
:param client: HTTP client to use for discovery requests.
:param target: Target URL whose Webmention endpoint should be discovered.
:returns: Discovered Webmention endpoint, or ``None`` if none is advertised.
- :raises TemporarySenderError: If the target returns a temporary HTTP failure.
- :raises PermanentSenderError: If GET returns a permanent HTTP failure.
+ :raises RetryableDeliveryError: If the target returns a temporary HTTP failure.
+ :raises PermanentDeliveryError: If GET returns a permanent HTTP failure.
"""
try:
head_response = client.head(target, headers=DISCOVERY_HEADERS)
@@ -165,25 +165,25 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
# or drop it even when fetching the document with GET would succeed.
pass
else:
- if endpoint := endpoint_from_headers(head_response):
+ if endpoint := find_endpoint_in_headers(head_response):
return endpoint
with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
- if endpoint := endpoint_from_headers(response):
+ if endpoint := find_endpoint_in_headers(response):
return endpoint
- if is_temporary_status(response.status_code):
- raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
+ if is_retryable_status(response.status_code):
+ raise RetryableDeliveryError(f"Target returned HTTP {response.status_code}")
if not response.is_success:
- raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
+ raise PermanentDeliveryError(f"Target returned HTTP {response.status_code}")
- media_type = content_type_of(response.headers)
+ media_type = media_type_of(response.headers)
if media_type not in {"text/html", "application/xhtml+xml"}:
return None
- return endpoint_from_html(response, read_target_body(response))
+ return find_endpoint_in_html(response, read_target_body(response))
def post_webmention(
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index 58e0036..faed364 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -15,10 +15,12 @@ from flask import Flask
from huey import Huey as BaseHuey
from huey.api import TaskWrapper
-from .json_serializer import JSONSerializer
+from .json_serializer import TaggedJSONSerializer
-def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]:
+def _redis_class_and_options(
+ parsed: SplitResult,
+) -> tuple[type[BaseHuey], dict[str, Any]]:
"""Select a Redis Huey flavour while retaining the underlying URL scheme.
:param parsed: Parsed Redis backend URL, including any flavour suffix.
@@ -47,7 +49,9 @@ def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]
return backends[flavour], {"url": fixed_url}
-def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]:
+def _file_class_and_options(
+ url: str, scheme: str
+) -> tuple[type[BaseHuey], dict[str, Any]]:
"""Extract a relative or absolute SQLite/file storage path.
:param url: Backend URL containing the storage filename or path.
@@ -86,7 +90,7 @@ def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]
return backend, {option: path}
-class _LazyTask:
+class _DeferredTask:
"""Stable public task handle which follows the extension's current backend.
Huey creates task classes and registers them when its decorator is applied.
@@ -111,7 +115,7 @@ class _LazyTask:
self._registered: TaskWrapper | None = None
update_wrapper(self, func)
- def register(self) -> None:
+ def bind(self) -> None:
"""Bind the original function to the latest initialized backend.
Register the task decorator again so its stable handle uses the currently
@@ -120,10 +124,10 @@ class _LazyTask:
backend = self._extension.huey
decorator = backend.periodic_task if self._periodic else backend.task
self._registered = decorator(*self._args, **self._kwargs)(
- self._extension._in_app_context(self._func)
+ self._extension._with_app_context(self._func)
)
- def _current(self) -> TaskWrapper:
+ def _bound_task(self) -> TaskWrapper:
if self._registered is None:
raise RuntimeError(
"Huey has not been initialized. Call huey.init_app(app) first."
@@ -137,7 +141,7 @@ class _LazyTask:
:param kwargs: Keyword arguments passed to the queued task.
:returns: Result of enqueueing the task on the current backend.
"""
- return self._current()(*args, **kwargs)
+ return self._bound_task()(*args, **kwargs)
def __getattr__(self, name: str) -> Any:
"""Delegate Huey task operations (call_local, schedule, s, etc.).
@@ -145,7 +149,7 @@ class _LazyTask:
:param name: Huey task attribute to delegate.
:returns: Requested attribute of the currently bound Huey task.
"""
- return getattr(self._current(), name)
+ return getattr(self._bound_task(), name)
class Huey:
@@ -162,7 +166,7 @@ class Huey:
"""
self.app: Flask | None = None
self._huey: BaseHuey | None = None
- self._tasks: list[_LazyTask] = []
+ self._task_handles: list[_DeferredTask] = []
if app is not None:
self.init_app(app)
@@ -192,7 +196,7 @@ class Huey:
raise RuntimeError(
"HUEY.serializer cannot be overridden; JSON is required."
)
- config["serializer"] = JSONSerializer(
+ config["serializer"] = TaggedJSONSerializer(
compression=config.pop("compression", False),
use_zlib=config.pop("use_zlib", False),
)
@@ -205,8 +209,8 @@ class Huey:
# Import order is immaterial: tasks declared before initialization are
# registered now, and previously imported handles follow this backend.
- for task in self._tasks:
- task.register()
+ for task in self._task_handles:
+ task.bind()
app.extensions["huey"] = self
@@ -224,7 +228,7 @@ class Huey:
)
return self._huey
- def _in_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]:
+ def _with_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]:
"""Return a task wrapper that executes with the configured Flask app.
:param func: Task function to wrap in a Flask application context.
@@ -241,23 +245,23 @@ class Huey:
return wrapped
- def _decorate_task[**P, R](
+ def _register_task[**P, R](
self,
func: Callable[P, R],
task_args: tuple[Any, ...],
task_kwargs: dict[str, Any],
*,
periodic: bool,
- ) -> _LazyTask:
- handle = _LazyTask(self, func, task_args, task_kwargs, periodic=periodic)
- self._tasks.append(handle)
+ ) -> _DeferredTask:
+ handle = _DeferredTask(self, func, task_args, task_kwargs, periodic=periodic)
+ self._task_handles.append(handle)
if self._huey is not None:
- handle.register()
+ handle.bind()
return handle
def task[**P, R](
self, *task_args: Any, **task_kwargs: Any
- ) -> Callable[[Callable[P, R]], _LazyTask]:
+ ) -> Callable[[Callable[P, R]], _DeferredTask]:
"""Declare a task even before :meth:`init_app` has run.
:param task_args: Positional arguments for the Huey task decorator.
@@ -265,14 +269,14 @@ class Huey:
:returns: Decorator that registers and returns a lazily bound task.
"""
- def decorator(func: Callable[P, R]) -> _LazyTask:
- return self._decorate_task(func, task_args, task_kwargs, periodic=False)
+ def decorator(func: Callable[P, R]) -> _DeferredTask:
+ return self._register_task(func, task_args, task_kwargs, periodic=False)
return decorator
def periodic_task[**P, R](
self, *task_args: Any, **task_kwargs: Any
- ) -> Callable[[Callable[P, R]], _LazyTask]:
+ ) -> Callable[[Callable[P, R]], _DeferredTask]:
"""Declare a periodic task against the current or next backend.
:param task_args: Positional arguments for the Huey periodic-task decorator.
@@ -280,8 +284,8 @@ class Huey:
:returns: Decorator that registers and returns a lazily bound periodic task.
"""
- def decorator(func: Callable[P, R]) -> _LazyTask:
- return self._decorate_task(func, task_args, task_kwargs, periodic=True)
+ def decorator(func: Callable[P, R]) -> _DeferredTask:
+ return self._register_task(func, task_args, task_kwargs, periodic=True)
return decorator
@@ -309,9 +313,9 @@ class Huey:
scheme = parsed.scheme.lower()
if scheme.partition("+")[0] in {"redis", "rediss"}:
- return _redis_backend(parsed)
+ return _redis_class_and_options(parsed)
if scheme in {"sqlite", "file"}:
- return _path_backend(url, scheme)
+ return _file_class_and_options(url, scheme)
if scheme in {"postgres", "postgresql"}:
from huey import PostgresHuey
diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py
index 7e51358..804cfee 100644
--- a/webmentions_ssg/tasks/json_serializer.py
+++ b/webmentions_ssg/tasks/json_serializer.py
@@ -14,7 +14,7 @@ from huey.serializer import Serializer
from huey.utils import SKIPPED, Error
-class JSONSerializer(Serializer):
+class TaggedJSONSerializer(Serializer):
"""A deliberately restricted, non-pickle serializer for our Huey tasks.
JSON has no native UUID, datetime, tuple or Huey message representation.
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index b1762d5..eb540de 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -22,16 +22,16 @@ def send_webmention_digest() -> None:
"""
# Resolve the lock against the current Huey backend, not the import-time one.
with huey.lock_task("send-webmention-digest"):
- _send_webmention_digest()
+ _send_digest_locked()
-def _send_webmention_digest() -> None:
+def _send_digest_locked() -> None:
"""Do the work while the current backend's digest lock is held.
Collect verified mentions, send the configured message, and persist notification
timestamps.
"""
- recipient = current_app.config.get("NOTIFICATION_EMAIL")
+ recipient = current_app.config.get("DIGEST_EMAIL")
if not recipient:
return
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index eb228b3..313a8e4 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -20,13 +20,13 @@ from ..net import (
AddressResolutionError,
ResponseTooLargeError,
UnsafeAddressError,
- build_client,
- content_type_of,
- is_temporary_status,
+ build_public_http_client,
+ is_retryable_status,
+ media_type_of,
read_limited_body,
)
from ..version import APP_NAME, VERSION
-from .mentions import html_mentions_target, text_mentions_target
+from .target_matching import html_mentions_target, text_mentions_target
class VerificationError(Exception):
@@ -37,14 +37,14 @@ class SourceGoneError(VerificationError):
"""The source explicitly reports that it has been removed."""
-class TemporaryFetchError(Exception):
+class RetryableFetchError(Exception):
"""Fetching the source may succeed when retried later."""
RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
-def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
+def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]:
"""Fetch a webmention source document.
The request follows redirects while enforcing the configured timeout,
@@ -53,13 +53,13 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
:param source_url: URL of the source document.
:returns: HTTP response and response body.
:raises SourceGoneError: If the source returns HTTP 410.
- :raises TemporaryFetchError: If the source returns a temporary HTTP error.
+ :raises RetryableFetchError: If the source returns a temporary HTTP error.
:raises VerificationError: If the source returns another unsuccessful HTTP status or
exceeds the configured maximum response size.
"""
try:
with (
- build_client(
+ build_public_http_client(
f"{APP_NAME}/{VERSION} ReceivedWebmention",
accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
) as client,
@@ -69,8 +69,8 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
pass
elif response.status_code == HTTPStatus.GONE:
raise SourceGoneError("Source returned HTTP 410")
- elif is_temporary_status(response.status_code):
- raise TemporaryFetchError(
+ elif is_retryable_status(response.status_code):
+ raise RetryableFetchError(
f"Source returned HTTP {response.status_code}"
)
else:
@@ -78,7 +78,9 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
try:
return response, read_limited_body(
- response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
+ response,
+ current_app.config["MAX_RECEIVED_SOURCE_BYTES"],
+ label="Source",
)
except ResponseTooLargeError as exc:
raise VerificationError(str(exc)) from exc
@@ -86,7 +88,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
except UnsafeAddressError as exc:
raise VerificationError("Source resolves to a non-public address") from exc
except AddressResolutionError as exc:
- raise TemporaryFetchError("Source hostname could not be resolved") from exc
+ raise RetryableFetchError("Source hostname could not be resolved") from exc
def source_mentions_target(source_url: str, target_url: str) -> bool:
@@ -100,11 +102,11 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
:returns: Whether the source mentions the target URL.
:raises VerificationError: If the source has an unsupported or missing content type.
:raises SourceGoneError: If the source explicitly reports that it is gone.
- :raises TemporaryFetchError: If fetching the source fails temporarily.
+ :raises RetryableFetchError: If fetching the source fails temporarily.
"""
- response, body = fetch_source(source_url)
+ response, body = fetch_source_document(source_url)
- media_type = content_type_of(response.headers)
+ media_type = media_type_of(response.headers)
match media_type:
case "text/html" | "application/xhtml+xml":
@@ -127,13 +129,13 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
class VerificationClaim:
"""Source and lease version for one atomically claimed verification."""
- identifier: uuid.UUID
+ webmention_uuid: uuid.UUID
source: str
target: str
- started_at: datetime
+ lease_started_at: datetime
-def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
+def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
"""Claim an eligible row, returning its source, target, and lease version.
:param webmention_uuid: Identifier of the received Webmention to claim.
@@ -157,10 +159,10 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
return None
claim = VerificationClaim(
- identifier=webmention_uuid,
+ webmention_uuid=webmention_uuid,
source=webmention.source,
target=webmention.target,
- started_at=datetime.now(UTC),
+ lease_started_at=datetime.now(UTC),
)
changed = db.session.execute(
sa.update(ReceivedWebmention)
@@ -172,7 +174,7 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
.values(
status=ReceivedStatus.VERIFYING,
failure_reason=None,
- updated_at=claim.started_at,
+ updated_at=claim.lease_started_at,
)
)
@@ -184,12 +186,14 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
return claim
-def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
+def determine_verification_outcome(
+ claim: VerificationClaim,
+) -> tuple[ReceivedStatus, str | None]:
"""Map verification results and permanent errors to a final status/reason.
:param claim: Claim containing the source and target URLs to verify.
:returns: Final verification status and optional reason.
- :raises TemporaryFetchError: If the source fetch fails transiently and should be
+ :raises RetryableFetchError: If the source fetch fails transiently and should be
retried.
:raises httpx.RequestError: If a transient HTTP request error should be retried.
"""
@@ -199,12 +203,12 @@ def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
return ReceivedStatus.DELETED, str(exc)
except VerificationError as exc:
return ReceivedStatus.FAILED, str(exc)
- except TemporaryFetchError, httpx.RequestError:
+ except RetryableFetchError, httpx.RequestError:
raise # These errors must be persisted and then retried by Huey.
except Exception as exc:
# InvalidURL is not a RequestError; unexpected source errors are terminal.
current_app.logger.exception(
- "Unexpected error verifying ReceivedWebmention %s", claim.identifier
+ "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid
)
return ReceivedStatus.FAILED, str(exc) or type(exc).__name__
@@ -226,9 +230,9 @@ def release_claim(
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
- ReceivedWebmention.uuid == claim.identifier,
+ ReceivedWebmention.uuid == claim.webmention_uuid,
ReceivedWebmention.status == ReceivedStatus.VERIFYING,
- ReceivedWebmention.updated_at == claim.started_at,
+ ReceivedWebmention.updated_at == claim.lease_started_at,
)
.values(status=status, failure_reason=reason, updated_at=datetime.now(UTC))
)
@@ -246,15 +250,15 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
lease.
:param webmention_uuid: Identifier of the received webmention to verify.
- :raises TemporaryFetchError: If fetching the source fails temporarily.
+ :raises RetryableFetchError: If fetching the source fails temporarily.
:raises httpx.RequestError: If an HTTP request fails while fetching.
"""
- if (claim := claim_eligible_row(webmention_uuid)) is None:
+ if (claim := claim_for_verification(webmention_uuid)) is None:
return
try:
- status, reason = check_source(claim)
- except (TemporaryFetchError, httpx.RequestError) as exc:
+ status, reason = determine_verification_outcome(claim)
+ except (RetryableFetchError, httpx.RequestError) as exc:
release_claim(
claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched"
)
@@ -263,7 +267,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
release_claim(claim, status, reason)
-def recover_stale_webmentions() -> int:
+def recover_stale_verifications() -> int:
"""Requeue verification attempts that were abandoned or lost from the queue.
Recover both stale ``verifying`` rows (workers that died) and stale
@@ -280,7 +284,9 @@ def recover_stale_webmentions() -> int:
:returns: Number of webmentions requeued.
"""
now = datetime.now(UTC)
- cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+ cutoff = now - timedelta(
+ seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"]
+ )
stale = db.session.execute(
sa.select(
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 13d2ceb..034b2cf 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -20,15 +20,15 @@ from .sender import send_webmention
from .source_parser import ScannedSource, SourceScanError, scan_source_file
-def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
- """Create a source model from a newly discovered document.
+def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
+ """Build a source model from a newly discovered document.
- A sent Webmention is created for each discovered target at the initial source
+ A sent Webmention is built for each discovered target at the initial source
revision.
:param scanned: Scanned source data.
:param scan_time: Time at which the source was discovered.
- :returns: Newly created source model.
+ :returns: Newly built source model.
"""
source = Source(
path=scanned.path,
@@ -48,7 +48,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
return source
-def update_source(
+def apply_scan_to_source(
source: Source, scanned: ScannedSource, scan_time: datetime
) -> Source:
"""Update an existing source from newly scanned data.
@@ -113,24 +113,24 @@ def scan_sources() -> None:
"""
# Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
- _scan_locked()
+ _scan_and_enqueue()
@dataclass(frozen=True)
-class ScanResult:
+class TreeScanResult:
"""Count and discovered paths from a traversal of the source tree."""
scanned_count: int
seen_paths: frozenset[str]
-def scan_tree(
+def reconcile_source_tree(
root: Path,
base_url: str | None,
sources_by_path: dict[str, Source],
paths_by_url: dict[str, str],
scan_time: datetime,
-) -> ScanResult:
+) -> TreeScanResult:
"""Reconcile found pages and return the paths seen, even on parse failures.
:param root: Root of the generated static-site output tree.
@@ -157,7 +157,7 @@ def scan_tree(
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
+ ignored_hostnames=current_app.config["IGNORED_TARGET_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -182,10 +182,10 @@ def scan_tree(
try:
if (source := sources_by_path.get(scanned.path)) is None:
- source = create_source(scanned, scan_time)
+ source = build_source(scanned, scan_time)
db.session.add(source)
else:
- source = update_source(source, scanned, scan_time)
+ source = apply_scan_to_source(source, scanned, scan_time)
sources_by_path[source.path] = source
paths_by_url[source.url] = source.path
@@ -193,10 +193,10 @@ def scan_tree(
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- return ScanResult(scanned_count, frozenset(seen_paths))
+ return TreeScanResult(scanned_count, frozenset(seen_paths))
-def mark_missing(
+def mark_missing_sources_deleted(
sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
) -> int:
"""Mark absent, previously active sources deleted and return their count.
@@ -225,7 +225,7 @@ def mark_missing(
return marked
-def queue_pending(scan_time: datetime) -> int:
+def enqueue_due_sends(scan_time: datetime) -> int:
"""Enqueue due Webmentions and return the number of tasks submitted.
:param scan_time: Current scan timestamp used to evaluate retry deadlines.
@@ -249,7 +249,7 @@ def queue_pending(scan_time: datetime) -> int:
return queued
-def _scan_locked() -> None:
+def _scan_and_enqueue() -> None:
"""Scan sources while the caller holds the shared scan lock.
Reconcile the generated source tree, persist changes, and enqueue eligible outgoing
@@ -257,7 +257,7 @@ def _scan_locked() -> None:
:raises RuntimeError: If the source directory or configured base URL is invalid.
"""
- directory = current_app.config.get("SOURCE_DIRECTORY")
+ directory = current_app.config.get("SITE_OUTPUT_DIRECTORY")
if directory is None:
return
@@ -265,9 +265,9 @@ def _scan_locked() -> None:
if not root.is_dir():
raise RuntimeError(f"Source directory is not a directory: {root}")
- if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
+ if (base_url := current_app.config.get("SITE_BASE_URL")) is not None:
if not is_http_url(base_url):
- raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
+ raise RuntimeError("SITE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
sources_by_path = {
@@ -281,7 +281,9 @@ def _scan_locked() -> None:
# take ownership merely by appearing earlier in the traversal.
paths_by_url = {source.url: source.path for source in sources_by_path.values()}
scan_time = datetime.now(UTC)
- scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time)
+ scan = reconcile_source_tree(
+ root, base_url, sources_by_path, paths_by_url, scan_time
+ )
if scan.scanned_count == 0:
current_app.logger.warning(
@@ -290,9 +292,11 @@ def _scan_locked() -> None:
)
return
- missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time)
+ missing_count = mark_missing_sources_deleted(
+ sources_by_path, scan.seen_paths, scan_time
+ )
db.session.commit()
- pending_count = queue_pending(scan_time)
+ pending_count = enqueue_due_sends(scan_time)
current_app.logger.info(
"Webmention source scan complete: %d sources scanned, "
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index cf8ab9d..c33cf06 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -19,20 +19,20 @@ from ..models import SentWebmention, SentWebmentionStatus
from ..net import (
AddressResolutionError,
UnsafeAddressError,
- build_client,
+ build_public_http_client,
is_http_url,
- is_temporary_status,
+ is_retryable_status,
)
from ..version import APP_NAME, VERSION
from .discovery import (
- PermanentSenderError,
- TemporarySenderError,
+ PermanentDeliveryError,
+ RetryableDeliveryError,
discover_webmention_endpoint,
post_webmention,
)
-def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
+def revision_still_pending(webmention: SentWebmention, revision: int) -> bool:
"""Check whether a send attempt still represents the desired revision.
The Webmention is refreshed from the database before comparing its desired
@@ -47,7 +47,7 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
return webmention.desired_revision == revision and webmention.pending
-def retry_delay(attempt: int) -> timedelta:
+def backoff_delay(attempt: int) -> timedelta:
"""Calculate the capped exponential delay after a numbered send attempt.
:param attempt: One-based send attempt number.
@@ -76,7 +76,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
# The pending/retry checks must happen *under* this lock.
try:
with huey.lock_task(f"send-webmention-{webmention_uuid}"):
- _send_locked(webmention_uuid)
+ _process_send_locked(webmention_uuid)
except TaskLockedException:
# Another worker is already sending this Webmention. The active worker
# owns its result; a later scan will requeue it if it remains pending.
@@ -84,12 +84,12 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
@dataclass(frozen=True)
-class SendAttempt:
+class ClaimedSendAttempt:
"""Claimed source revision and retry budget for one send."""
- identifier: uuid.UUID
+ webmention_uuid: uuid.UUID
revision: int
- number: int
+ attempt_number: int
max_attempts: int
source: str
target: str
@@ -112,7 +112,7 @@ class DeliveryResult:
temporary_failure: bool = False
-def is_due(webmention: SentWebmention, now: datetime) -> bool:
+def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
"""Return whether an unprocessed Webmention has reached its retry deadline.
:param webmention: Outgoing Webmention whose retry eligibility is checked.
@@ -130,7 +130,9 @@ def is_due(webmention: SentWebmention, now: datetime) -> bool:
return aware_deadline <= now
-def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | None:
+def reserve_send_attempt(
+ webmention: SentWebmention, now: datetime
+) -> ClaimedSendAttempt | None:
"""Atomically reserve one send attempt, or finish an exhausted revision.
An attempt's next retry deadline is persisted *before* the HTTP request so
@@ -172,10 +174,10 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
db.session.commit()
return None
- attempt = SendAttempt(
- identifier=webmention.uuid,
+ attempt = ClaimedSendAttempt(
+ webmention_uuid=webmention.uuid,
revision=revision,
- number=webmention.attempt_count + 1,
+ attempt_number=webmention.attempt_count + 1,
max_attempts=max_attempts,
source=webmention.source.url,
target=webmention.target,
@@ -183,9 +185,9 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
changed = db.session.execute(
sa.update(SentWebmention)
.where(
- SentWebmention.uuid == attempt.identifier,
+ SentWebmention.uuid == attempt.webmention_uuid,
SentWebmention.desired_revision == revision,
- SentWebmention.attempt_count == attempt.number - 1,
+ SentWebmention.attempt_count == attempt.attempt_number - 1,
SentWebmention.pending,
sa.or_(
SentWebmention.next_retry_at.is_(None),
@@ -193,9 +195,9 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
),
)
.values(
- attempt_count=attempt.number,
+ attempt_count=attempt.attempt_number,
last_attempted_at=now,
- next_retry_at=now + retry_delay(attempt.number),
+ next_retry_at=now + backoff_delay(attempt.attempt_number),
endpoint=None,
response_status=None,
status_url=None,
@@ -211,7 +213,7 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
return attempt
-def deliver_webmention(
+def attempt_delivery(
source: str, target: str, *, webmention_uuid: uuid.UUID
) -> DeliveryResult:
"""Discover and POST a Webmention, returning an outcome without DB writes.
@@ -227,9 +229,9 @@ def deliver_webmention(
try:
if not is_http_url(target):
- raise PermanentSenderError(f"Invalid target URL: {target!r}")
+ raise PermanentDeliveryError(f"Invalid target URL: {target!r}")
- with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
+ with build_public_http_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
@@ -243,9 +245,9 @@ def deliver_webmention(
)
if not 200 <= response_status <= 299:
message = f"Webmention endpoint returned HTTP {response_status}"
- if is_temporary_status(response_status):
- raise TemporarySenderError(message)
- raise PermanentSenderError(message)
+ if is_retryable_status(response_status):
+ raise RetryableDeliveryError(message)
+ raise PermanentDeliveryError(message)
return DeliveryResult(
status=SentWebmentionStatus.SENT,
@@ -254,7 +256,7 @@ def deliver_webmention(
status_url=status_url,
)
except (
- PermanentSenderError,
+ PermanentDeliveryError,
UnsafeAddressError,
httpx.InvalidURL,
httpx.UnsupportedProtocol,
@@ -267,7 +269,7 @@ def deliver_webmention(
endpoint=endpoint,
response_status=response_status,
)
- except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
+ except (RetryableDeliveryError, AddressResolutionError, httpx.RequestError) as exc:
return DeliveryResult(
status=SentWebmentionStatus.FAILED,
reason=str(exc) or "Webmention request failed",
@@ -289,9 +291,9 @@ def deliver_webmention(
)
-def record_outcome(
+def record_delivery_outcome(
webmention: SentWebmention,
- attempt: SendAttempt,
+ attempt: ClaimedSendAttempt,
result: DeliveryResult,
*,
terminal: bool,
@@ -305,7 +307,7 @@ def record_outcome(
:returns: ``True`` if the outcome was recorded for the current revision.
"""
# A scanner may have reopened or superseded the revision during I/O.
- if not attempt_is_current(webmention, attempt.revision):
+ if not revision_still_pending(webmention, attempt.revision):
return False
webmention.status = result.status
@@ -330,7 +332,7 @@ def record_outcome(
return True
-def _send_locked(webmention_uuid: uuid.UUID) -> bool:
+def _process_send_locked(webmention_uuid: uuid.UUID) -> bool:
"""Process a due Webmention while holding its per-UUID task lock.
:param webmention_uuid: Identifier of the outgoing Webmention being processed.
@@ -344,18 +346,18 @@ def _send_locked(webmention_uuid: uuid.UUID) -> bool:
return False
now = datetime.now(UTC)
- if not is_due(webmention, now):
+ if not is_ready_to_send(webmention, now):
return False
- if (attempt := claim_attempt(webmention, now)) is None:
+ if (attempt := reserve_send_attempt(webmention, now)) is None:
return False
- result = deliver_webmention(
+ result = attempt_delivery(
attempt.source, attempt.target, webmention_uuid=webmention_uuid
)
- terminal = result.terminal or attempt.number >= attempt.max_attempts
+ terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts
- if not record_outcome(webmention, attempt, result, terminal=terminal):
+ if not record_delivery_outcome(webmention, attempt, result, terminal=terminal):
return False
if result.temporary_failure and not terminal:
@@ -363,13 +365,13 @@ def _send_locked(webmention_uuid: uuid.UUID) -> bool:
current_app.logger.info(
"Webmention %s send attempt %d failed; will retry",
webmention_uuid,
- attempt.number,
+ attempt.attempt_number,
)
elif result.temporary_failure:
current_app.logger.warning(
"Webmention %s exhausted %d send attempts: %s",
webmention_uuid,
- attempt.number,
+ attempt.attempt_number,
result.reason,
)
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 31d0c98..e063525 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -18,11 +18,11 @@ import xxhash
from bs4 import BeautifulSoup, Tag
from bs4.element import NavigableString, PageElement
-from ..net import hostname_of, is_http_url
+from ..net import is_http_url, normalized_hostname
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-CONTENT_BLOCK_TAGS = frozenset(
+BLOCK_LEVEL_TAGS = frozenset(
{
"address",
"article",
@@ -83,7 +83,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
return urljoin(base_url, f"{quote(directory, safe='/')}/")
-def canonical_url(
+def declared_canonical_url(
document: BeautifulSoup, *, relative_path: Path, base_url: str | None
) -> str | None:
"""Return the canonical URL declared by a source document.
@@ -111,7 +111,7 @@ def canonical_url(
if base_url is None:
raise SourceScanError(
"Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
+ "SITE_BASE_URL is not configured"
)
resolved = urljoin(source_url_for_path(relative_path, base_url), href)
@@ -124,7 +124,7 @@ def canonical_url(
return resolved
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+def mf2_string_values(entry: dict[str, Any], property_name: str) -> Iterator[str]:
"""Yield URL values from a microformats property.
:param entry: Parsed microformats entry.
@@ -144,7 +144,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
yield from (value for value in values if isinstance(value, str))
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+def parse_h_entry(element: Tag, base_url: str) -> dict[str, Any]:
"""Parse and validate an h-entry with mf2py.
:param element: HTML element containing the h-entry.
@@ -175,7 +175,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
return entry
-def primary_entry(
+def find_primary_h_entry(
document: BeautifulSoup, source_url: str
) -> tuple[Tag, dict[str, Any]]:
"""Return and validate the primary h-entry of a source document.
@@ -195,9 +195,9 @@ def primary_entry(
raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
+ mf2_entry = parse_h_entry(entry_element, source_url)
- urls = tuple(property_urls(mf2_entry, "url"))
+ urls = tuple(mf2_string_values(mf2_entry, "url"))
if source_url not in urls:
raise SourceScanError(
@@ -207,15 +207,15 @@ def primary_entry(
return entry_element, mf2_entry
-def content_element(entry: Tag) -> Tag:
+def find_e_content(entry_element: Tag) -> Tag:
"""Return the e-content element of an h-entry.
- :param entry: HTML element containing the h-entry.
+ :param entry_element: HTML element containing the h-entry.
:returns: The h-entry's e-content element.
:raises SourceScanError: If the h-entry does not contain exactly one e-content
element.
"""
- contents = entry.find_all(class_="e-content")
+ contents = entry_element.find_all(class_="e-content")
if len(contents) != 1:
raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
@@ -223,7 +223,7 @@ def content_element(entry: Tag) -> Tag:
return contents[0]
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+def normalize_target_url(value: str, *, base_url: str, source_url: str) -> str | None:
"""Resolve and validate a possible Webmention target URL.
Empty values, non-HTTP URLs, and URLs referring to the source itself are
@@ -249,7 +249,7 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non
return target
-def iter_targets(
+def iter_outgoing_targets(
content: Tag,
mf2_entry: dict[str, Any],
*,
@@ -277,18 +277,19 @@ def iter_targets(
)
reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ mf2_string_values(mf2_entry, property_name)
+ for property_name in REACTION_PROPERTIES
)
for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
+ target = normalize_target_url(value, base_url=base_url, source_url=source_url)
if target is not None:
- hostname = hostname_of(target) or ""
+ hostname = normalized_hostname(target) or ""
if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
yield target
-def normalized_content(content: Tag) -> str:
+def stable_content_text(content: Tag) -> str:
"""Extract stable reader-facing content without generated HTML attributes.
:param content: Parsed e-content element to normalize.
@@ -312,7 +313,7 @@ def normalized_content(content: Tag) -> str:
return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+ return f" {text} " if node.name in BLOCK_LEVEL_TAGS else text
return " ".join(extract(content).split())
@@ -346,22 +347,24 @@ def scan_source_file(
relative_path = path.relative_to(root)
document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+ source_url = declared_canonical_url(
+ document, relative_path=relative_path, base_url=base_url
+ )
if source_url is None:
if base_url is None:
raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ "Document has no canonical URL and SITE_BASE_URL is not configured"
)
source_url = source_url_for_path(relative_path, base_url)
- entry_element, mf2_entry = primary_entry(document, source_url)
+ entry_element, mf2_entry = find_primary_h_entry(document, source_url)
- content = content_element(entry_element)
+ content = find_e_content(entry_element)
targets = frozenset(
- iter_targets(
+ iter_outgoing_targets(
content,
mf2_entry,
base_url=source_url,
@@ -370,7 +373,7 @@ def scan_source_file(
)
)
- stable_content = normalized_content(content)
+ stable_content = stable_content_text(content)
fingerprint = json.dumps(
{"content": stable_content, "targets": sorted(targets)},
ensure_ascii=False,
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/target_matching.py
index 4c3918b..4c3918b 100644
--- a/webmentions_ssg/tasks/mentions.py
+++ b/webmentions_ssg/tasks/target_matching.py