aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/consumer.py2
-rw-r--r--webmentions_ssg/tasks/discovery.py26
-rw-r--r--webmentions_ssg/tasks/extension.py74
-rw-r--r--webmentions_ssg/tasks/mentions.py4
-rw-r--r--webmentions_ssg/tasks/notifications.py12
-rw-r--r--webmentions_ssg/tasks/receiver.py42
-rw-r--r--webmentions_ssg/tasks/retention.py5
-rw-r--r--webmentions_ssg/tasks/scanner.py50
-rw-r--r--webmentions_ssg/tasks/sender.py41
-rw-r--r--webmentions_ssg/tasks/source_parser.py40
10 files changed, 220 insertions, 76 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 362ebe8..13334bc 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -17,7 +17,7 @@ def create_consumer() -> Huey:
The application is initialized and the periodic scanner, notification,
verification recovery and retention tasks are registered.
- :return: Configured Huey instance.
+ :returns: Configured Huey instance.
"""
app = create_app()
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index becac58..6d01e53 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -41,9 +41,8 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str:
:param response: Response from which the endpoint was discovered.
:param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
+ :returns: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL.
"""
try:
endpoint = urljoin(str(response.url), href.strip())
@@ -60,7 +59,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None:
"""Parse a Link header value into its target and relations.
:param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
+ :returns: Link target and relation names, or ``None`` if the value is invalid.
"""
value = value.strip()
@@ -89,8 +88,8 @@ def endpoint_from_headers(response: httpx.Response) -> str | None:
"""Find a Webmention endpoint in the response Link headers.
:param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
+ :returns: First valid advertised Webmention endpoint, or ``None`` if none is found.
+ Invalid candidates are ignored.
"""
for header in response.headers.get_list("Link"):
for value in LINK_SPLIT.split(header):
@@ -115,8 +114,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
:param response: HTTP response from which the document was retrieved.
:param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
+ :returns: First valid advertised Webmention endpoint, or ``None`` if none is found.
+ Invalid candidates are ignored.
"""
document = BeautifulSoup(body, "html5lib")
@@ -132,7 +131,12 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
def read_target_body(response: httpx.Response) -> bytes:
- """Return a bounded target body, using the shared HTTP size guard."""
+ """Return a bounded target body, using the shared HTTP size guard.
+
+ :param response: HTTP response to read with the configured target-size limit.
+ :returns: Response body, provided it is within the target-size limit.
+ :raises PermanentSenderError: If the target document is too large.
+ """
try:
return read_limited_body(
response, current_app.config["MAX_TARGET_BYTES"], label="Target"
@@ -150,7 +154,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
:param client: HTTP client to use for discovery requests.
:param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :returns: Discovered Webmention endpoint, or ``None`` if none is advertised.
:raises TemporarySenderError: If the target returns a temporary HTTP failure.
:raises PermanentSenderError: If GET returns a permanent HTTP failure.
"""
@@ -191,7 +195,7 @@ def post_webmention(
:param endpoint: Webmention endpoint URL.
:param source: Source URL of the Webmention.
:param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
+ :returns: HTTP response status code and optional status URL.
"""
with client.stream(
"POST", endpoint, data={"source": source, "target": target}
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index b77a140..58e0036 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -19,7 +19,12 @@ from .json_serializer import JSONSerializer
def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Select a Redis Huey flavour while retaining the underlying URL scheme."""
+ """Select a Redis Huey flavour while retaining the underlying URL scheme.
+
+ :param parsed: Parsed Redis backend URL, including any flavour suffix.
+ :returns: Huey class and constructor options for the requested Redis flavour.
+ :raises RuntimeError: If the Redis backend flavour is unsupported.
+ """
from huey import (
PriorityRedisExpireHuey,
PriorityRedisHuey,
@@ -43,7 +48,13 @@ def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]
def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Extract a relative or absolute SQLite/file storage path."""
+ """Extract a relative or absolute SQLite/file storage path.
+
+ :param url: Backend URL containing the storage filename or path.
+ :param scheme: Selected file or SQLite backend scheme.
+ :returns: Huey class and constructor options for the file-backed queue.
+ :raises RuntimeError: If the storage URL is malformed or omits the path.
+ """
from huey import FileHuey, SqliteHuey
if scheme == "sqlite":
@@ -101,7 +112,11 @@ class _LazyTask:
update_wrapper(self, func)
def register(self) -> None:
- """Bind the original function to the latest initialized backend."""
+ """Bind the original function to the latest initialized backend.
+
+ Register the task decorator again so its stable handle uses the currently
+ initialized Huey instance.
+ """
backend = self._extension.huey
decorator = backend.periodic_task if self._periodic else backend.task
self._registered = decorator(*self._args, **self._kwargs)(
@@ -116,11 +131,20 @@ class _LazyTask:
return self._registered
def __call__(self, *args: Any, **kwargs: Any) -> Any:
- """Enqueue a task using the active backend."""
+ """Enqueue a task using the active backend.
+
+ :param args: Positional arguments passed to the queued task.
+ :param kwargs: Keyword arguments passed to the queued task.
+ :returns: Result of enqueueing the task on the current backend.
+ """
return self._current()(*args, **kwargs)
def __getattr__(self, name: str) -> Any:
- """Delegate Huey task operations (call_local, schedule, s, etc.)."""
+ """Delegate Huey task operations (call_local, schedule, s, etc.).
+
+ :param name: Huey task attribute to delegate.
+ :returns: Requested attribute of the currently bound Huey task.
+ """
return getattr(self._current(), name)
@@ -150,8 +174,8 @@ class Huey:
resulting extension is registered with the application.
:param app: Flask application to initialize.
- :raises RuntimeError: If the configured Huey backend URL is invalid or uses
- an unsupported scheme.
+ :raises RuntimeError: If the configured Huey backend URL is invalid or uses an
+ unsupported scheme.
"""
config: dict[str, Any] = {
"name": app.import_name,
@@ -190,6 +214,7 @@ class Huey:
def huey(self) -> BaseHuey:
"""Return the initialized Huey instance.
+ :returns: Active, initialized Huey backend.
:raises RuntimeError: If the extension has not been initialized.
"""
if self._huey is None:
@@ -200,7 +225,12 @@ class Huey:
return self._huey
def _in_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]:
- """Return a task wrapper that executes with the configured Flask app."""
+ """Return a task wrapper that executes with the configured Flask app.
+
+ :param func: Task function to wrap in a Flask application context.
+ :returns: Task wrapper that enters the configured Flask application context.
+ :raises RuntimeError: If no Flask application has been registered.
+ """
@wraps(func)
def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
@@ -228,7 +258,12 @@ class Huey:
def task[**P, R](
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, R]], _LazyTask]:
- """Declare a task even before :meth:`init_app` has run."""
+ """Declare a task even before :meth:`init_app` has run.
+
+ :param task_args: Positional arguments for the Huey task decorator.
+ :param task_kwargs: Keyword options for the Huey task decorator.
+ :returns: Decorator that registers and returns a lazily bound task.
+ """
def decorator(func: Callable[P, R]) -> _LazyTask:
return self._decorate_task(func, task_args, task_kwargs, periodic=False)
@@ -238,7 +273,12 @@ class Huey:
def periodic_task[**P, R](
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, R]], _LazyTask]:
- """Declare a periodic task against the current or next backend."""
+ """Declare a periodic task against the current or next backend.
+
+ :param task_args: Positional arguments for the Huey periodic-task decorator.
+ :param task_kwargs: Keyword options for the Huey periodic-task decorator.
+ :returns: Decorator that registers and returns a lazily bound periodic task.
+ """
def decorator(func: Callable[P, R]) -> _LazyTask:
return self._decorate_task(func, task_args, task_kwargs, periodic=True)
@@ -249,7 +289,7 @@ class Huey:
"""Forward an unknown attribute to the underlying Huey instance.
:param name: Name of the attribute to retrieve.
- :return: Attribute from the initialized Huey instance.
+ :returns: Attribute from the initialized Huey instance.
:raises RuntimeError: If the extension has not been initialized.
"""
return getattr(self.huey, name)
@@ -258,12 +298,12 @@ class Huey:
def backend_from_url(url: str) -> tuple[type[BaseHuey], dict[str, Any]]:
"""Determine the Huey backend and storage options from a URL.
- :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's
- slash convention: three slashes for relative paths and four for
- absolute paths (not RFC 8089 file-URI semantics).
- :return: Huey backend class and keyword arguments for its storage backend.
- :raises RuntimeError: If the URL is malformed for the selected backend or
- uses an unsupported scheme.
+ :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's slash
+ convention: three slashes for relative paths and four for absolute
+ paths (not RFC 8089 file-URI semantics).
+ :returns: Huey backend class and keyword arguments for its storage backend.
+ :raises RuntimeError: If the URL is malformed for the selected backend or uses
+ an unsupported scheme.
"""
parsed = urlsplit(url)
scheme = parsed.scheme.lower()
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py
index 8f41902..4c3918b 100644
--- a/webmentions_ssg/tasks/mentions.py
+++ b/webmentions_ssg/tasks/mentions.py
@@ -35,7 +35,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
:param body: HTML document body.
:param source_url: URL from which the document was retrieved.
:param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
+ :returns: Whether a supported HTML URL attribute references the target URL.
"""
document = BeautifulSoup(body, "html.parser")
base_url = source_url
@@ -63,7 +63,7 @@ def text_mentions_target(body: str, target_url: str) -> bool:
:param body: Plain-text document body.
:param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
+ :returns: Whether the target IRI occurs in the document.
"""
for match in IRI_PATTERN.finditer(body):
candidate = match.group()
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index fc1b475..b1762d5 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -15,14 +15,22 @@ from ..models import ReceivedStatus, ReceivedWebmention
def send_webmention_digest() -> None:
- """Send one digest of verified, previously unreported Webmentions."""
+ """Send one digest of verified, previously unreported Webmentions.
+
+ Acquire the digest lock before notifying recipients about previously unreported
+ verified mentions.
+ """
# Resolve the lock against the current Huey backend, not the import-time one.
with huey.lock_task("send-webmention-digest"):
_send_webmention_digest()
def _send_webmention_digest() -> None:
- """Do the work while the current backend's digest lock is held."""
+ """Do the work while the current backend's digest lock is held.
+
+ Collect verified mentions, send the configured message, and persist notification
+ timestamps.
+ """
recipient = current_app.config.get("NOTIFICATION_EMAIL")
if not recipient:
return
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 31fb64c..eb228b3 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -51,11 +51,11 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
redirect limit, and maximum response size.
:param source_url: URL of the source document.
- :return: HTTP response and response body.
+ :returns: HTTP response and response body.
:raises SourceGoneError: If the source returns HTTP 410.
:raises TemporaryFetchError: If the source returns a temporary HTTP error.
- :raises VerificationError: If the source returns another unsuccessful HTTP
- status or exceeds the configured maximum response size.
+ :raises VerificationError: If the source returns another unsuccessful HTTP status or
+ exceeds the configured maximum response size.
"""
try:
with (
@@ -97,9 +97,8 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
:param source_url: URL of the source document.
:param target_url: Exact target URL to look for.
- :return: Whether the source mentions the target URL.
- :raises VerificationError: If the source has an unsupported or missing
- content type.
+ :returns: Whether the source mentions the target URL.
+ :raises VerificationError: If the source has an unsupported or missing content type.
:raises SourceGoneError: If the source explicitly reports that it is gone.
:raises TemporaryFetchError: If fetching the source fails temporarily.
"""
@@ -135,7 +134,11 @@ class VerificationClaim:
def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
- """Claim an eligible row, returning its source, target, and lease version."""
+ """Claim an eligible row, returning its source, target, and lease version.
+
+ :param webmention_uuid: Identifier of the received Webmention to claim.
+ :returns: Verification claim, or ``None`` if the row cannot be claimed.
+ """
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
if webmention is None:
@@ -182,7 +185,14 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
- """Map verification results and permanent errors to a final status/reason."""
+ """Map verification results and permanent errors to a final status/reason.
+
+ :param claim: Claim containing the source and target URLs to verify.
+ :returns: Final verification status and optional reason.
+ :raises TemporaryFetchError: If the source fetch fails transiently and should be
+ retried.
+ :raises httpx.RequestError: If a transient HTTP request error should be retried.
+ """
try:
mentions_target = source_mentions_target(claim.source, claim.target)
except SourceGoneError as exc:
@@ -206,7 +216,13 @@ def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
def release_claim(
claim: VerificationClaim, status: ReceivedStatus, reason: str | None
) -> bool:
- """Persist an outcome only if this worker still owns the verification lease."""
+ """Persist an outcome only if this worker still owns the verification lease.
+
+ :param claim: Verification lease acquired by this worker.
+ :param status: Final status to persist for the claimed Webmention.
+ :param reason: Optional explanation for a failed or deleted Webmention.
+ :returns: ``True`` if the worker still owned the lease and persisted the outcome.
+ """
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
@@ -226,6 +242,9 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
Temporary fetch failures are recorded and re-raised for Huey's retries.
+ Retry transient fetch errors; persist only results belonging to the currently held
+ lease.
+
:param webmention_uuid: Identifier of the received webmention to verify.
:raises TemporaryFetchError: If fetching the source fails temporarily.
:raises httpx.RequestError: If an HTTP request fails while fetching.
@@ -255,7 +274,10 @@ def recover_stale_webmentions() -> int:
Persist a recovery count so repeatedly lost or crashing tasks eventually
terminate rather than being requeued forever.
- :return: Number of webmentions requeued.
+ Eligible expired verification leases are returned to the queue, subject to recovery
+ limits.
+
+ :returns: Number of webmentions requeued.
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 86283b7..2c35cc7 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -20,6 +20,11 @@ def purge_old_received_webmentions() -> int:
The status/time predicate is checked by the database in one DELETE, so a
row recently requeued for verification is not purged based on a stale read.
Outgoing sources and Webmentions are intentionally unaffected.
+
+ Use the configured retention period and leave verified or in-progress rows
+ untouched.
+
+ :returns: Number of expired terminal Webmentions removed.
"""
cutoff = datetime.now(UTC) - timedelta(
days=current_app.config["RECEIVED_RETENTION_DAYS"]
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index ce03185..13d2ceb 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -28,7 +28,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
:param scanned: Scanned source data.
:param scan_time: Time at which the source was discovered.
- :return: Newly created source model.
+ :returns: Newly created source model.
"""
source = Source(
path=scanned.path,
@@ -60,7 +60,7 @@ def update_source(
:param source: Existing source model to update.
:param scanned: Newly scanned source data.
:param scan_time: Time at which the source was scanned.
- :return: Updated source model.
+ :returns: Updated source model.
:raises SourceScanError: If the public URL of the source has changed.
"""
if source.url != scanned.url:
@@ -106,8 +106,10 @@ def scan_sources() -> None:
successfully processes no sources is treated as inconclusive: it neither
marks sources deleted nor queues Webmentions.
- :raises RuntimeError: If the configured source directory or base URL is
- invalid.
+ Require a configured source root and public base URL before scanning generated
+ pages.
+
+ :raises RuntimeError: If the configured source directory or base URL is invalid.
"""
# Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
@@ -129,7 +131,15 @@ def scan_tree(
paths_by_url: dict[str, str],
scan_time: datetime,
) -> ScanResult:
- """Reconcile found pages and return the paths seen, even on parse failures."""
+ """Reconcile found pages and return the paths seen, even on parse failures.
+
+ :param root: Root of the generated static-site output tree.
+ :param base_url: Optional public base URL used for source URL resolution.
+ :param sources_by_path: Known source records indexed by relative path.
+ :param paths_by_url: Known relative paths indexed by public source URL.
+ :param scan_time: Timestamp associated with the current scan.
+ :returns: Scan result containing discovered paths and processing counts.
+ """
seen_paths: set[str] = set()
scanned_count = 0
@@ -189,7 +199,13 @@ def scan_tree(
def mark_missing(
sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
) -> int:
- """Mark absent, previously active sources deleted and return their count."""
+ """Mark absent, previously active sources deleted and return their count.
+
+ :param sources_by_path: Known source records indexed by relative path.
+ :param seen_paths: Paths found during this scan, including failed parses.
+ :param scan_time: Timestamp to record for sources newly marked missing.
+ :returns: Number of previously active sources marked as deleted.
+ """
marked = 0
for path, source in sources_by_path.items():
if source.deleted_at is not None or path in seen_paths:
@@ -210,7 +226,11 @@ def mark_missing(
def queue_pending(scan_time: datetime) -> int:
- """Enqueue due Webmentions and return the number of tasks submitted."""
+ """Enqueue due Webmentions and return the number of tasks submitted.
+
+ :param scan_time: Current scan timestamp used to evaluate retry deadlines.
+ :returns: Number of due Webmention tasks enqueued.
+ """
queued = 0
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
@@ -230,7 +250,13 @@ def queue_pending(scan_time: datetime) -> int:
def _scan_locked() -> None:
- """Scan sources while the caller holds the shared scan lock."""
+ """Scan sources while the caller holds the shared scan lock.
+
+ Reconcile the generated source tree, persist changes, and enqueue eligible outgoing
+ mentions.
+
+ :raises RuntimeError: If the source directory or configured base URL is invalid.
+ """
directory = current_app.config.get("SOURCE_DIRECTORY")
if directory is None:
return
@@ -279,7 +305,13 @@ def _scan_locked() -> None:
@huey.task()
def manual_scan_sources() -> None:
- """Run a source scan, deferring it while another scan holds the lock."""
+ """Run a source scan, deferring it while another scan holds the lock.
+
+ If another scan holds the lock, arrange for the scan to run later instead of
+ overlapping it.
+
+ :raises RetryTask: If another worker currently holds the scan lock.
+ """
try:
scan_sources()
except TaskLockedException as exc:
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 339dfe6..cf8ab9d 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -40,7 +40,7 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
:param webmention: Sent Webmention being processed.
:param revision: Source revision represented by the current attempt.
- :return: Whether the attempt is still current and requires processing.
+ :returns: Whether the attempt is still current and requires processing.
"""
db.session.refresh(webmention)
@@ -48,7 +48,11 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
def retry_delay(attempt: int) -> timedelta:
- """Calculate the capped exponential delay after a numbered send attempt."""
+ """Calculate the capped exponential delay after a numbered send attempt.
+
+ :param attempt: One-based send attempt number.
+ :returns: Delay until the next attempt, capped by the configured maximum.
+ """
base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"]))
maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"]))
return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum))
@@ -63,6 +67,8 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
scanner requeues them when due; Huey does not retry them independently.
The retry budget is per desired source revision.
+ Run the delivery workflow under a lock associated with the outgoing Webmention.
+
:param webmention_uuid: Identifier of the sent Webmention to process.
"""
# Huey locks have static names when used as decorators. Acquire one keyed by
@@ -107,7 +113,12 @@ class DeliveryResult:
def is_due(webmention: SentWebmention, now: datetime) -> bool:
- """Return whether an unprocessed Webmention has reached its retry deadline."""
+ """Return whether an unprocessed Webmention has reached its retry deadline.
+
+ :param webmention: Outgoing Webmention whose retry eligibility is checked.
+ :param now: Current timestamp against which the retry deadline is compared.
+ :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed.
+ """
if not webmention.pending:
return False
@@ -124,6 +135,10 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
An attempt's next retry deadline is persisted *before* the HTTP request so
a lost worker cannot cause an immediate duplicate send.
+
+ :param webmention: Outgoing Webmention to reserve for sending.
+ :param now: Timestamp at which the attempt is claimed.
+ :returns: Reserved attempt details, or ``None`` if no attempt can be started.
"""
revision = webmention.desired_revision
max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
@@ -199,7 +214,13 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
def deliver_webmention(
source: str, target: str, *, webmention_uuid: uuid.UUID
) -> DeliveryResult:
- """Discover and POST a Webmention, returning an outcome without DB writes."""
+ """Discover and POST a Webmention, returning an outcome without DB writes.
+
+ :param source: Public source URL advertising the target link.
+ :param target: Target URL to notify.
+ :param webmention_uuid: Identifier used for delivery logging.
+ :returns: Delivery status, endpoint, response details, and any failure reason.
+ """
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
@@ -275,7 +296,14 @@ def record_outcome(
*,
terminal: bool,
) -> bool:
- """Record an outcome only while the claimed source revision is current."""
+ """Record an outcome only while the claimed source revision is current.
+
+ :param webmention: Outgoing Webmention being updated.
+ :param attempt: Claimed revision and attempt details.
+ :param result: Result of the delivery attempt.
+ :param terminal: Whether this outcome ends attempts for the current revision.
+ :returns: ``True`` if the outcome was recorded for the current revision.
+ """
# A scanner may have reopened or superseded the revision during I/O.
if not attempt_is_current(webmention, attempt.revision):
return False
@@ -305,7 +333,8 @@ def record_outcome(
def _send_locked(webmention_uuid: uuid.UUID) -> bool:
"""Process a due Webmention while holding its per-UUID task lock.
- :return: Whether this worker recorded a delivery outcome.
+ :param webmention_uuid: Identifier of the outgoing Webmention being processed.
+ :returns: Whether this worker recorded a delivery outcome.
"""
webmention = db.session.get(SentWebmention, webmention_uuid)
if webmention is None:
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 294402c..31d0c98 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -76,7 +76,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
:param relative_path: Path of the source document relative to the source root.
:param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
+ :returns: Public URL corresponding to the source document.
"""
directory = relative_path.parent.as_posix()
@@ -93,9 +93,9 @@ def canonical_url(
:param document: Parsed HTML source document.
:param relative_path: Path of the source document relative to the source root.
:param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
+ :returns: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved, or does
+ not resolve to an HTTP or HTTPS URL.
"""
if (link := document.select_one('link[rel~="canonical"][href]')) is None:
return None
@@ -129,7 +129,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
:param entry: Parsed microformats entry.
:param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
+ :returns: Iterator over string values of the property.
"""
properties = entry.get("properties")
@@ -149,7 +149,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
:param element: HTML element containing the h-entry.
:param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
+ :returns: Parsed h-entry.
:raises SourceScanError: If parsing does not produce exactly one valid h-entry.
"""
parsed = mf2py.parse(doc=str(element), url=base_url)
@@ -185,9 +185,9 @@ def primary_entry(
:param document: Parsed HTML source document.
:param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
+ :returns: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry or its
+ ``u-url`` does not match the source URL.
"""
entries = document.find_all(class_="h-entry")
@@ -211,9 +211,9 @@ def content_element(entry: Tag) -> Tag:
"""Return the e-content element of an h-entry.
:param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
+ :returns: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one e-content
+ element.
"""
contents = entry.find_all(class_="e-content")
@@ -232,8 +232,8 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non
:param value: URL reference to normalize.
:param base_url: Base URL against which relative references are resolved.
:param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
+ :returns: Normalized target URL, or ``None`` if the value is not a valid Webmention
+ target.
"""
if not (value := value.strip()):
return None
@@ -268,7 +268,7 @@ def iter_targets(
:param base_url: Base URL against which relative targets are resolved.
:param source_url: URL of the source document.
:param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
+ :returns: Iterator over outgoing Webmention target URLs.
"""
hrefs = (
href
@@ -289,7 +289,11 @@ def iter_targets(
def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
+ """Extract stable reader-facing content without generated HTML attributes.
+
+ :param content: Parsed e-content element to normalize.
+ :returns: Normalized text used to fingerprint reader-facing content.
+ """
def extract(node: PageElement) -> str:
if isinstance(node, NavigableString):
@@ -326,9 +330,9 @@ def scan_source_file(
:param root: Root directory containing generated source documents.
:param base_url: Configured source base URL, if available.
:param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
+ :returns: Scanned source metadata and discovered targets.
:raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
+ Webmention source.
:raises OSError: If the source document cannot be read.
"""
# Use the path as published for its URL, but read only files whose