aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/consumer.py25
-rw-r--r--webmentions_ssg/tasks/extension.py93
-rw-r--r--webmentions_ssg/tasks/notifications.py8
-rw-r--r--webmentions_ssg/tasks/receiver.py2
-rw-r--r--webmentions_ssg/tasks/scanner.py4
-rw-r--r--webmentions_ssg/tasks/sender.py2
6 files changed, 101 insertions, 33 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 047c010..474e775 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -6,45 +6,40 @@
from huey import Huey, crontab
-from .. import create_app, extensions
+from .. import create_app
+from ..extensions import huey as queue
+from . import notifications, receiver, retention, scanner
def create_consumer() -> Huey:
"""Create and configure the Huey consumer.
- The application is initialized, task modules are imported to register their
- tasks, and periodic scanner, notification and verification recovery tasks
- are registered.
+ The application is initialized and the periodic scanner, notification,
+ verification recovery and retention tasks are registered.
:return: Configured Huey instance.
"""
app = create_app()
- from . import notifications, receiver, retention, scanner
-
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
- extensions.huey.periodic_task(schedule)(scanner.scan_sources)
+ queue.periodic_task(schedule)(scanner.scan_sources)
recovery_schedule = crontab(
*app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True
)
- extensions.huey.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
+ queue.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
notification_schedule = crontab(
*app.config["NOTIFICATION_SCHEDULE"].split(), strict=True
)
- extensions.huey.periodic_task(notification_schedule)(
- notifications.send_webmention_digest
- )
+ queue.periodic_task(notification_schedule)(notifications.send_webmention_digest)
retention_schedule = crontab(
*app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True
)
- extensions.huey.periodic_task(retention_schedule)(
- retention.purge_old_received_webmentions
- )
+ queue.periodic_task(retention_schedule)(retention.purge_old_received_webmentions)
- return extensions.huey.huey
+ return queue.huey
huey = create_consumer()
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index ee9c442..0e031e6 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -4,8 +4,10 @@
"""Integrate Huey tasks with Flask application contexts."""
+from __future__ import annotations
+
from collections.abc import Callable
-from functools import wraps
+from functools import update_wrapper, wraps
from typing import Any, ParamSpec, TypeVar
from urllib.parse import urlsplit, urlunsplit
@@ -19,6 +21,55 @@ P = ParamSpec("P")
R = TypeVar("R")
+class _LazyTask:
+ """Stable public task handle which follows the extension's current backend.
+
+ Huey creates task classes and registers them when its decorator is applied.
+ Keep that backend-specific TaskWrapper private, so imported task handles can
+ be used across Flask app factories without referring to an obsolete queue.
+ """
+
+ def __init__(
+ self,
+ extension: Huey,
+ func: Callable[..., Any],
+ args: tuple[Any, ...],
+ kwargs: dict[str, Any],
+ *,
+ periodic: bool,
+ ) -> None:
+ self._extension = extension
+ self._func = func
+ self._args = args
+ self._kwargs = kwargs
+ self._periodic = periodic
+ self._registered: TaskWrapper | None = None
+ update_wrapper(self, func)
+
+ def register(self) -> None:
+ """Bind the original function to the latest initialized backend."""
+ backend = self._extension.huey
+ decorator = backend.periodic_task if self._periodic else backend.task
+ self._registered = decorator(*self._args, **self._kwargs)(
+ self._extension._in_app_context(self._func)
+ )
+
+ def _current(self) -> TaskWrapper:
+ if self._registered is None:
+ raise RuntimeError(
+ "Huey has not been initialized. Call huey.init_app(app) first."
+ )
+ return self._registered
+
+ def __call__(self, *args: Any, **kwargs: Any) -> Any:
+ """Enqueue a task using the active backend."""
+ return self._current()(*args, **kwargs)
+
+ def __getattr__(self, name: str) -> Any:
+ """Delegate Huey task operations (call_local, schedule, s, etc.)."""
+ return getattr(self._current(), name)
+
+
class Huey:
"""Provide Flask integration for a Huey instance.
@@ -33,6 +84,7 @@ class Huey:
"""
self.app: Flask | None = None
self._huey: BaseHuey | None = None
+ self._tasks: list[_LazyTask] = []
if app is not None:
self.init_app(app)
@@ -73,6 +125,11 @@ class Huey:
self.app = app
self._huey = huey_class(**config, **storage_kwargs)
+ # Import order is immaterial: tasks declared before initialization are
+ # registered now, and previously imported handles follow this backend.
+ for task in self._tasks:
+ task.register()
+
app.extensions["huey"] = self
@property
@@ -85,7 +142,7 @@ class Huey:
if self._huey is None:
raise RuntimeError(
"Huey has not been initialized. "
- "Call huey.init_app(app) before importing tasks."
+ "Call huey.init_app(app) before using tasks."
)
return self._huey
@@ -101,25 +158,37 @@ class Huey:
return wrapped
+ def _decorate_task(
+ self,
+ func: Callable[P, R],
+ task_args: tuple[Any, ...],
+ task_kwargs: dict[str, Any],
+ *,
+ periodic: bool,
+ ) -> _LazyTask:
+ handle = _LazyTask(self, func, task_args, task_kwargs, periodic=periodic)
+ self._tasks.append(handle)
+ if self._huey is not None:
+ handle.register()
+ return handle
+
def task(
self, *task_args: Any, **task_kwargs: Any
- ) -> Callable[[Callable[P, object]], TaskWrapper]:
- """Return a Huey task decorator with Flask application context."""
+ ) -> Callable[[Callable[P, R]], _LazyTask]:
+ """Declare a task even before :meth:`init_app` has run."""
- def decorator(func: Callable[P, R]) -> TaskWrapper:
- return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func))
+ def decorator(func: Callable[P, R]) -> _LazyTask:
+ return self._decorate_task(func, task_args, task_kwargs, periodic=False)
return decorator
def periodic_task(
self, *task_args: Any, **task_kwargs: Any
- ) -> Callable[[Callable[P, object]], TaskWrapper]:
- """Return a periodic Huey decorator with Flask application context."""
+ ) -> Callable[[Callable[P, R]], _LazyTask]:
+ """Declare a periodic task against the current or next backend."""
- def decorator(func: Callable[P, R]) -> TaskWrapper:
- return self.huey.periodic_task(*task_args, **task_kwargs)(
- self._in_app_context(func)
- )
+ def decorator(func: Callable[P, R]) -> _LazyTask:
+ return self._decorate_task(func, task_args, task_kwargs, periodic=True)
return decorator
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index fe12e93..fc1b475 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -14,9 +14,15 @@ from ..extensions import db, huey, mail
from ..models import ReceivedStatus, ReceivedWebmention
-@huey.lock_task("send-webmention-digest")
def send_webmention_digest() -> None:
"""Send one digest of verified, previously unreported Webmentions."""
+ # Resolve the lock against the current Huey backend, not the import-time one.
+ with huey.lock_task("send-webmention-digest"):
+ _send_webmention_digest()
+
+
+def _send_webmention_digest() -> None:
+ """Do the work while the current backend's digest lock is held."""
recipient = current_app.config.get("NOTIFICATION_EMAIL")
if not recipient:
return
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 4c4134c..2963dc7 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -12,7 +12,6 @@ import httpx
import sqlalchemy as sa
from flask import current_app
-from .. import APP_NAME, VERSION
from ..extensions import db, huey
from ..models import ReceivedStatus, ReceivedWebmention
from ..net import (
@@ -22,6 +21,7 @@ from ..net import (
is_temporary_status,
read_limited_body,
)
+from ..version import APP_NAME, VERSION
from .mentions import html_mentions_target, text_mentions_target
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 353d538..263b8a5 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -110,9 +110,7 @@ def scan_sources() -> None:
:raises RuntimeError: If the configured source directory or base URL is
invalid.
"""
- # Resolve the lock at execution time. The global Huey extension can be
- # reinitialized by Flask app factories (notably in the test suite), so an
- # import-time lock decorator may point at an obsolete storage backend.
+ # Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
scan_sources_task()
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 463208d..db99476 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -13,7 +13,6 @@ import sqlalchemy as sa
from flask import current_app
from huey.exceptions import TaskLockedException
-from .. import APP_NAME, VERSION
from ..extensions import db, huey
from ..models import SentWebmention, SentWebmentionStatus
from ..net import (
@@ -23,6 +22,7 @@ from ..net import (
is_http_url,
is_temporary_status,
)
+from ..version import APP_NAME, VERSION
from .discovery import (
PermanentSenderError,
TemporarySenderError,