aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/cli.py8
-rw-r--r--webmentions_ssg/migrations/env.py21
-rw-r--r--webmentions_ssg/tasks/extension.py144
-rw-r--r--webmentions_ssg/tasks/scanner.py16
-rw-r--r--webmentions_ssg/tasks/sender.py56
-rw-r--r--webmentions_ssg/tasks/verification.py22
6 files changed, 129 insertions, 138 deletions
diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py
index 225a39f..3448bbb 100644
--- a/webmentions_ssg/cli.py
+++ b/webmentions_ssg/cli.py
@@ -18,6 +18,8 @@ from .extensions import db, mail
from .models import User
USERS_CLI = AppGroup("users", help="Manage users.")
+MAIL_CLI = AppGroup("mail", help="Handle mail")
+DB_CLI = AppGroup("db", help="Handle database")
def _get_user_or_fail(username: str):
@@ -118,9 +120,6 @@ def delete_user(username: str, yes: bool) -> None:
click.echo(f"Deleted user {username!r}.")
-MAIL_CLI = AppGroup("mail", help="Handle mail")
-
-
@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration")
@click.option("--to", "recipient", help="Recipient address (defaults to DIGEST_EMAIL).")
def send_test_mail(recipient: str | None) -> None:
@@ -157,9 +156,6 @@ def send_test_mail(recipient: str | None) -> None:
click.echo(f"Test email sent to {recipient}.")
-DB_CLI = AppGroup("db", help="Handle database")
-
-
def _existing_table_names() -> set[str]:
"""Return existing table names without modifying the database schema.
diff --git a/webmentions_ssg/migrations/env.py b/webmentions_ssg/migrations/env.py
index 86d9d99..bd66546 100644
--- a/webmentions_ssg/migrations/env.py
+++ b/webmentions_ssg/migrations/env.py
@@ -17,6 +17,10 @@ config = context.config
logger = logging.getLogger("alembic.env")
+# Database metadata shared by both migration modes.
+target_db = current_app.extensions["migrate"].db
+
+
def get_engine():
return current_app.extensions["migrate"].db.engine
@@ -28,19 +32,6 @@ def get_engine_url():
return str(get_engine().url).replace("%", "%%")
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-config.set_main_option("sqlalchemy.url", get_engine_url())
-target_db = current_app.extensions["migrate"].db
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
def get_metadata():
if hasattr(target_db, "metadatas"):
return target_db.metadatas[None]
@@ -97,6 +88,10 @@ def run_migrations_online():
context.run_migrations()
+# Configure Alembic after the engine URL helper is defined.
+config.set_main_option("sqlalchemy.url", get_engine_url())
+
+
if context.is_offline_mode():
run_migrations_offline()
else:
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index 3f172f4..a878e16 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -16,78 +16,6 @@ from huey.api import TaskWrapper
from .json_serializer import TaggedJSONSerializer
-def _redis_class_and_options(
- parsed: SplitResult,
-) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Select a Redis Huey flavour while retaining the underlying URL scheme.
-
- :param parsed: Parsed Redis backend URL, including any flavour suffix.
- :returns: Huey class and constructor options for the requested Redis flavour.
- :raises RuntimeError: If the Redis backend flavour is unsupported.
- """
- from huey import (
- PriorityRedisExpireHuey,
- PriorityRedisHuey,
- RedisExpireHuey,
- RedisHuey,
- )
-
- scheme = parsed.scheme.lower()
- base_scheme, separator, flavour = scheme.partition("+")
- backends = {
- "": RedisHuey,
- "priority": PriorityRedisHuey,
- "expire": RedisExpireHuey,
- "priority+expire": PriorityRedisExpireHuey,
- }
- if flavour not in backends or (separator and not flavour):
- raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}")
-
- fixed_url = urlunsplit(parsed._replace(scheme=base_scheme))
- return backends[flavour], {"url": fixed_url}
-
-
-def _file_class_and_options(
- url: str, scheme: str
-) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Extract a relative or absolute SQLite/file storage path.
-
- :param url: Backend URL containing the storage filename or path.
- :param scheme: Selected file or SQLite backend scheme.
- :returns: Huey class and constructor options for the file-backed queue.
- :raises RuntimeError: If the storage URL is malformed or omits the path.
- """
- from huey import FileHuey, SqliteHuey
-
- if scheme == "sqlite":
- backend, option, description, example = (
- SqliteHuey,
- "filename",
- "SQLite Huey",
- "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)",
- )
- missing = "SQLite Huey URL must include a database path."
- else:
- backend, option, description, example = (
- FileHuey,
- "path",
- "File Huey",
- "file:///huey-queue (relative) or file:////var/huey-queue (absolute)",
- )
- missing = "File Huey URL must include a directory path."
-
- prefix = f"{scheme}:///"
- if not url.startswith(prefix):
- raise RuntimeError(f"{description} URLs must look like {example}")
-
- # Four slashes retain the initial '/' of an absolute path; three do not.
- path = url.removeprefix(prefix)
- if not path:
- raise RuntimeError(missing)
-
- return backend, {option: path}
-
-
class _DeferredTask:
"""Stable public task handle which follows the extension's current backend.
@@ -328,3 +256,75 @@ class FlaskHuey:
return BlackHoleHuey, {}
raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}")
+
+
+def _redis_class_and_options(
+ parsed: SplitResult,
+) -> tuple[type[BaseHuey], dict[str, Any]]:
+ """Select a Redis Huey flavour while retaining the underlying URL scheme.
+
+ :param parsed: Parsed Redis backend URL, including any flavour suffix.
+ :returns: Huey class and constructor options for the requested Redis flavour.
+ :raises RuntimeError: If the Redis backend flavour is unsupported.
+ """
+ from huey import (
+ PriorityRedisExpireHuey,
+ PriorityRedisHuey,
+ RedisExpireHuey,
+ RedisHuey,
+ )
+
+ scheme = parsed.scheme.lower()
+ base_scheme, separator, flavour = scheme.partition("+")
+ backends = {
+ "": RedisHuey,
+ "priority": PriorityRedisHuey,
+ "expire": RedisExpireHuey,
+ "priority+expire": PriorityRedisExpireHuey,
+ }
+ if flavour not in backends or (separator and not flavour):
+ raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}")
+
+ fixed_url = urlunsplit(parsed._replace(scheme=base_scheme))
+ return backends[flavour], {"url": fixed_url}
+
+
+def _file_class_and_options(
+ url: str, scheme: str
+) -> tuple[type[BaseHuey], dict[str, Any]]:
+ """Extract a relative or absolute SQLite/file storage path.
+
+ :param url: Backend URL containing the storage filename or path.
+ :param scheme: Selected file or SQLite backend scheme.
+ :returns: Huey class and constructor options for the file-backed queue.
+ :raises RuntimeError: If the storage URL is malformed or omits the path.
+ """
+ from huey import FileHuey, SqliteHuey
+
+ if scheme == "sqlite":
+ backend, option, description, example = (
+ SqliteHuey,
+ "filename",
+ "SQLite Huey",
+ "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)",
+ )
+ missing = "SQLite Huey URL must include a database path."
+ else:
+ backend, option, description, example = (
+ FileHuey,
+ "path",
+ "File Huey",
+ "file:///huey-queue (relative) or file:////var/huey-queue (absolute)",
+ )
+ missing = "File Huey URL must include a directory path."
+
+ prefix = f"{scheme}:///"
+ if not url.startswith(prefix):
+ raise RuntimeError(f"{description} URLs must look like {example}")
+
+ # Four slashes retain the initial '/' of an absolute path; three do not.
+ path = url.removeprefix(prefix)
+ if not path:
+ raise RuntimeError(missing)
+
+ return backend, {option: path}
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index fb0c770..6869477 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -20,6 +20,14 @@ from .sender import send_webmention
from .source_parser import ScannedSource, SourceScanError, scan_source_file
+@dataclass(frozen=True)
+class TreeScanResult:
+ """Count and discovered paths from a traversal of the source tree."""
+
+ scanned_count: int
+ seen_paths: frozenset[str]
+
+
def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument:
"""Build a source model from a newly discovered document.
@@ -118,14 +126,6 @@ def scan_sources() -> None:
_scan_and_enqueue()
-@dataclass(frozen=True)
-class TreeScanResult:
- """Count and discovered paths from a traversal of the source tree."""
-
- scanned_count: int
- seen_paths: frozenset[str]
-
-
def reconcile_source_tree(
root: Path,
base_url: str | None,
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 3f64bed..6b14ace 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -32,6 +32,34 @@ from .endpoint_discovery import (
)
+@dataclass(frozen=True)
+class ClaimedSendAttempt:
+ """Claimed source revision and retry budget for one send."""
+
+ webmention_uuid: uuid.UUID
+ revision: int
+ attempt_number: int
+ max_attempts: int
+ source: str
+ target: str
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+ """HTTP outcome, with whether the failure is eligible for another attempt.
+
+ ``retryable`` describes the failure type; it does not account for the
+ remaining retry budget, which may already have been exhausted.
+ """
+
+ status: SentWebmentionStatus
+ reason: str | None = None
+ endpoint: str | None = None
+ response_status: int | None = None
+ status_url: str | None = None
+ retryable: bool = False
+
+
def revision_still_pending(webmention: SentWebmention, revision: int) -> bool:
"""Check whether a send attempt still represents the desired revision.
@@ -83,34 +111,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
-@dataclass(frozen=True)
-class ClaimedSendAttempt:
- """Claimed source revision and retry budget for one send."""
-
- webmention_uuid: uuid.UUID
- revision: int
- attempt_number: int
- max_attempts: int
- source: str
- target: str
-
-
-@dataclass(frozen=True)
-class DeliveryResult:
- """HTTP outcome, with whether the failure is eligible for another attempt.
-
- ``retryable`` describes the failure type; it does not account for the
- remaining retry budget, which may already have been exhausted.
- """
-
- status: SentWebmentionStatus
- reason: str | None = None
- endpoint: str | None = None
- response_status: int | None = None
- status_url: str | None = None
- retryable: bool = False
-
-
def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool:
"""Return whether an unprocessed Webmention has reached its retry deadline.
diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py
index a614815..b0f4d0f 100644
--- a/webmentions_ssg/tasks/verification.py
+++ b/webmentions_ssg/tasks/verification.py
@@ -29,6 +29,9 @@ from ..version import APP_NAME, VERSION
from .target_matching import html_mentions_target, text_mentions_target
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+
+
class VerificationError(Exception):
"""The source permanently failed ReceivedWebmention verification."""
@@ -41,7 +44,14 @@ class RetryableFetchError(Exception):
"""Fetching the source may succeed when retried later."""
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+@dataclass(frozen=True)
+class VerificationClaim:
+ """Source and lease version for one atomically claimed verification."""
+
+ webmention_uuid: uuid.UUID
+ source: str
+ target: str
+ lease_started_at: datetime
def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]:
@@ -125,16 +135,6 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
)
-@dataclass(frozen=True)
-class VerificationClaim:
- """Source and lease version for one atomically claimed verification."""
-
- webmention_uuid: uuid.UUID
- source: str
- target: str
- lease_started_at: datetime
-
-
def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
"""Claim an eligible row, returning its source, target, and lease version.