diff options
| author | Dennis Fink | 2026-11-16 09:27:29 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-16 09:27:29 +0100 |
| commit | 85b10c38f39732421a230d938adddc7805d0e001 (patch) | |
| tree | 396f29962fac45245c6f1486e188ad905588659d | |
| parent | eb404d9a1a328278c92753e65100579e7ad98e48 (diff) | |
| download | webmentions-ssg-85b10c38f39732421a230d938adddc7805d0e001.tar.gz webmentions-ssg-85b10c38f39732421a230d938adddc7805d0e001.zip | |
style(core): reorder module declarations
Group globals and data classes near module imports, keep primary classes
and functions together, and move implementation helpers after their
public callers.
Reorder Alembic setup so configuration happens after the required
helpers are defined, without changing runtime behavior.
Diffstat (limited to '')
| -rw-r--r-- | tests/test_net.py | 62 | ||||
| -rw-r--r-- | webmentions_ssg/cli.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/env.py | 21 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 144 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 16 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 56 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/verification.py | 22 |
7 files changed, 160 insertions, 169 deletions
diff --git a/tests/test_net.py b/tests/test_net.py index 24785ee..472ae91 100644 --- a/tests/test_net.py +++ b/tests/test_net.py @@ -31,6 +31,37 @@ from webmentions_ssg.net import ( ) +class FakeStream(httpcore.NetworkStream): + """In-memory TCP/TLS stream so HTTP framing and SNI can be tested.""" + + def __init__(self, response: bytes) -> None: + self.response = response + self.writes = bytearray() + self.sni: str | None = None + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + part, self.response = self.response[:max_bytes], self.response[max_bytes:] + return part + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self.writes.extend(buffer) + + def start_tls( + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> httpcore.NetworkStream: + self.sni = server_hostname + return self + + def get_extra_info(self, info: str): + return None + + def close(self) -> None: + pass + + @pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"]) def test_backend_rejects_private_ip_without_socket( monkeypatch: pytest.MonkeyPatch, host: str @@ -125,37 +156,6 @@ def test_read_limited_body_raises_specific_size_error(mode: str) -> None: read_limited_body(response, 10, label="Source") -class FakeStream(httpcore.NetworkStream): - """In-memory TCP/TLS stream so HTTP framing and SNI can be tested.""" - - def __init__(self, response: bytes) -> None: - self.response = response - self.writes = bytearray() - self.sni: str | None = None - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - part, self.response = self.response[:max_bytes], self.response[max_bytes:] - return part - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - self.writes.extend(buffer) - - def start_tls( - self, - ssl_context, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> httpcore.NetworkStream: - self.sni = server_hostname - return self - - def get_extra_info(self, info: str): - return None - - def close(self) -> None: - pass - - @pytest.mark.parametrize("scheme", ["http", "https"]) def test_transport_keeps_original_http_host_and_tls_sni( monkeypatch: pytest.MonkeyPatch, scheme: str diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index 225a39f..3448bbb 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -18,6 +18,8 @@ from .extensions import db, mail from .models import User USERS_CLI = AppGroup("users", help="Manage users.") +MAIL_CLI = AppGroup("mail", help="Handle mail") +DB_CLI = AppGroup("db", help="Handle database") def _get_user_or_fail(username: str): @@ -118,9 +120,6 @@ def delete_user(username: str, yes: bool) -> None: click.echo(f"Deleted user {username!r}.") -MAIL_CLI = AppGroup("mail", help="Handle mail") - - @MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") @click.option("--to", "recipient", help="Recipient address (defaults to DIGEST_EMAIL).") def send_test_mail(recipient: str | None) -> None: @@ -157,9 +156,6 @@ def send_test_mail(recipient: str | None) -> None: click.echo(f"Test email sent to {recipient}.") -DB_CLI = AppGroup("db", help="Handle database") - - def _existing_table_names() -> set[str]: """Return existing table names without modifying the database schema. diff --git a/webmentions_ssg/migrations/env.py b/webmentions_ssg/migrations/env.py index 86d9d99..bd66546 100644 --- a/webmentions_ssg/migrations/env.py +++ b/webmentions_ssg/migrations/env.py @@ -17,6 +17,10 @@ config = context.config logger = logging.getLogger("alembic.env") +# Database metadata shared by both migration modes. +target_db = current_app.extensions["migrate"].db + + def get_engine(): return current_app.extensions["migrate"].db.engine @@ -28,19 +32,6 @@ def get_engine_url(): return str(get_engine().url).replace("%", "%%") -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option("sqlalchemy.url", get_engine_url()) -target_db = current_app.extensions["migrate"].db - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - def get_metadata(): if hasattr(target_db, "metadatas"): return target_db.metadatas[None] @@ -97,6 +88,10 @@ def run_migrations_online(): context.run_migrations() +# Configure Alembic after the engine URL helper is defined. +config.set_main_option("sqlalchemy.url", get_engine_url()) + + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index 3f172f4..a878e16 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -16,78 +16,6 @@ from huey.api import TaskWrapper from .json_serializer import TaggedJSONSerializer -def _redis_class_and_options( - parsed: SplitResult, -) -> tuple[type[BaseHuey], dict[str, Any]]: - """Select a Redis Huey flavour while retaining the underlying URL scheme. - - :param parsed: Parsed Redis backend URL, including any flavour suffix. - :returns: Huey class and constructor options for the requested Redis flavour. - :raises RuntimeError: If the Redis backend flavour is unsupported. - """ - from huey import ( - PriorityRedisExpireHuey, - PriorityRedisHuey, - RedisExpireHuey, - RedisHuey, - ) - - scheme = parsed.scheme.lower() - base_scheme, separator, flavour = scheme.partition("+") - backends = { - "": RedisHuey, - "priority": PriorityRedisHuey, - "expire": RedisExpireHuey, - "priority+expire": PriorityRedisExpireHuey, - } - if flavour not in backends or (separator and not flavour): - raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") - - fixed_url = urlunsplit(parsed._replace(scheme=base_scheme)) - return backends[flavour], {"url": fixed_url} - - -def _file_class_and_options( - url: str, scheme: str -) -> tuple[type[BaseHuey], dict[str, Any]]: - """Extract a relative or absolute SQLite/file storage path. - - :param url: Backend URL containing the storage filename or path. - :param scheme: Selected file or SQLite backend scheme. - :returns: Huey class and constructor options for the file-backed queue. - :raises RuntimeError: If the storage URL is malformed or omits the path. - """ - from huey import FileHuey, SqliteHuey - - if scheme == "sqlite": - backend, option, description, example = ( - SqliteHuey, - "filename", - "SQLite Huey", - "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)", - ) - missing = "SQLite Huey URL must include a database path." - else: - backend, option, description, example = ( - FileHuey, - "path", - "File Huey", - "file:///huey-queue (relative) or file:////var/huey-queue (absolute)", - ) - missing = "File Huey URL must include a directory path." - - prefix = f"{scheme}:///" - if not url.startswith(prefix): - raise RuntimeError(f"{description} URLs must look like {example}") - - # Four slashes retain the initial '/' of an absolute path; three do not. - path = url.removeprefix(prefix) - if not path: - raise RuntimeError(missing) - - return backend, {option: path} - - class _DeferredTask: """Stable public task handle which follows the extension's current backend. @@ -328,3 +256,75 @@ class FlaskHuey: return BlackHoleHuey, {} raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") + + +def _redis_class_and_options( + parsed: SplitResult, +) -> tuple[type[BaseHuey], dict[str, Any]]: + """Select a Redis Huey flavour while retaining the underlying URL scheme. + + :param parsed: Parsed Redis backend URL, including any flavour suffix. + :returns: Huey class and constructor options for the requested Redis flavour. + :raises RuntimeError: If the Redis backend flavour is unsupported. + """ + from huey import ( + PriorityRedisExpireHuey, + PriorityRedisHuey, + RedisExpireHuey, + RedisHuey, + ) + + scheme = parsed.scheme.lower() + base_scheme, separator, flavour = scheme.partition("+") + backends = { + "": RedisHuey, + "priority": PriorityRedisHuey, + "expire": RedisExpireHuey, + "priority+expire": PriorityRedisExpireHuey, + } + if flavour not in backends or (separator and not flavour): + raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") + + fixed_url = urlunsplit(parsed._replace(scheme=base_scheme)) + return backends[flavour], {"url": fixed_url} + + +def _file_class_and_options( + url: str, scheme: str +) -> tuple[type[BaseHuey], dict[str, Any]]: + """Extract a relative or absolute SQLite/file storage path. + + :param url: Backend URL containing the storage filename or path. + :param scheme: Selected file or SQLite backend scheme. + :returns: Huey class and constructor options for the file-backed queue. + :raises RuntimeError: If the storage URL is malformed or omits the path. + """ + from huey import FileHuey, SqliteHuey + + if scheme == "sqlite": + backend, option, description, example = ( + SqliteHuey, + "filename", + "SQLite Huey", + "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)", + ) + missing = "SQLite Huey URL must include a database path." + else: + backend, option, description, example = ( + FileHuey, + "path", + "File Huey", + "file:///huey-queue (relative) or file:////var/huey-queue (absolute)", + ) + missing = "File Huey URL must include a directory path." + + prefix = f"{scheme}:///" + if not url.startswith(prefix): + raise RuntimeError(f"{description} URLs must look like {example}") + + # Four slashes retain the initial '/' of an absolute path; three do not. + path = url.removeprefix(prefix) + if not path: + raise RuntimeError(missing) + + return backend, {option: path} diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index fb0c770..6869477 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -20,6 +20,14 @@ from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file +@dataclass(frozen=True) +class TreeScanResult: + """Count and discovered paths from a traversal of the source tree.""" + + scanned_count: int + seen_paths: frozenset[str] + + def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. @@ -118,14 +126,6 @@ def scan_sources() -> None: _scan_and_enqueue() -@dataclass(frozen=True) -class TreeScanResult: - """Count and discovered paths from a traversal of the source tree.""" - - scanned_count: int - seen_paths: frozenset[str] - - def reconcile_source_tree( root: Path, base_url: str | None, diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 3f64bed..6b14ace 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -32,6 +32,34 @@ from .endpoint_discovery import ( ) +@dataclass(frozen=True) +class ClaimedSendAttempt: + """Claimed source revision and retry budget for one send.""" + + webmention_uuid: uuid.UUID + revision: int + attempt_number: int + max_attempts: int + source: str + target: str + + +@dataclass(frozen=True) +class DeliveryResult: + """HTTP outcome, with whether the failure is eligible for another attempt. + + ``retryable`` describes the failure type; it does not account for the + remaining retry budget, which may already have been exhausted. + """ + + status: SentWebmentionStatus + reason: str | None = None + endpoint: str | None = None + response_status: int | None = None + status_url: str | None = None + retryable: bool = False + + def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """Check whether a send attempt still represents the desired revision. @@ -83,34 +111,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return -@dataclass(frozen=True) -class ClaimedSendAttempt: - """Claimed source revision and retry budget for one send.""" - - webmention_uuid: uuid.UUID - revision: int - attempt_number: int - max_attempts: int - source: str - target: str - - -@dataclass(frozen=True) -class DeliveryResult: - """HTTP outcome, with whether the failure is eligible for another attempt. - - ``retryable`` describes the failure type; it does not account for the - remaining retry budget, which may already have been exhausted. - """ - - status: SentWebmentionStatus - reason: str | None = None - endpoint: str | None = None - response_status: int | None = None - status_url: str | None = None - retryable: bool = False - - def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: """Return whether an unprocessed Webmention has reached its retry deadline. diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index a614815..b0f4d0f 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -29,6 +29,9 @@ from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -41,7 +44,14 @@ class RetryableFetchError(Exception): """Fetching the source may succeed when retried later.""" -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +@dataclass(frozen=True) +class VerificationClaim: + """Source and lease version for one atomically claimed verification.""" + + webmention_uuid: uuid.UUID + source: str + target: str + lease_started_at: datetime def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: @@ -125,16 +135,6 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: ) -@dataclass(frozen=True) -class VerificationClaim: - """Source and lease version for one atomically claimed verification.""" - - webmention_uuid: uuid.UUID - source: str - target: str - lease_started_at: datetime - - def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: """Claim an eligible row, returning its source, target, and lease version. |
