diff options
| -rw-r--r-- | tests/test_net.py | 62 | ||||
| -rw-r--r-- | webmentions_ssg/cli.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/env.py | 21 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 144 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 16 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 56 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/verification.py | 22 |
7 files changed, 160 insertions, 169 deletions
diff --git a/tests/test_net.py b/tests/test_net.py index 24785ee..472ae91 100644 --- a/tests/test_net.py +++ b/tests/test_net.py @@ -31,6 +31,37 @@ from webmentions_ssg.net import ( ) +class FakeStream(httpcore.NetworkStream): + """In-memory TCP/TLS stream so HTTP framing and SNI can be tested.""" + + def __init__(self, response: bytes) -> None: + self.response = response + self.writes = bytearray() + self.sni: str | None = None + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + part, self.response = self.response[:max_bytes], self.response[max_bytes:] + return part + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self.writes.extend(buffer) + + def start_tls( + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> httpcore.NetworkStream: + self.sni = server_hostname + return self + + def get_extra_info(self, info: str): + return None + + def close(self) -> None: + pass + + @pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"]) def test_backend_rejects_private_ip_without_socket( monkeypatch: pytest.MonkeyPatch, host: str @@ -125,37 +156,6 @@ def test_read_limited_body_raises_specific_size_error(mode: str) -> None: read_limited_body(response, 10, label="Source") -class FakeStream(httpcore.NetworkStream): - """In-memory TCP/TLS stream so HTTP framing and SNI can be tested.""" - - def __init__(self, response: bytes) -> None: - self.response = response - self.writes = bytearray() - self.sni: str | None = None - - def read(self, max_bytes: int, timeout: float | None = None) -> bytes: - part, self.response = self.response[:max_bytes], self.response[max_bytes:] - return part - - def write(self, buffer: bytes, timeout: float | None = None) -> None: - self.writes.extend(buffer) - - def start_tls( - self, - ssl_context, - server_hostname: str | None = None, - timeout: float | None = None, - ) -> httpcore.NetworkStream: - self.sni = server_hostname - return self - - def get_extra_info(self, info: str): - return None - - def close(self) -> None: - pass - - @pytest.mark.parametrize("scheme", ["http", "https"]) def test_transport_keeps_original_http_host_and_tls_sni( monkeypatch: pytest.MonkeyPatch, scheme: str diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index 225a39f..3448bbb 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -18,6 +18,8 @@ from .extensions import db, mail from .models import User USERS_CLI = AppGroup("users", help="Manage users.") +MAIL_CLI = AppGroup("mail", help="Handle mail") +DB_CLI = AppGroup("db", help="Handle database") def _get_user_or_fail(username: str): @@ -118,9 +120,6 @@ def delete_user(username: str, yes: bool) -> None: click.echo(f"Deleted user {username!r}.") -MAIL_CLI = AppGroup("mail", help="Handle mail") - - @MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") @click.option("--to", "recipient", help="Recipient address (defaults to DIGEST_EMAIL).") def send_test_mail(recipient: str | None) -> None: @@ -157,9 +156,6 @@ def send_test_mail(recipient: str | None) -> None: click.echo(f"Test email sent to {recipient}.") -DB_CLI = AppGroup("db", help="Handle database") - - def _existing_table_names() -> set[str]: """Return existing table names without modifying the database schema. diff --git a/webmentions_ssg/migrations/env.py b/webmentions_ssg/migrations/env.py index 86d9d99..bd66546 100644 --- a/webmentions_ssg/migrations/env.py +++ b/webmentions_ssg/migrations/env.py @@ -17,6 +17,10 @@ config = context.config logger = logging.getLogger("alembic.env") +# Database metadata shared by both migration modes. +target_db = current_app.extensions["migrate"].db + + def get_engine(): return current_app.extensions["migrate"].db.engine @@ -28,19 +32,6 @@ def get_engine_url(): return str(get_engine().url).replace("%", "%%") -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option("sqlalchemy.url", get_engine_url()) -target_db = current_app.extensions["migrate"].db - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - def get_metadata(): if hasattr(target_db, "metadatas"): return target_db.metadatas[None] @@ -97,6 +88,10 @@ def run_migrations_online(): context.run_migrations() +# Configure Alembic after the engine URL helper is defined. +config.set_main_option("sqlalchemy.url", get_engine_url()) + + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index 3f172f4..a878e16 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -16,78 +16,6 @@ from huey.api import TaskWrapper from .json_serializer import TaggedJSONSerializer -def _redis_class_and_options( - parsed: SplitResult, -) -> tuple[type[BaseHuey], dict[str, Any]]: - """Select a Redis Huey flavour while retaining the underlying URL scheme. - - :param parsed: Parsed Redis backend URL, including any flavour suffix. - :returns: Huey class and constructor options for the requested Redis flavour. - :raises RuntimeError: If the Redis backend flavour is unsupported. - """ - from huey import ( - PriorityRedisExpireHuey, - PriorityRedisHuey, - RedisExpireHuey, - RedisHuey, - ) - - scheme = parsed.scheme.lower() - base_scheme, separator, flavour = scheme.partition("+") - backends = { - "": RedisHuey, - "priority": PriorityRedisHuey, - "expire": RedisExpireHuey, - "priority+expire": PriorityRedisExpireHuey, - } - if flavour not in backends or (separator and not flavour): - raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") - - fixed_url = urlunsplit(parsed._replace(scheme=base_scheme)) - return backends[flavour], {"url": fixed_url} - - -def _file_class_and_options( - url: str, scheme: str -) -> tuple[type[BaseHuey], dict[str, Any]]: - """Extract a relative or absolute SQLite/file storage path. - - :param url: Backend URL containing the storage filename or path. - :param scheme: Selected file or SQLite backend scheme. - :returns: Huey class and constructor options for the file-backed queue. - :raises RuntimeError: If the storage URL is malformed or omits the path. - """ - from huey import FileHuey, SqliteHuey - - if scheme == "sqlite": - backend, option, description, example = ( - SqliteHuey, - "filename", - "SQLite Huey", - "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)", - ) - missing = "SQLite Huey URL must include a database path." - else: - backend, option, description, example = ( - FileHuey, - "path", - "File Huey", - "file:///huey-queue (relative) or file:////var/huey-queue (absolute)", - ) - missing = "File Huey URL must include a directory path." - - prefix = f"{scheme}:///" - if not url.startswith(prefix): - raise RuntimeError(f"{description} URLs must look like {example}") - - # Four slashes retain the initial '/' of an absolute path; three do not. - path = url.removeprefix(prefix) - if not path: - raise RuntimeError(missing) - - return backend, {option: path} - - class _DeferredTask: """Stable public task handle which follows the extension's current backend. @@ -328,3 +256,75 @@ class FlaskHuey: return BlackHoleHuey, {} raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") + + +def _redis_class_and_options( + parsed: SplitResult, +) -> tuple[type[BaseHuey], dict[str, Any]]: + """Select a Redis Huey flavour while retaining the underlying URL scheme. + + :param parsed: Parsed Redis backend URL, including any flavour suffix. + :returns: Huey class and constructor options for the requested Redis flavour. + :raises RuntimeError: If the Redis backend flavour is unsupported. + """ + from huey import ( + PriorityRedisExpireHuey, + PriorityRedisHuey, + RedisExpireHuey, + RedisHuey, + ) + + scheme = parsed.scheme.lower() + base_scheme, separator, flavour = scheme.partition("+") + backends = { + "": RedisHuey, + "priority": PriorityRedisHuey, + "expire": RedisExpireHuey, + "priority+expire": PriorityRedisExpireHuey, + } + if flavour not in backends or (separator and not flavour): + raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") + + fixed_url = urlunsplit(parsed._replace(scheme=base_scheme)) + return backends[flavour], {"url": fixed_url} + + +def _file_class_and_options( + url: str, scheme: str +) -> tuple[type[BaseHuey], dict[str, Any]]: + """Extract a relative or absolute SQLite/file storage path. + + :param url: Backend URL containing the storage filename or path. + :param scheme: Selected file or SQLite backend scheme. + :returns: Huey class and constructor options for the file-backed queue. + :raises RuntimeError: If the storage URL is malformed or omits the path. + """ + from huey import FileHuey, SqliteHuey + + if scheme == "sqlite": + backend, option, description, example = ( + SqliteHuey, + "filename", + "SQLite Huey", + "sqlite:///huey.db (relative) or sqlite:////var/huey.db (absolute)", + ) + missing = "SQLite Huey URL must include a database path." + else: + backend, option, description, example = ( + FileHuey, + "path", + "File Huey", + "file:///huey-queue (relative) or file:////var/huey-queue (absolute)", + ) + missing = "File Huey URL must include a directory path." + + prefix = f"{scheme}:///" + if not url.startswith(prefix): + raise RuntimeError(f"{description} URLs must look like {example}") + + # Four slashes retain the initial '/' of an absolute path; three do not. + path = url.removeprefix(prefix) + if not path: + raise RuntimeError(missing) + + return backend, {option: path} diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index fb0c770..6869477 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -20,6 +20,14 @@ from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file +@dataclass(frozen=True) +class TreeScanResult: + """Count and discovered paths from a traversal of the source tree.""" + + scanned_count: int + seen_paths: frozenset[str] + + def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. @@ -118,14 +126,6 @@ def scan_sources() -> None: _scan_and_enqueue() -@dataclass(frozen=True) -class TreeScanResult: - """Count and discovered paths from a traversal of the source tree.""" - - scanned_count: int - seen_paths: frozenset[str] - - def reconcile_source_tree( root: Path, base_url: str | None, diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 3f64bed..6b14ace 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -32,6 +32,34 @@ from .endpoint_discovery import ( ) +@dataclass(frozen=True) +class ClaimedSendAttempt: + """Claimed source revision and retry budget for one send.""" + + webmention_uuid: uuid.UUID + revision: int + attempt_number: int + max_attempts: int + source: str + target: str + + +@dataclass(frozen=True) +class DeliveryResult: + """HTTP outcome, with whether the failure is eligible for another attempt. + + ``retryable`` describes the failure type; it does not account for the + remaining retry budget, which may already have been exhausted. + """ + + status: SentWebmentionStatus + reason: str | None = None + endpoint: str | None = None + response_status: int | None = None + status_url: str | None = None + retryable: bool = False + + def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """Check whether a send attempt still represents the desired revision. @@ -83,34 +111,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return -@dataclass(frozen=True) -class ClaimedSendAttempt: - """Claimed source revision and retry budget for one send.""" - - webmention_uuid: uuid.UUID - revision: int - attempt_number: int - max_attempts: int - source: str - target: str - - -@dataclass(frozen=True) -class DeliveryResult: - """HTTP outcome, with whether the failure is eligible for another attempt. - - ``retryable`` describes the failure type; it does not account for the - remaining retry budget, which may already have been exhausted. - """ - - status: SentWebmentionStatus - reason: str | None = None - endpoint: str | None = None - response_status: int | None = None - status_url: str | None = None - retryable: bool = False - - def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: """Return whether an unprocessed Webmention has reached its retry deadline. diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index a614815..b0f4d0f 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -29,6 +29,9 @@ from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -41,7 +44,14 @@ class RetryableFetchError(Exception): """Fetching the source may succeed when retried later.""" -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +@dataclass(frozen=True) +class VerificationClaim: + """Source and lease version for one atomically claimed verification.""" + + webmention_uuid: uuid.UUID + source: str + target: str + lease_started_at: datetime def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: @@ -125,16 +135,6 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: ) -@dataclass(frozen=True) -class VerificationClaim: - """Source and lease version for one atomically claimed verification.""" - - webmention_uuid: uuid.UUID - source: str - target: str - lease_started_at: datetime - - def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: """Claim an eligible row, returning its source, target, and lease version. |
