diff options
Diffstat (limited to '')
33 files changed, 474 insertions, 139 deletions
diff --git a/tests/tasks/test_extension.py b/tests/tasks/test_extension.py index 312696d..323353e 100644 --- a/tests/tasks/test_extension.py +++ b/tests/tasks/test_extension.py @@ -96,7 +96,10 @@ def test_task_raises_without_app(app: Flask, periodic: bool) -> None: @pytest.mark.parametrize("periodic", [False, True], ids=["task", "periodic-task"]) def test_task_declaration_before_init_and_rebind_after_init(periodic: bool) -> None: - """Imported task handles do not capture the first application backend.""" + """Verify that imported tasks switch to the latest application backend. + + :param periodic: Pytest fixture providing the periodic-task test function. + """ queue = Huey() observed_apps: list[str] = [] diff --git a/tests/tasks/test_notifications.py b/tests/tasks/test_notifications.py index f87fd47..876c2b3 100644 --- a/tests/tasks/test_notifications.py +++ b/tests/tasks/test_notifications.py @@ -135,7 +135,11 @@ def test_digest_skips_when_no_new_mentions( def test_digest_lock_uses_current_backend( notifications: ModuleType, app: Flask ) -> None: - """A digest imported before reinitialization uses the new backend's lock.""" + """Verify that a previously imported digest uses the new backend lock. + + :param notifications: Digest notification module under test. + :param app: Configured Flask application fixture. + """ with app.app_context(): previous_backend = huey.huey huey.init_app(app) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8fa61d5..49494d2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,7 +43,10 @@ def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fl def _script_directory() -> ScriptDirectory: - """Open the migration scripts configured for this application.""" + """Open the migration scripts configured for this application. + + :returns: Configured Alembic script directory for the application. + """ return ScriptDirectory(str(migrate.directory)) @@ -229,7 +232,10 @@ def test_create_db_preserves_logging( def test_upgrade_backfills_portable_digest_for_existing_received_rows( database_app: Flask, ) -> None: - """Upgrade a pre-fingerprint SQLite database without losing historical rows.""" + """Upgrade a pre-fingerprint SQLite database without losing historical rows. + + :param database_app: Flask application connected to the legacy SQLite schema. + """ from webmentions_ssg.models import received_url_digest with database_app.app_context(): @@ -277,7 +283,10 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows( def test_upgrade_converts_received_status_without_losing_rows( database_app: Flask, ) -> None: - """An existing text status survives the non-native Enum migration.""" + """Verify that status rows survive conversion to the non-native Enum. + + :param database_app: Flask application connected to the legacy SQLite schema. + """ identifier = "0" * 31 + "1" with database_app.app_context(): migrate_upgrade(revision="e62b1a7c4d90") @@ -389,7 +398,12 @@ def test_mail_cli_reports_smtp_failure(app: Flask) -> None: def _add_user(app: Flask, username: str, password: str = "original") -> None: - """Persist a user using the same model as the CLI.""" + """Persist a user using the same model as the CLI. + + :param app: Flask application in which to create the user. + :param username: Username to persist. + :param password: Plain-text password for the test user. + """ with app.app_context(): db.session.add(User(username=username, password=password)) db.session.commit() diff --git a/tests/test_init.py b/tests/test_init.py index ccfdfd8..2b5ea00 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -28,7 +28,7 @@ def test_create_app_exposes_version_in_templates(app: Flask) -> None: def test_importing_views_and_tasks_before_creating_app() -> None: - """Module imports alone must never need an initialized Huey backend.""" + """Verify that imports do not need an initialized Huey backend.""" script = """ from webmentions_ssg import huey from webmentions_ssg.tasks.scanner import manual_scan_sources diff --git a/tests/test_models.py b/tests/test_models.py index 85c216e..01e2cec 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -27,7 +27,7 @@ from webmentions_ssg.models import ( def test_uuid_annotations_and_relationships_resolve() -> None: - """Keep UUID columns and forward relationships mapped without future annotations.""" + """Verify that UUID columns and forward relationships map correctly.""" for model in (ReceivedWebmention, Source, SentWebmention): assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID] assert sa.inspect(model).columns["uuid"].primary_key diff --git a/tests/test_net.py b/tests/test_net.py index 4ac138d..b778310 100644 --- a/tests/test_net.py +++ b/tests/test_net.py @@ -42,7 +42,10 @@ def test_backend_rejects_private_ip_without_socket( def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None: - """The address is resolved exactly once and the socket sees only that IP.""" + """Verify that DNS rebinding cannot change the connected socket peer. + + :param monkeypatch: Pytest fixture for replacing the DNS resolver and socket calls. + """ dns = Mock(return_value=("93.184.216.34",)) monkeypatch.setattr("webmentions_ssg.net.public_addresses", dns) socket_instance = Mock() diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 83e3f12..223f35d 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -32,8 +32,8 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: initialized. :param config_class: Configuration class to use, or ``None`` to select the - development or production configuration automatically. - :return: Configured Flask application. + development or production configuration automatically. + :returns: Configured Flask application. """ development = os.environ.get("WEBMENTIONS_SSG_DEV", "").lower() in { "1", @@ -119,7 +119,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: def inject_version(): """Add the application version to the template context. - :return: Template context containing the application version. + :returns: Template context containing the application version. """ return {"VERSION": VERSION} diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index a5f3089..08abaa1 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -28,7 +28,13 @@ MIN_PASSWORD_LENGTH = 8 @cache def dummy_password_hash() -> str: - """Lazily generate a reusable hash for unknown login usernames.""" + """Lazily generate a reusable hash for unknown login usernames. + + The hash is calculated once and reused so missing accounts follow a comparable + password-check path. + + :returns: Reusable password hash for timing-safe unknown-user authentication. + """ return generate_password_hash(secrets.token_urlsafe()) @@ -37,6 +43,10 @@ def validate_password(password: str) -> None: Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases, but reject empty passwords and control characters. + + :param password: Proposed new password to validate. + :raises ValueError: If the password is empty, too short, or contains control + characters. """ if not password: raise ValueError("Password must not be empty.") @@ -53,7 +63,7 @@ def load_user(session_id: object) -> User | None: """Load a user from the database by identifier. :param session_id: Database ID and password-hash fingerprint stored in the session. - :return: Matching user, or ``None`` if no user exists with the identifier. + :returns: Matching user, or ``None`` if no user exists with the identifier. """ if not isinstance(session_id, str): return None @@ -69,7 +79,11 @@ def load_user(session_id: object) -> User | None: def login_username_limit_key() -> str: - """Group usernames across IPs without exposing plaintext to rate-limit storage.""" + """Group usernames across IPs without exposing plaintext to rate-limit storage. + + :returns: Hashed username rate-limit key, or an IP-specific key if no username was + submitted. + """ username = request.form.get("username", "").strip().casefold() if not username: return f"missing-username:{get_remote_address()}" @@ -78,7 +92,11 @@ def login_username_limit_key() -> str: def is_safe_next_url(next_url: str) -> bool: - """Allow redirects only to paths on this site.""" + """Allow redirects only to paths on this site. + + :param next_url: Requested post-login redirect path. + :returns: ``True`` if the redirect is a safe site-local path. + """ if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"): return False try: diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index 7a68651..5c2d534 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -21,7 +21,12 @@ USERS_CLI = AppGroup("users", help="Manage users.") def _require_user(username: str): - """Return a user by username or report a CLI-friendly error.""" + """Return a user by username or report a CLI-friendly error. + + :param username: Username to look up in the database. + :returns: Matching user record. + :raises click.ClickException: If no matching username exists. + """ user = db.session.scalar(sa.select(User).where(User.username == username)) if user is None: raise click.ClickException(f"User {username!r} does not exist.") @@ -59,7 +64,10 @@ def create_user(username: str, password: str) -> None: @USERS_CLI.command("list", help="List registered users.") def list_users() -> None: - """List user IDs and usernames in alphabetical order.""" + """List user IDs and usernames in alphabetical order. + + Write either a sorted ID/username table or a message if no accounts exist. + """ users = db.session.scalars(sa.select(User).order_by(User.username)).all() if not users: click.echo("No users found.") @@ -155,14 +163,22 @@ DB_CLI = AppGroup("db", help="Handle database") def _table_names() -> set[str]: - """Return existing table names without modifying the database schema.""" + """Return existing table names without modifying the database schema. + + :returns: Names of tables in the configured database. + """ with db.engine.connect() as connection: return set(inspect(connection).get_table_names()) @DB_CLI.command("create-db", help="Create a new database using all migrations") def create_db() -> None: - """Initialize an empty database and record its Alembic revision.""" + """Initialize an empty database and record its Alembic revision. + + Refuse to initialize a nonempty schema, then run the complete Alembic upgrade chain. + + :raises click.ClickException: If the database already contains tables. + """ if _table_names(): raise click.ClickException( "The database already contains tables. " @@ -176,7 +192,12 @@ def create_db() -> None: @DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision") def upgrade_db() -> None: - """Apply pending migrations to a previously initialized database.""" + """Apply pending migrations to a previously initialized database. + + Require a previously migrated schema before applying pending Alembic revisions. + + :raises click.ClickException: If the database has no Alembic revision table. + """ if "alembic_version" not in _table_names(): raise click.ClickException( "No Flask-Migrate revision table found. " diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 33647a1..6e6b4bf 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -120,7 +120,13 @@ class TestingConfig(DefaultConfig): def normalize_hostnames(value: object) -> set[str]: - """Validate and normalize a collection of hostnames.""" + """Validate and normalize a collection of hostnames. + + :param value: List, tuple, set, or frozenset of hostnames to normalize. + :returns: Normalized, lowercase hostnames without trailing dots. + :raises TypeError: If the input is not a supported collection type. + :raises ValueError: If a collection item is not a nonempty hostname string. + """ if not isinstance(value, (list, tuple, set, frozenset)): raise TypeError("Expected a list, tuple, or set of hostnames.") @@ -137,7 +143,13 @@ def normalize_hostnames(value: object) -> set[str]: def validate_secret_key(value: object) -> str | bytes: - """Require a non-placeholder, sufficiently long deployment signing key.""" + """Require a non-placeholder, sufficiently long deployment signing key. + + :param value: Configured Flask signing key to validate. + :returns: The original validated signing key. + :raises TypeError: If the configured key is neither text nor bytes. + :raises RuntimeError: If the key is too short, blank, or a known placeholder. + """ if not isinstance(value, (str, bytes)): raise TypeError( "Configure a persistent SECRET_KEY of at least 32 random bytes " @@ -172,6 +184,14 @@ def validate_config( The caller applies the returned values once; validation does not mutate the Flask application or install extensions. + + :param config: Current Flask configuration to check. + :param config_class: Configuration class selected for the application. + :returns: Normalized settings for the caller to apply. + :raises TypeError: If a hostname setting has an invalid collection type. + :raises ValueError: If a hostname or a numeric setting is invalid. + :raises RuntimeError: If production security or shared-storage configuration is + unsafe. """ normalized: dict[str, object] = {} for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"): diff --git a/webmentions_ssg/db_helpers.py b/webmentions_ssg/db_helpers.py index 6e1c2d2..d4cc148 100644 --- a/webmentions_ssg/db_helpers.py +++ b/webmentions_ssg/db_helpers.py @@ -10,5 +10,9 @@ from sqlalchemy.engine import CursorResult, Result def rows_affected(result: Result[Any]) -> int: - """Return the row count from a SQL UPDATE or DELETE result.""" + """Return the row count from a SQL UPDATE or DELETE result. + + :param result: SQLAlchemy result of an UPDATE or DELETE operation. + :returns: Number of rows affected by the statement. + """ return cast(CursorResult[Any], result).rowcount diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py index f5fba51..e3dcdc0 100644 --- a/webmentions_ssg/forms.py +++ b/webmentions_ssg/forms.py @@ -143,7 +143,11 @@ class EndpointForm(FlaskForm): ) def validate_source(self, field: Field) -> None: - """Reject a source URL that equals the target URL.""" + """Reject a source URL that equals the target URL. + + :param field: Source URL field being validated. + :raises ValidationError: If the source URL equals the target URL. + """ if field.data == self.target.data: raise ValidationError(field.gettext("Field must not be equal to target.")) diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py index 6f337be..5547e45 100644 --- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py +++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Track webmention digest notifications +"""Track webmention digest notifications. Revision ID: 408fc2eca9ce Revises: be6e229d6a3d diff --git a/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py b/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py index 0a77094..9522337 100644 --- a/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py +++ b/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Webmention sender +"""Webmention sender. Revision ID: 681a2a9a1bc0 Revises: f63641044cc1 diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py index 66fa289..d5ad4dc 100644 --- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Add Webmention retry backoff +"""Add Webmention retry backoff. Revision ID: 85f2b226ec99 Revises: 408fc2eca9ce diff --git a/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py b/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py index 3ba49dd..058fe18 100644 --- a/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py +++ b/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Do not limit username and password hash length +"""Do not limit username and password hash length. Revision ID: be6e229d6a3d Revises: 681a2a9a1bc0 diff --git a/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py b/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py index b6064fe..c678c39 100644 --- a/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py +++ b/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Initial database schema +"""Initial database schema. Revision ID: f63641044cc1 Revises: diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 622e3b8..33f27d1 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -85,6 +85,9 @@ class User(UserMixin, Base): Only a one-way fingerprint is placed in Flask's signed cookie, never the password hash itself. Rehashing the password invalidates all previously issued session identifiers without a schema change. + + :returns: Session identifier combining the user ID and password-hash + fingerprint. """ fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() return f"{self.id}:{fingerprint}" @@ -93,7 +96,7 @@ class User(UserMixin, Base): """Check a plain-text password against the stored password hash. :param password: Plain-text password to check. - :return: Whether the password matches the stored hash. + :returns: Whether the password matches the stored hash. """ return check_password_hash(self.password_hash, password) @@ -160,7 +163,12 @@ class ReceivedWebmention(Base): def verification_reset_values( *, at: datetime, recoveries: int = 0 ) -> dict[str, object]: - """Return the columns to reset when verification is explicitly reopened.""" + """Return the columns to reset when verification is explicitly reopened. + + :param at: Timestamp to record for the newly opened verification. + :param recoveries: Number of stale-lease recovery attempts to retain. + :returns: Column values for resetting the verification state. + """ return { "status": ReceivedStatus.RECEIVED, "failure_reason": None, @@ -169,7 +177,11 @@ class ReceivedWebmention(Base): } def reset_for_verification(self) -> None: - """Reopen an existing row for manual verification.""" + """Reopen an existing row for manual verification. + + Clear the previous outcome and restore the received state with a fresh + timestamp. + """ for key, value in self.verification_reset_values(at=datetime.now(UTC)).items(): setattr(self, key, value) @@ -304,14 +316,21 @@ class SentWebmention(Base): return f"<SentWebmention {self.target!r}>" def reopen(self, revision: int, active: bool) -> None: - """Reopen delivery for a revision while preserving send history.""" + """Reopen delivery for a revision while preserving send history. + + :param revision: Source revision to make the desired delivery revision. + :param active: Whether the target still appears in the current source. + """ self.desired_revision = revision self.active = active self.attempt_count = 0 self.next_retry_at = None def resend(self) -> None: - """Retry the current revision, clearing its previous delivery result.""" + """Retry the current revision, clearing its previous delivery result. + + Reset attempt and response state without changing the desired source revision. + """ self.reopen(self.desired_revision, self.active) self.processed_revision = None self.status = None @@ -331,7 +350,10 @@ class SentWebmention(Base): @pending.inplace.expression @classmethod def _pending_expression(cls) -> sa.ColumnElement[bool]: - """Express pending-revision status in database queries.""" + """Express pending-revision status in database queries. + + :returns: SQLAlchemy expression selecting unprocessed source revisions. + """ return sa.or_( cls.processed_revision.is_(None), cls.processed_revision < cls.desired_revision, @@ -347,13 +369,18 @@ def uuid7_to_datetime(identifier: UUID) -> datetime: """Convert a UUID version 7 timestamp to a datetime. :param identifier: UUID whose embedded timestamp should be converted. - :return: Timestamp as a timezone-aware UTC datetime. + :returns: Timestamp as a timezone-aware UTC datetime. """ return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) def received_url_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair.""" + """Fingerprint an unambiguous, ordered source/target pair. + + :param source: Webmention source URL. + :param target: Webmention target URL. + :returns: SHA-256 hexadecimal digest of the ordered URL pair. + """ encoded = json.dumps( [source, target], ensure_ascii=False, separators=(",", ":") ).encode("utf-8") @@ -361,5 +388,9 @@ def received_url_digest(source: str, target: str) -> str: def sent_target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID).""" + """Fingerprint an outgoing target (unique within each source ID). + + :param target: Outgoing Webmention target URL. + :returns: SHA-256 hexadecimal digest of the target URL. + """ return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py index c9c2bec..262434a 100644 --- a/webmentions_ssg/net.py +++ b/webmentions_ssg/net.py @@ -179,7 +179,13 @@ class PublicHTTPTransport(httpx.HTTPTransport): def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client: - """Return the hardened HTTP client shared by receiver and sender.""" + """Return the hardened HTTP client shared by receiver and sender. + + :param user_agent: User-Agent header sent with each request. + :param accept: Optional Accept header for the client. + :returns: HTTP client configured with public-only transport, deadlines, and bounded + redirects. + """ headers = {"User-Agent": user_agent} if accept is not None: headers["Accept"] = accept @@ -199,6 +205,9 @@ def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> ``is_global`` alone does not reliably exclude IPv6 transition addresses across Python versions or networks with NAT64/6to4 routing. + + :param address: IPv4 or IPv6 address to classify. + :returns: ``True`` if the address is globally routable and safe to connect to. """ if isinstance(address, ipaddress.IPv6Address): # The well-known NAT64 prefix can translate an otherwise global IPv6 @@ -237,6 +246,12 @@ def public_addresses( entirety. IP literals are handled without DNS, including IPv6 literals. The returned strings must be used for the actual TCP connection; checking them and then connecting to ``hostname`` reintroduces DNS rebinding. + + :param hostname: Hostname or IP literal to validate. + :param lifetime: Optional maximum time allotted for DNS resolution, in seconds. + :returns: Checked numeric IP addresses to use for the TCP connection. + :raises AddressResolutionError: If DNS resolution fails or returns no addresses. + :raises UnsafeAddressError: If any resolved address is not publicly routable. """ try: addresses = (ipaddress.ip_address(hostname),) @@ -253,7 +268,11 @@ def public_addresses( def hostname_of(url: str) -> str | None: - """Return a normalized URL hostname, or None for malformed URLs.""" + """Return a normalized URL hostname, or None for malformed URLs. + + :param url: URL from which to extract a hostname. + :returns: Normalized hostname, or ``None`` when the URL is invalid or has no host. + """ try: hostname = urlsplit(url).hostname except TypeError, ValueError: @@ -262,7 +281,11 @@ def hostname_of(url: str) -> str | None: def content_type_of(headers: Mapping[str, str]) -> str: - """Return a normalized media type without Content-Type parameters.""" + """Return a normalized media type without Content-Type parameters. + + :param headers: HTTP response headers to inspect. + :returns: Lowercase media type without parameters, or an empty string if absent. + """ return headers.get("Content-Type", "").partition(";")[0].strip().lower() @@ -270,7 +293,7 @@ def is_http_url(url: str) -> bool: """Check whether a URL is an absolute HTTP or HTTPS URL. :param url: URL to validate. - :return: Whether the URL uses HTTP or HTTPS and contains a hostname. + :returns: Whether the URL uses HTTP or HTTPS and contains a hostname. """ try: parsed = urlsplit(url) @@ -281,7 +304,15 @@ def is_http_url(url: str) -> bool: def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes: - """Read a streaming response, rejecting oversized declared or actual bodies.""" + """Read a streaming response, rejecting oversized declared or actual bodies. + + :param response: Streaming HTTP response to consume. + :param max_bytes: Maximum permitted response body size in bytes. + :param label: Description of the response for size-limit errors. + :returns: Response body if it fits within the configured byte limit. + :raises ResponseTooLargeError: If declared or streamed content exceeds the byte + limit. + """ if (content_length := response.headers.get("Content-Length")) is not None: try: declared_length = int(content_length) @@ -300,7 +331,11 @@ def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) - def is_temporary_status(status: int) -> bool: - """Return whether an HTTP status is normally retryable.""" + """Return whether an HTTP status is normally retryable. + + :param status: HTTP response status code to classify. + :returns: ``True`` if the HTTP status indicates a retryable failure. + """ return ( status in { diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 362ebe8..13334bc 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -17,7 +17,7 @@ def create_consumer() -> Huey: The application is initialized and the periodic scanner, notification, verification recovery and retention tasks are registered. - :return: Configured Huey instance. + :returns: Configured Huey instance. """ app = create_app() diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py index becac58..6d01e53 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/discovery.py @@ -41,9 +41,8 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str: :param response: Response from which the endpoint was discovered. :param href: Endpoint reference to resolve. - :return: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or - HTTPS URL. + :returns: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL. """ try: endpoint = urljoin(str(response.url), href.strip()) @@ -60,7 +59,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None: """Parse a Link header value into its target and relations. :param value: Link header value to parse. - :return: Link target and relation names, or ``None`` if the value is invalid. + :returns: Link target and relation names, or ``None`` if the value is invalid. """ value = value.strip() @@ -89,8 +88,8 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: """Find a Webmention endpoint in the response Link headers. :param response: HTTP response whose headers should be inspected. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. + :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. + Invalid candidates are ignored. """ for header in response.headers.get_list("Link"): for value in LINK_SPLIT.split(header): @@ -115,8 +114,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: :param response: HTTP response from which the document was retrieved. :param body: HTML document body. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. + :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. + Invalid candidates are ignored. """ document = BeautifulSoup(body, "html5lib") @@ -132,7 +131,12 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: def read_target_body(response: httpx.Response) -> bytes: - """Return a bounded target body, using the shared HTTP size guard.""" + """Return a bounded target body, using the shared HTTP size guard. + + :param response: HTTP response to read with the configured target-size limit. + :returns: Response body, provided it is within the target-size limit. + :raises PermanentSenderError: If the target document is too large. + """ try: return read_limited_body( response, current_app.config["MAX_TARGET_BYTES"], label="Target" @@ -150,7 +154,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non :param client: HTTP client to use for discovery requests. :param target: Target URL whose Webmention endpoint should be discovered. - :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :returns: Discovered Webmention endpoint, or ``None`` if none is advertised. :raises TemporarySenderError: If the target returns a temporary HTTP failure. :raises PermanentSenderError: If GET returns a permanent HTTP failure. """ @@ -191,7 +195,7 @@ def post_webmention( :param endpoint: Webmention endpoint URL. :param source: Source URL of the Webmention. :param target: Target URL of the Webmention. - :return: HTTP response status code and optional status URL. + :returns: HTTP response status code and optional status URL. """ with client.stream( "POST", endpoint, data={"source": source, "target": target} diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index b77a140..58e0036 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -19,7 +19,12 @@ from .json_serializer import JSONSerializer def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]: - """Select a Redis Huey flavour while retaining the underlying URL scheme.""" + """Select a Redis Huey flavour while retaining the underlying URL scheme. + + :param parsed: Parsed Redis backend URL, including any flavour suffix. + :returns: Huey class and constructor options for the requested Redis flavour. + :raises RuntimeError: If the Redis backend flavour is unsupported. + """ from huey import ( PriorityRedisExpireHuey, PriorityRedisHuey, @@ -43,7 +48,13 @@ def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]] def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]: - """Extract a relative or absolute SQLite/file storage path.""" + """Extract a relative or absolute SQLite/file storage path. + + :param url: Backend URL containing the storage filename or path. + :param scheme: Selected file or SQLite backend scheme. + :returns: Huey class and constructor options for the file-backed queue. + :raises RuntimeError: If the storage URL is malformed or omits the path. + """ from huey import FileHuey, SqliteHuey if scheme == "sqlite": @@ -101,7 +112,11 @@ class _LazyTask: update_wrapper(self, func) def register(self) -> None: - """Bind the original function to the latest initialized backend.""" + """Bind the original function to the latest initialized backend. + + Register the task decorator again so its stable handle uses the currently + initialized Huey instance. + """ backend = self._extension.huey decorator = backend.periodic_task if self._periodic else backend.task self._registered = decorator(*self._args, **self._kwargs)( @@ -116,11 +131,20 @@ class _LazyTask: return self._registered def __call__(self, *args: Any, **kwargs: Any) -> Any: - """Enqueue a task using the active backend.""" + """Enqueue a task using the active backend. + + :param args: Positional arguments passed to the queued task. + :param kwargs: Keyword arguments passed to the queued task. + :returns: Result of enqueueing the task on the current backend. + """ return self._current()(*args, **kwargs) def __getattr__(self, name: str) -> Any: - """Delegate Huey task operations (call_local, schedule, s, etc.).""" + """Delegate Huey task operations (call_local, schedule, s, etc.). + + :param name: Huey task attribute to delegate. + :returns: Requested attribute of the currently bound Huey task. + """ return getattr(self._current(), name) @@ -150,8 +174,8 @@ class Huey: resulting extension is registered with the application. :param app: Flask application to initialize. - :raises RuntimeError: If the configured Huey backend URL is invalid or uses - an unsupported scheme. + :raises RuntimeError: If the configured Huey backend URL is invalid or uses an + unsupported scheme. """ config: dict[str, Any] = { "name": app.import_name, @@ -190,6 +214,7 @@ class Huey: def huey(self) -> BaseHuey: """Return the initialized Huey instance. + :returns: Active, initialized Huey backend. :raises RuntimeError: If the extension has not been initialized. """ if self._huey is None: @@ -200,7 +225,12 @@ class Huey: return self._huey def _in_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]: - """Return a task wrapper that executes with the configured Flask app.""" + """Return a task wrapper that executes with the configured Flask app. + + :param func: Task function to wrap in a Flask application context. + :returns: Task wrapper that enters the configured Flask application context. + :raises RuntimeError: If no Flask application has been registered. + """ @wraps(func) def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: @@ -228,7 +258,12 @@ class Huey: def task[**P, R]( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, R]], _LazyTask]: - """Declare a task even before :meth:`init_app` has run.""" + """Declare a task even before :meth:`init_app` has run. + + :param task_args: Positional arguments for the Huey task decorator. + :param task_kwargs: Keyword options for the Huey task decorator. + :returns: Decorator that registers and returns a lazily bound task. + """ def decorator(func: Callable[P, R]) -> _LazyTask: return self._decorate_task(func, task_args, task_kwargs, periodic=False) @@ -238,7 +273,12 @@ class Huey: def periodic_task[**P, R]( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, R]], _LazyTask]: - """Declare a periodic task against the current or next backend.""" + """Declare a periodic task against the current or next backend. + + :param task_args: Positional arguments for the Huey periodic-task decorator. + :param task_kwargs: Keyword options for the Huey periodic-task decorator. + :returns: Decorator that registers and returns a lazily bound periodic task. + """ def decorator(func: Callable[P, R]) -> _LazyTask: return self._decorate_task(func, task_args, task_kwargs, periodic=True) @@ -249,7 +289,7 @@ class Huey: """Forward an unknown attribute to the underlying Huey instance. :param name: Name of the attribute to retrieve. - :return: Attribute from the initialized Huey instance. + :returns: Attribute from the initialized Huey instance. :raises RuntimeError: If the extension has not been initialized. """ return getattr(self.huey, name) @@ -258,12 +298,12 @@ class Huey: def backend_from_url(url: str) -> tuple[type[BaseHuey], dict[str, Any]]: """Determine the Huey backend and storage options from a URL. - :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's - slash convention: three slashes for relative paths and four for - absolute paths (not RFC 8089 file-URI semantics). - :return: Huey backend class and keyword arguments for its storage backend. - :raises RuntimeError: If the URL is malformed for the selected backend or - uses an unsupported scheme. + :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's slash + convention: three slashes for relative paths and four for absolute + paths (not RFC 8089 file-URI semantics). + :returns: Huey backend class and keyword arguments for its storage backend. + :raises RuntimeError: If the URL is malformed for the selected backend or uses + an unsupported scheme. """ parsed = urlsplit(url) scheme = parsed.scheme.lower() diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py index 8f41902..4c3918b 100644 --- a/webmentions_ssg/tasks/mentions.py +++ b/webmentions_ssg/tasks/mentions.py @@ -35,7 +35,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: :param body: HTML document body. :param source_url: URL from which the document was retrieved. :param target_url: Exact URL to look for. - :return: Whether a supported HTML URL attribute references the target URL. + :returns: Whether a supported HTML URL attribute references the target URL. """ document = BeautifulSoup(body, "html.parser") base_url = source_url @@ -63,7 +63,7 @@ def text_mentions_target(body: str, target_url: str) -> bool: :param body: Plain-text document body. :param target_url: Exact IRI to look for. - :return: Whether the target IRI occurs in the document. + :returns: Whether the target IRI occurs in the document. """ for match in IRI_PATTERN.finditer(body): candidate = match.group() diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index fc1b475..b1762d5 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -15,14 +15,22 @@ from ..models import ReceivedStatus, ReceivedWebmention def send_webmention_digest() -> None: - """Send one digest of verified, previously unreported Webmentions.""" + """Send one digest of verified, previously unreported Webmentions. + + Acquire the digest lock before notifying recipients about previously unreported + verified mentions. + """ # Resolve the lock against the current Huey backend, not the import-time one. with huey.lock_task("send-webmention-digest"): _send_webmention_digest() def _send_webmention_digest() -> None: - """Do the work while the current backend's digest lock is held.""" + """Do the work while the current backend's digest lock is held. + + Collect verified mentions, send the configured message, and persist notification + timestamps. + """ recipient = current_app.config.get("NOTIFICATION_EMAIL") if not recipient: return diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 31fb64c..eb228b3 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -51,11 +51,11 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: redirect limit, and maximum response size. :param source_url: URL of the source document. - :return: HTTP response and response body. + :returns: HTTP response and response body. :raises SourceGoneError: If the source returns HTTP 410. :raises TemporaryFetchError: If the source returns a temporary HTTP error. - :raises VerificationError: If the source returns another unsuccessful HTTP - status or exceeds the configured maximum response size. + :raises VerificationError: If the source returns another unsuccessful HTTP status or + exceeds the configured maximum response size. """ try: with ( @@ -97,9 +97,8 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: :param source_url: URL of the source document. :param target_url: Exact target URL to look for. - :return: Whether the source mentions the target URL. - :raises VerificationError: If the source has an unsupported or missing - content type. + :returns: Whether the source mentions the target URL. + :raises VerificationError: If the source has an unsupported or missing content type. :raises SourceGoneError: If the source explicitly reports that it is gone. :raises TemporaryFetchError: If fetching the source fails temporarily. """ @@ -135,7 +134,11 @@ class VerificationClaim: def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: - """Claim an eligible row, returning its source, target, and lease version.""" + """Claim an eligible row, returning its source, target, and lease version. + + :param webmention_uuid: Identifier of the received Webmention to claim. + :returns: Verification claim, or ``None`` if the row cannot be claimed. + """ webmention = db.session.get(ReceivedWebmention, webmention_uuid) if webmention is None: @@ -182,7 +185,14 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]: - """Map verification results and permanent errors to a final status/reason.""" + """Map verification results and permanent errors to a final status/reason. + + :param claim: Claim containing the source and target URLs to verify. + :returns: Final verification status and optional reason. + :raises TemporaryFetchError: If the source fetch fails transiently and should be + retried. + :raises httpx.RequestError: If a transient HTTP request error should be retried. + """ try: mentions_target = source_mentions_target(claim.source, claim.target) except SourceGoneError as exc: @@ -206,7 +216,13 @@ def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]: def release_claim( claim: VerificationClaim, status: ReceivedStatus, reason: str | None ) -> bool: - """Persist an outcome only if this worker still owns the verification lease.""" + """Persist an outcome only if this worker still owns the verification lease. + + :param claim: Verification lease acquired by this worker. + :param status: Final status to persist for the claimed Webmention. + :param reason: Optional explanation for a failed or deleted Webmention. + :returns: ``True`` if the worker still owned the lease and persisted the outcome. + """ changed = db.session.execute( sa.update(ReceivedWebmention) .where( @@ -226,6 +242,9 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: Temporary fetch failures are recorded and re-raised for Huey's retries. + Retry transient fetch errors; persist only results belonging to the currently held + lease. + :param webmention_uuid: Identifier of the received webmention to verify. :raises TemporaryFetchError: If fetching the source fails temporarily. :raises httpx.RequestError: If an HTTP request fails while fetching. @@ -255,7 +274,10 @@ def recover_stale_webmentions() -> int: Persist a recovery count so repeatedly lost or crashing tasks eventually terminate rather than being requeued forever. - :return: Number of webmentions requeued. + Eligible expired verification leases are returned to the queue, subject to recovery + limits. + + :returns: Number of webmentions requeued. """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 86283b7..2c35cc7 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -20,6 +20,11 @@ def purge_old_received_webmentions() -> int: The status/time predicate is checked by the database in one DELETE, so a row recently requeued for verification is not purged based on a stale read. Outgoing sources and Webmentions are intentionally unaffected. + + Use the configured retention period and leave verified or in-progress rows + untouched. + + :returns: Number of expired terminal Webmentions removed. """ cutoff = datetime.now(UTC) - timedelta( days=current_app.config["RECEIVED_RETENTION_DAYS"] diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index ce03185..13d2ceb 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -28,7 +28,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. - :return: Newly created source model. + :returns: Newly created source model. """ source = Source( path=scanned.path, @@ -60,7 +60,7 @@ def update_source( :param source: Existing source model to update. :param scanned: Newly scanned source data. :param scan_time: Time at which the source was scanned. - :return: Updated source model. + :returns: Updated source model. :raises SourceScanError: If the public URL of the source has changed. """ if source.url != scanned.url: @@ -106,8 +106,10 @@ def scan_sources() -> None: successfully processes no sources is treated as inconclusive: it neither marks sources deleted nor queues Webmentions. - :raises RuntimeError: If the configured source directory or base URL is - invalid. + Require a configured source root and public base URL before scanning generated + pages. + + :raises RuntimeError: If the configured source directory or base URL is invalid. """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): @@ -129,7 +131,15 @@ def scan_tree( paths_by_url: dict[str, str], scan_time: datetime, ) -> ScanResult: - """Reconcile found pages and return the paths seen, even on parse failures.""" + """Reconcile found pages and return the paths seen, even on parse failures. + + :param root: Root of the generated static-site output tree. + :param base_url: Optional public base URL used for source URL resolution. + :param sources_by_path: Known source records indexed by relative path. + :param paths_by_url: Known relative paths indexed by public source URL. + :param scan_time: Timestamp associated with the current scan. + :returns: Scan result containing discovered paths and processing counts. + """ seen_paths: set[str] = set() scanned_count = 0 @@ -189,7 +199,13 @@ def scan_tree( def mark_missing( sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime ) -> int: - """Mark absent, previously active sources deleted and return their count.""" + """Mark absent, previously active sources deleted and return their count. + + :param sources_by_path: Known source records indexed by relative path. + :param seen_paths: Paths found during this scan, including failed parses. + :param scan_time: Timestamp to record for sources newly marked missing. + :returns: Number of previously active sources marked as deleted. + """ marked = 0 for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: @@ -210,7 +226,11 @@ def mark_missing( def queue_pending(scan_time: datetime) -> int: - """Enqueue due Webmentions and return the number of tasks submitted.""" + """Enqueue due Webmentions and return the number of tasks submitted. + + :param scan_time: Current scan timestamp used to evaluate retry deadlines. + :returns: Number of due Webmention tasks enqueued. + """ queued = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) @@ -230,7 +250,13 @@ def queue_pending(scan_time: datetime) -> int: def _scan_locked() -> None: - """Scan sources while the caller holds the shared scan lock.""" + """Scan sources while the caller holds the shared scan lock. + + Reconcile the generated source tree, persist changes, and enqueue eligible outgoing + mentions. + + :raises RuntimeError: If the source directory or configured base URL is invalid. + """ directory = current_app.config.get("SOURCE_DIRECTORY") if directory is None: return @@ -279,7 +305,13 @@ def _scan_locked() -> None: @huey.task() def manual_scan_sources() -> None: - """Run a source scan, deferring it while another scan holds the lock.""" + """Run a source scan, deferring it while another scan holds the lock. + + If another scan holds the lock, arrange for the scan to run later instead of + overlapping it. + + :raises RetryTask: If another worker currently holds the scan lock. + """ try: scan_sources() except TaskLockedException as exc: diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 339dfe6..cf8ab9d 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -40,7 +40,7 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: :param webmention: Sent Webmention being processed. :param revision: Source revision represented by the current attempt. - :return: Whether the attempt is still current and requires processing. + :returns: Whether the attempt is still current and requires processing. """ db.session.refresh(webmention) @@ -48,7 +48,11 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: def retry_delay(attempt: int) -> timedelta: - """Calculate the capped exponential delay after a numbered send attempt.""" + """Calculate the capped exponential delay after a numbered send attempt. + + :param attempt: One-based send attempt number. + :returns: Delay until the next attempt, capped by the configured maximum. + """ base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) @@ -63,6 +67,8 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: scanner requeues them when due; Huey does not retry them independently. The retry budget is per desired source revision. + Run the delivery workflow under a lock associated with the outgoing Webmention. + :param webmention_uuid: Identifier of the sent Webmention to process. """ # Huey locks have static names when used as decorators. Acquire one keyed by @@ -107,7 +113,12 @@ class DeliveryResult: def is_due(webmention: SentWebmention, now: datetime) -> bool: - """Return whether an unprocessed Webmention has reached its retry deadline.""" + """Return whether an unprocessed Webmention has reached its retry deadline. + + :param webmention: Outgoing Webmention whose retry eligibility is checked. + :param now: Current timestamp against which the retry deadline is compared. + :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed. + """ if not webmention.pending: return False @@ -124,6 +135,10 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No An attempt's next retry deadline is persisted *before* the HTTP request so a lost worker cannot cause an immediate duplicate send. + + :param webmention: Outgoing Webmention to reserve for sending. + :param now: Timestamp at which the attempt is claimed. + :returns: Reserved attempt details, or ``None`` if no attempt can be started. """ revision = webmention.desired_revision max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) @@ -199,7 +214,13 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No def deliver_webmention( source: str, target: str, *, webmention_uuid: uuid.UUID ) -> DeliveryResult: - """Discover and POST a Webmention, returning an outcome without DB writes.""" + """Discover and POST a Webmention, returning an outcome without DB writes. + + :param source: Public source URL advertising the target link. + :param target: Target URL to notify. + :param webmention_uuid: Identifier used for delivery logging. + :returns: Delivery status, endpoint, response details, and any failure reason. + """ endpoint: str | None = None response_status: int | None = None status_url: str | None = None @@ -275,7 +296,14 @@ def record_outcome( *, terminal: bool, ) -> bool: - """Record an outcome only while the claimed source revision is current.""" + """Record an outcome only while the claimed source revision is current. + + :param webmention: Outgoing Webmention being updated. + :param attempt: Claimed revision and attempt details. + :param result: Result of the delivery attempt. + :param terminal: Whether this outcome ends attempts for the current revision. + :returns: ``True`` if the outcome was recorded for the current revision. + """ # A scanner may have reopened or superseded the revision during I/O. if not attempt_is_current(webmention, attempt.revision): return False @@ -305,7 +333,8 @@ def record_outcome( def _send_locked(webmention_uuid: uuid.UUID) -> bool: """Process a due Webmention while holding its per-UUID task lock. - :return: Whether this worker recorded a delivery outcome. + :param webmention_uuid: Identifier of the outgoing Webmention being processed. + :returns: Whether this worker recorded a delivery outcome. """ webmention = db.session.get(SentWebmention, webmention_uuid) if webmention is None: diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index 294402c..31d0c98 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -76,7 +76,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: :param relative_path: Path of the source document relative to the source root. :param base_url: Base URL under which source documents are published. - :return: Public URL corresponding to the source document. + :returns: Public URL corresponding to the source document. """ directory = relative_path.parent.as_posix() @@ -93,9 +93,9 @@ def canonical_url( :param document: Parsed HTML source document. :param relative_path: Path of the source document relative to the source root. :param base_url: Configured source base URL, if available. - :return: Canonical URL, or ``None`` if none is declared. - :raises SourceScanError: If the canonical URL is empty, cannot be resolved, - or does not resolve to an HTTP or HTTPS URL. + :returns: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, or does + not resolve to an HTTP or HTTPS URL. """ if (link := document.select_one('link[rel~="canonical"][href]')) is None: return None @@ -129,7 +129,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: :param entry: Parsed microformats entry. :param property_name: Name of the property to inspect. - :return: Iterator over string values of the property. + :returns: Iterator over string values of the property. """ properties = entry.get("properties") @@ -149,7 +149,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: :param element: HTML element containing the h-entry. :param base_url: Base URL used when parsing the microformats data. - :return: Parsed h-entry. + :returns: Parsed h-entry. :raises SourceScanError: If parsing does not produce exactly one valid h-entry. """ parsed = mf2py.parse(doc=str(element), url=base_url) @@ -185,9 +185,9 @@ def primary_entry( :param document: Parsed HTML source document. :param source_url: Public URL of the source document. - :return: h-entry element and its parsed microformats representation. - :raises SourceScanError: If the document does not contain exactly one h-entry - or its ``u-url`` does not match the source URL. + :returns: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry or its + ``u-url`` does not match the source URL. """ entries = document.find_all(class_="h-entry") @@ -211,9 +211,9 @@ def content_element(entry: Tag) -> Tag: """Return the e-content element of an h-entry. :param entry: HTML element containing the h-entry. - :return: The h-entry's e-content element. - :raises SourceScanError: If the h-entry does not contain exactly one - e-content element. + :returns: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one e-content + element. """ contents = entry.find_all(class_="e-content") @@ -232,8 +232,8 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non :param value: URL reference to normalize. :param base_url: Base URL against which relative references are resolved. :param source_url: URL of the source document. - :return: Normalized target URL, or ``None`` if the value is not a valid - Webmention target. + :returns: Normalized target URL, or ``None`` if the value is not a valid Webmention + target. """ if not (value := value.strip()): return None @@ -268,7 +268,7 @@ def iter_targets( :param base_url: Base URL against which relative targets are resolved. :param source_url: URL of the source document. :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Iterator over outgoing Webmention target URLs. + :returns: Iterator over outgoing Webmention target URLs. """ hrefs = ( href @@ -289,7 +289,11 @@ def iter_targets( def normalized_content(content: Tag) -> str: - """Extract stable reader-facing content without generated HTML attributes.""" + """Extract stable reader-facing content without generated HTML attributes. + + :param content: Parsed e-content element to normalize. + :returns: Normalized text used to fingerprint reader-facing content. + """ def extract(node: PageElement) -> str: if isinstance(node, NavigableString): @@ -326,9 +330,9 @@ def scan_source_file( :param root: Root directory containing generated source documents. :param base_url: Configured source base URL, if available. :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Scanned source metadata and discovered targets. + :returns: Scanned source metadata and discovered targets. :raises SourceScanError: If the document cannot be safely interpreted as a - Webmention source. + Webmention source. :raises OSError: If the source document cannot be read. """ # Use the path as published for its URL, but read only files whose diff --git a/webmentions_ssg/validation.py b/webmentions_ssg/validation.py index e39d4d5..d1d8129 100644 --- a/webmentions_ssg/validation.py +++ b/webmentions_ssg/validation.py @@ -8,5 +8,9 @@ import unicodedata def is_control_character(character: str) -> bool: - """Identify Unicode control characters, including C0, DEL, and C1.""" + """Identify Unicode control characters, including C0, DEL, and C1. + + :param character: Unicode character to classify. + :returns: ``True`` for C0, DEL, or C1 control characters. + """ return unicodedata.category(character) == "Cc" diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py index f095de8..b9ab53e 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/admin.py @@ -22,7 +22,10 @@ from .blueprint import root_page def validate_admin_action() -> None: - """Reject invalid administrative action forms with HTTP 400.""" + """Reject invalid administrative action forms with HTTP 400. + + Abort the request when the posted form does not pass CSRF or other validation. + """ if not forms.AdminActionForm().validate_on_submit(): abort(400) @@ -32,7 +35,7 @@ def validate_admin_action() -> None: def received() -> ResponseReturnValue: """Display received Webmentions. - :return: Rendered page containing the paginated received Webmentions. + :returns: Rendered page containing the paginated received Webmentions. """ webmentions = db.paginate( sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), @@ -53,7 +56,7 @@ def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: """Delete a received Webmention. :param identifier: Identifier of the Webmention to delete. - :return: Redirect response to the received Webmentions view. + :returns: Redirect response to the received Webmentions view. """ validate_admin_action() @@ -77,7 +80,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: The Webmention status is reset before a new verification task is queued. :param identifier: Identifier of the Webmention to reverify. - :return: Redirect response to the received Webmentions view. + :returns: Redirect response to the received Webmentions view. """ validate_admin_action() @@ -101,7 +104,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: def sent() -> ResponseReturnValue: """Display sources with sent Webmentions. - :return: Rendered page containing the paginated source list. + :returns: Rendered page containing the paginated source list. """ sources = db.paginate( sa.select(Source) @@ -122,7 +125,7 @@ def sent() -> ResponseReturnValue: def rescan_sent_sources() -> ResponseReturnValue: """Queue a manual scan of sent Webmention sources. - :return: Redirect response to the sent Webmentions view. + :returns: Redirect response to the sent Webmentions view. """ validate_admin_action() @@ -138,7 +141,7 @@ def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: """Display the sent Webmentions associated with a source. :param source_identifier: Identifier of the source to display. - :return: Rendered source details page. + :returns: Rendered source details page. """ source = db.get_or_404(Source, source_identifier) @@ -172,7 +175,7 @@ def resend_sent_webmention( :param source_identifier: Identifier of the source owning the Webmention. :param webmention_identifier: Identifier of the sent Webmention to resend. - :return: Redirect response to the source details view. + :returns: Redirect response to the source details view. """ validate_admin_action() diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index a7badfd..85ec398 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -26,7 +26,14 @@ class UrlDigestCollisionError(Exception): def find_webmention_by_digest(source: str, target: str) -> ReceivedWebmention | None: - """Return the matching Webmention, or raise on a SHA-256 collision.""" + """Return the matching Webmention, or raise on a SHA-256 collision. + + :param source: Exact source URL to look up. + :param target: Exact target URL to look up. + :returns: Matching Webmention, or ``None`` if none is stored. + :raises UrlDigestCollisionError: If the digest matches a different source/target + pair. + """ webmention = db.session.execute( sa.select(ReceivedWebmention).where( ReceivedWebmention.url_digest == received_url_digest(source, target) @@ -43,7 +50,14 @@ def find_webmention_by_digest(source: str, target: str) -> ReceivedWebmention | def insert_webmention(source: str, target: str) -> ReceivedWebmention | None: - """Insert and return a Webmention, or None if another request won the race.""" + """Insert and return a Webmention, or None if another request won the race. + + Use an atomic insert and treat uniqueness conflicts as concurrent submissions. + + :param source: Source URL of the new Webmention. + :param target: Target URL of the new Webmention. + :returns: New Webmention, or ``None`` if a concurrent insert won. + """ webmention = ReceivedWebmention(uuid=uuid.uuid7(), source=source, target=target) db.session.add(webmention) @@ -57,7 +71,13 @@ def insert_webmention(source: str, target: str) -> ReceivedWebmention | None: def reopen_if_cooled_down(webmention: ReceivedWebmention) -> bool: - """Atomically reopen an old Webmention and return whether it was reopened.""" + """Atomically reopen an old Webmention and return whether it was reopened. + + Only a terminal Webmention beyond the cooldown interval can be reset. + + :param webmention: Existing Webmention to conditionally reopen. + :returns: ``True`` when the Webmention was reopened for verification. + """ now = datetime.now(UTC) cutoff = now - timedelta( seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] @@ -80,6 +100,13 @@ def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None: """Return (identifier, enqueue flag), or None on a digest conflict. A new submission is committed before it can be queued for verification. + + Commit new submissions before returning whether a verification task should be + queued. + + :param source: Submitted Webmention source URL. + :param target: Submitted Webmention target URL. + :returns: Webmention identifier and enqueue flag, or ``None`` on a digest collision. """ try: if (webmention := find_webmention_by_digest(source, target)) is None: @@ -108,8 +135,8 @@ def endpoint() -> ResponseReturnValue: reverification. Concurrent insertion of the same Webmention is handled by retrieving and updating the row created by the competing request. - :return: HTTP 201 response with the Webmention status URL in the - ``Location`` header, or validation errors with HTTP 400. + :returns: HTTP 201 response with the Webmention status URL in the ``Location`` + header, or validation errors with HTTP 400. """ form = forms.EndpointForm(meta={"csrf": False}) @@ -139,7 +166,7 @@ def status(identifier: uuid.UUID) -> ResponseReturnValue: """Display the verification status of a received Webmention. :param identifier: Identifier of the Webmention to display. - :return: Rendered Webmention status page. + :returns: Rendered Webmention status page. """ webmention = db.get_or_404(ReceivedWebmention, identifier) diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py index 870c967..371997d 100644 --- a/webmentions_ssg/views/session.py +++ b/webmentions_ssg/views/session.py @@ -21,7 +21,7 @@ from .blueprint import root_page def index() -> ResponseReturnValue: """Redirect to the received Webmentions view. - :return: Redirect response to the received Webmentions view. + :returns: Redirect response to the received Webmentions view. """ return redirect(url_for("root.received")) @@ -45,7 +45,7 @@ def login() -> ResponseReturnValue: successful login, the user is redirected to the requested local URL when provided. - :return: Rendered login page or redirect response. + :returns: Rendered login page or redirect response. """ if current_user.is_authenticated: return redirect(url_for("root.index")) @@ -77,7 +77,7 @@ def login() -> ResponseReturnValue: def logout() -> ResponseReturnValue: """End the current user's login session. - :return: Redirect response to the application index. + :returns: Redirect response to the application index. """ logout_user() return redirect(url_for("root.index")) |
