aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--tests/tasks/test_extension.py5
-rw-r--r--tests/tasks/test_notifications.py6
-rw-r--r--tests/test_cli.py22
-rw-r--r--tests/test_init.py2
-rw-r--r--tests/test_models.py2
-rw-r--r--tests/test_net.py5
-rw-r--r--webmentions_ssg/__init__.py6
-rw-r--r--webmentions_ssg/auth.py26
-rw-r--r--webmentions_ssg/cli.py31
-rw-r--r--webmentions_ssg/config.py24
-rw-r--r--webmentions_ssg/db_helpers.py6
-rw-r--r--webmentions_ssg/forms.py6
-rw-r--r--webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py2
-rw-r--r--webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py2
-rw-r--r--webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py2
-rw-r--r--webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py2
-rw-r--r--webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py2
-rw-r--r--webmentions_ssg/models.py49
-rw-r--r--webmentions_ssg/net.py47
-rw-r--r--webmentions_ssg/tasks/consumer.py2
-rw-r--r--webmentions_ssg/tasks/discovery.py26
-rw-r--r--webmentions_ssg/tasks/extension.py74
-rw-r--r--webmentions_ssg/tasks/mentions.py4
-rw-r--r--webmentions_ssg/tasks/notifications.py12
-rw-r--r--webmentions_ssg/tasks/receiver.py42
-rw-r--r--webmentions_ssg/tasks/retention.py5
-rw-r--r--webmentions_ssg/tasks/scanner.py50
-rw-r--r--webmentions_ssg/tasks/sender.py41
-rw-r--r--webmentions_ssg/tasks/source_parser.py40
-rw-r--r--webmentions_ssg/validation.py6
-rw-r--r--webmentions_ssg/views/admin.py19
-rw-r--r--webmentions_ssg/views/endpoint.py39
-rw-r--r--webmentions_ssg/views/session.py6
33 files changed, 474 insertions, 139 deletions
diff --git a/tests/tasks/test_extension.py b/tests/tasks/test_extension.py
index 312696d..323353e 100644
--- a/tests/tasks/test_extension.py
+++ b/tests/tasks/test_extension.py
@@ -96,7 +96,10 @@ def test_task_raises_without_app(app: Flask, periodic: bool) -> None:
@pytest.mark.parametrize("periodic", [False, True], ids=["task", "periodic-task"])
def test_task_declaration_before_init_and_rebind_after_init(periodic: bool) -> None:
- """Imported task handles do not capture the first application backend."""
+ """Verify that imported tasks switch to the latest application backend.
+
+ :param periodic: Pytest fixture providing the periodic-task test function.
+ """
queue = Huey()
observed_apps: list[str] = []
diff --git a/tests/tasks/test_notifications.py b/tests/tasks/test_notifications.py
index f87fd47..876c2b3 100644
--- a/tests/tasks/test_notifications.py
+++ b/tests/tasks/test_notifications.py
@@ -135,7 +135,11 @@ def test_digest_skips_when_no_new_mentions(
def test_digest_lock_uses_current_backend(
notifications: ModuleType, app: Flask
) -> None:
- """A digest imported before reinitialization uses the new backend's lock."""
+ """Verify that a previously imported digest uses the new backend lock.
+
+ :param notifications: Digest notification module under test.
+ :param app: Configured Flask application fixture.
+ """
with app.app_context():
previous_backend = huey.huey
huey.init_app(app)
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 8fa61d5..49494d2 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -43,7 +43,10 @@ def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Fl
def _script_directory() -> ScriptDirectory:
- """Open the migration scripts configured for this application."""
+ """Open the migration scripts configured for this application.
+
+ :returns: Configured Alembic script directory for the application.
+ """
return ScriptDirectory(str(migrate.directory))
@@ -229,7 +232,10 @@ def test_create_db_preserves_logging(
def test_upgrade_backfills_portable_digest_for_existing_received_rows(
database_app: Flask,
) -> None:
- """Upgrade a pre-fingerprint SQLite database without losing historical rows."""
+ """Upgrade a pre-fingerprint SQLite database without losing historical rows.
+
+ :param database_app: Flask application connected to the legacy SQLite schema.
+ """
from webmentions_ssg.models import received_url_digest
with database_app.app_context():
@@ -277,7 +283,10 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows(
def test_upgrade_converts_received_status_without_losing_rows(
database_app: Flask,
) -> None:
- """An existing text status survives the non-native Enum migration."""
+ """Verify that status rows survive conversion to the non-native Enum.
+
+ :param database_app: Flask application connected to the legacy SQLite schema.
+ """
identifier = "0" * 31 + "1"
with database_app.app_context():
migrate_upgrade(revision="e62b1a7c4d90")
@@ -389,7 +398,12 @@ def test_mail_cli_reports_smtp_failure(app: Flask) -> None:
def _add_user(app: Flask, username: str, password: str = "original") -> None:
- """Persist a user using the same model as the CLI."""
+ """Persist a user using the same model as the CLI.
+
+ :param app: Flask application in which to create the user.
+ :param username: Username to persist.
+ :param password: Plain-text password for the test user.
+ """
with app.app_context():
db.session.add(User(username=username, password=password))
db.session.commit()
diff --git a/tests/test_init.py b/tests/test_init.py
index ccfdfd8..2b5ea00 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -28,7 +28,7 @@ def test_create_app_exposes_version_in_templates(app: Flask) -> None:
def test_importing_views_and_tasks_before_creating_app() -> None:
- """Module imports alone must never need an initialized Huey backend."""
+ """Verify that imports do not need an initialized Huey backend."""
script = """
from webmentions_ssg import huey
from webmentions_ssg.tasks.scanner import manual_scan_sources
diff --git a/tests/test_models.py b/tests/test_models.py
index 85c216e..01e2cec 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -27,7 +27,7 @@ from webmentions_ssg.models import (
def test_uuid_annotations_and_relationships_resolve() -> None:
- """Keep UUID columns and forward relationships mapped without future annotations."""
+ """Verify that UUID columns and forward relationships map correctly."""
for model in (ReceivedWebmention, Source, SentWebmention):
assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID]
assert sa.inspect(model).columns["uuid"].primary_key
diff --git a/tests/test_net.py b/tests/test_net.py
index 4ac138d..b778310 100644
--- a/tests/test_net.py
+++ b/tests/test_net.py
@@ -42,7 +42,10 @@ def test_backend_rejects_private_ip_without_socket(
def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None:
- """The address is resolved exactly once and the socket sees only that IP."""
+ """Verify that DNS rebinding cannot change the connected socket peer.
+
+ :param monkeypatch: Pytest fixture for replacing the DNS resolver and socket calls.
+ """
dns = Mock(return_value=("93.184.216.34",))
monkeypatch.setattr("webmentions_ssg.net.public_addresses", dns)
socket_instance = Mock()
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index 83e3f12..223f35d 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -32,8 +32,8 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
initialized.
:param config_class: Configuration class to use, or ``None`` to select the
- development or production configuration automatically.
- :return: Configured Flask application.
+ development or production configuration automatically.
+ :returns: Configured Flask application.
"""
development = os.environ.get("WEBMENTIONS_SSG_DEV", "").lower() in {
"1",
@@ -119,7 +119,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
def inject_version():
"""Add the application version to the template context.
- :return: Template context containing the application version.
+ :returns: Template context containing the application version.
"""
return {"VERSION": VERSION}
diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py
index a5f3089..08abaa1 100644
--- a/webmentions_ssg/auth.py
+++ b/webmentions_ssg/auth.py
@@ -28,7 +28,13 @@ MIN_PASSWORD_LENGTH = 8
@cache
def dummy_password_hash() -> str:
- """Lazily generate a reusable hash for unknown login usernames."""
+ """Lazily generate a reusable hash for unknown login usernames.
+
+ The hash is calculated once and reused so missing accounts follow a comparable
+ password-check path.
+
+ :returns: Reusable password hash for timing-safe unknown-user authentication.
+ """
return generate_password_hash(secrets.token_urlsafe())
@@ -37,6 +43,10 @@ def validate_password(password: str) -> None:
Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases,
but reject empty passwords and control characters.
+
+ :param password: Proposed new password to validate.
+ :raises ValueError: If the password is empty, too short, or contains control
+ characters.
"""
if not password:
raise ValueError("Password must not be empty.")
@@ -53,7 +63,7 @@ def load_user(session_id: object) -> User | None:
"""Load a user from the database by identifier.
:param session_id: Database ID and password-hash fingerprint stored in the session.
- :return: Matching user, or ``None`` if no user exists with the identifier.
+ :returns: Matching user, or ``None`` if no user exists with the identifier.
"""
if not isinstance(session_id, str):
return None
@@ -69,7 +79,11 @@ def load_user(session_id: object) -> User | None:
def login_username_limit_key() -> str:
- """Group usernames across IPs without exposing plaintext to rate-limit storage."""
+ """Group usernames across IPs without exposing plaintext to rate-limit storage.
+
+ :returns: Hashed username rate-limit key, or an IP-specific key if no username was
+ submitted.
+ """
username = request.form.get("username", "").strip().casefold()
if not username:
return f"missing-username:{get_remote_address()}"
@@ -78,7 +92,11 @@ def login_username_limit_key() -> str:
def is_safe_next_url(next_url: str) -> bool:
- """Allow redirects only to paths on this site."""
+ """Allow redirects only to paths on this site.
+
+ :param next_url: Requested post-login redirect path.
+ :returns: ``True`` if the redirect is a safe site-local path.
+ """
if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"):
return False
try:
diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py
index 7a68651..5c2d534 100644
--- a/webmentions_ssg/cli.py
+++ b/webmentions_ssg/cli.py
@@ -21,7 +21,12 @@ USERS_CLI = AppGroup("users", help="Manage users.")
def _require_user(username: str):
- """Return a user by username or report a CLI-friendly error."""
+ """Return a user by username or report a CLI-friendly error.
+
+ :param username: Username to look up in the database.
+ :returns: Matching user record.
+ :raises click.ClickException: If no matching username exists.
+ """
user = db.session.scalar(sa.select(User).where(User.username == username))
if user is None:
raise click.ClickException(f"User {username!r} does not exist.")
@@ -59,7 +64,10 @@ def create_user(username: str, password: str) -> None:
@USERS_CLI.command("list", help="List registered users.")
def list_users() -> None:
- """List user IDs and usernames in alphabetical order."""
+ """List user IDs and usernames in alphabetical order.
+
+ Write either a sorted ID/username table or a message if no accounts exist.
+ """
users = db.session.scalars(sa.select(User).order_by(User.username)).all()
if not users:
click.echo("No users found.")
@@ -155,14 +163,22 @@ DB_CLI = AppGroup("db", help="Handle database")
def _table_names() -> set[str]:
- """Return existing table names without modifying the database schema."""
+ """Return existing table names without modifying the database schema.
+
+ :returns: Names of tables in the configured database.
+ """
with db.engine.connect() as connection:
return set(inspect(connection).get_table_names())
@DB_CLI.command("create-db", help="Create a new database using all migrations")
def create_db() -> None:
- """Initialize an empty database and record its Alembic revision."""
+ """Initialize an empty database and record its Alembic revision.
+
+ Refuse to initialize a nonempty schema, then run the complete Alembic upgrade chain.
+
+ :raises click.ClickException: If the database already contains tables.
+ """
if _table_names():
raise click.ClickException(
"The database already contains tables. "
@@ -176,7 +192,12 @@ def create_db() -> None:
@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision")
def upgrade_db() -> None:
- """Apply pending migrations to a previously initialized database."""
+ """Apply pending migrations to a previously initialized database.
+
+ Require a previously migrated schema before applying pending Alembic revisions.
+
+ :raises click.ClickException: If the database has no Alembic revision table.
+ """
if "alembic_version" not in _table_names():
raise click.ClickException(
"No Flask-Migrate revision table found. "
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 33647a1..6e6b4bf 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -120,7 +120,13 @@ class TestingConfig(DefaultConfig):
def normalize_hostnames(value: object) -> set[str]:
- """Validate and normalize a collection of hostnames."""
+ """Validate and normalize a collection of hostnames.
+
+ :param value: List, tuple, set, or frozenset of hostnames to normalize.
+ :returns: Normalized, lowercase hostnames without trailing dots.
+ :raises TypeError: If the input is not a supported collection type.
+ :raises ValueError: If a collection item is not a nonempty hostname string.
+ """
if not isinstance(value, (list, tuple, set, frozenset)):
raise TypeError("Expected a list, tuple, or set of hostnames.")
@@ -137,7 +143,13 @@ def normalize_hostnames(value: object) -> set[str]:
def validate_secret_key(value: object) -> str | bytes:
- """Require a non-placeholder, sufficiently long deployment signing key."""
+ """Require a non-placeholder, sufficiently long deployment signing key.
+
+ :param value: Configured Flask signing key to validate.
+ :returns: The original validated signing key.
+ :raises TypeError: If the configured key is neither text nor bytes.
+ :raises RuntimeError: If the key is too short, blank, or a known placeholder.
+ """
if not isinstance(value, (str, bytes)):
raise TypeError(
"Configure a persistent SECRET_KEY of at least 32 random bytes "
@@ -172,6 +184,14 @@ def validate_config(
The caller applies the returned values once; validation does not mutate
the Flask application or install extensions.
+
+ :param config: Current Flask configuration to check.
+ :param config_class: Configuration class selected for the application.
+ :returns: Normalized settings for the caller to apply.
+ :raises TypeError: If a hostname setting has an invalid collection type.
+ :raises ValueError: If a hostname or a numeric setting is invalid.
+ :raises RuntimeError: If production security or shared-storage configuration is
+ unsafe.
"""
normalized: dict[str, object] = {}
for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"):
diff --git a/webmentions_ssg/db_helpers.py b/webmentions_ssg/db_helpers.py
index 6e1c2d2..d4cc148 100644
--- a/webmentions_ssg/db_helpers.py
+++ b/webmentions_ssg/db_helpers.py
@@ -10,5 +10,9 @@ from sqlalchemy.engine import CursorResult, Result
def rows_affected(result: Result[Any]) -> int:
- """Return the row count from a SQL UPDATE or DELETE result."""
+ """Return the row count from a SQL UPDATE or DELETE result.
+
+ :param result: SQLAlchemy result of an UPDATE or DELETE operation.
+ :returns: Number of rows affected by the statement.
+ """
return cast(CursorResult[Any], result).rowcount
diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py
index f5fba51..e3dcdc0 100644
--- a/webmentions_ssg/forms.py
+++ b/webmentions_ssg/forms.py
@@ -143,7 +143,11 @@ class EndpointForm(FlaskForm):
)
def validate_source(self, field: Field) -> None:
- """Reject a source URL that equals the target URL."""
+ """Reject a source URL that equals the target URL.
+
+ :param field: Source URL field being validated.
+ :raises ValidationError: If the source URL equals the target URL.
+ """
if field.data == self.target.data:
raise ValidationError(field.gettext("Field must not be equal to target."))
diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
index 6f337be..5547e45 100644
--- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
+++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Track webmention digest notifications
+"""Track webmention digest notifications.
Revision ID: 408fc2eca9ce
Revises: be6e229d6a3d
diff --git a/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py b/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py
index 0a77094..9522337 100644
--- a/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py
+++ b/webmentions_ssg/migrations/versions/681a2a9a1bc0_webmention_sender.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Webmention sender
+"""Webmention sender.
Revision ID: 681a2a9a1bc0
Revises: f63641044cc1
diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
index 66fa289..d5ad4dc 100644
--- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
+++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Add Webmention retry backoff
+"""Add Webmention retry backoff.
Revision ID: 85f2b226ec99
Revises: 408fc2eca9ce
diff --git a/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py b/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py
index 3ba49dd..058fe18 100644
--- a/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py
+++ b/webmentions_ssg/migrations/versions/be6e229d6a3d_do_not_limit_username_and_password_hash_.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Do not limit username and password hash length
+"""Do not limit username and password hash length.
Revision ID: be6e229d6a3d
Revises: 681a2a9a1bc0
diff --git a/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py b/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py
index b6064fe..c678c39 100644
--- a/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py
+++ b/webmentions_ssg/migrations/versions/f63641044cc1_initial_database_schema.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Initial database schema
+"""Initial database schema.
Revision ID: f63641044cc1
Revises:
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 622e3b8..33f27d1 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -85,6 +85,9 @@ class User(UserMixin, Base):
Only a one-way fingerprint is placed in Flask's signed cookie, never
the password hash itself. Rehashing the password invalidates all
previously issued session identifiers without a schema change.
+
+ :returns: Session identifier combining the user ID and password-hash
+ fingerprint.
"""
fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest()
return f"{self.id}:{fingerprint}"
@@ -93,7 +96,7 @@ class User(UserMixin, Base):
"""Check a plain-text password against the stored password hash.
:param password: Plain-text password to check.
- :return: Whether the password matches the stored hash.
+ :returns: Whether the password matches the stored hash.
"""
return check_password_hash(self.password_hash, password)
@@ -160,7 +163,12 @@ class ReceivedWebmention(Base):
def verification_reset_values(
*, at: datetime, recoveries: int = 0
) -> dict[str, object]:
- """Return the columns to reset when verification is explicitly reopened."""
+ """Return the columns to reset when verification is explicitly reopened.
+
+ :param at: Timestamp to record for the newly opened verification.
+ :param recoveries: Number of stale-lease recovery attempts to retain.
+ :returns: Column values for resetting the verification state.
+ """
return {
"status": ReceivedStatus.RECEIVED,
"failure_reason": None,
@@ -169,7 +177,11 @@ class ReceivedWebmention(Base):
}
def reset_for_verification(self) -> None:
- """Reopen an existing row for manual verification."""
+ """Reopen an existing row for manual verification.
+
+ Clear the previous outcome and restore the received state with a fresh
+ timestamp.
+ """
for key, value in self.verification_reset_values(at=datetime.now(UTC)).items():
setattr(self, key, value)
@@ -304,14 +316,21 @@ class SentWebmention(Base):
return f"<SentWebmention {self.target!r}>"
def reopen(self, revision: int, active: bool) -> None:
- """Reopen delivery for a revision while preserving send history."""
+ """Reopen delivery for a revision while preserving send history.
+
+ :param revision: Source revision to make the desired delivery revision.
+ :param active: Whether the target still appears in the current source.
+ """
self.desired_revision = revision
self.active = active
self.attempt_count = 0
self.next_retry_at = None
def resend(self) -> None:
- """Retry the current revision, clearing its previous delivery result."""
+ """Retry the current revision, clearing its previous delivery result.
+
+ Reset attempt and response state without changing the desired source revision.
+ """
self.reopen(self.desired_revision, self.active)
self.processed_revision = None
self.status = None
@@ -331,7 +350,10 @@ class SentWebmention(Base):
@pending.inplace.expression
@classmethod
def _pending_expression(cls) -> sa.ColumnElement[bool]:
- """Express pending-revision status in database queries."""
+ """Express pending-revision status in database queries.
+
+ :returns: SQLAlchemy expression selecting unprocessed source revisions.
+ """
return sa.or_(
cls.processed_revision.is_(None),
cls.processed_revision < cls.desired_revision,
@@ -347,13 +369,18 @@ def uuid7_to_datetime(identifier: UUID) -> datetime:
"""Convert a UUID version 7 timestamp to a datetime.
:param identifier: UUID whose embedded timestamp should be converted.
- :return: Timestamp as a timezone-aware UTC datetime.
+ :returns: Timestamp as a timezone-aware UTC datetime.
"""
return datetime.fromtimestamp(identifier.time / 1000, tz=UTC)
def received_url_digest(source: str, target: str) -> str:
- """Fingerprint an unambiguous, ordered source/target pair."""
+ """Fingerprint an unambiguous, ordered source/target pair.
+
+ :param source: Webmention source URL.
+ :param target: Webmention target URL.
+ :returns: SHA-256 hexadecimal digest of the ordered URL pair.
+ """
encoded = json.dumps(
[source, target], ensure_ascii=False, separators=(",", ":")
).encode("utf-8")
@@ -361,5 +388,9 @@ def received_url_digest(source: str, target: str) -> str:
def sent_target_digest(target: str) -> str:
- """Fingerprint an outgoing target (unique within each source ID)."""
+ """Fingerprint an outgoing target (unique within each source ID).
+
+ :param target: Outgoing Webmention target URL.
+ :returns: SHA-256 hexadecimal digest of the target URL.
+ """
return sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py
index c9c2bec..262434a 100644
--- a/webmentions_ssg/net.py
+++ b/webmentions_ssg/net.py
@@ -179,7 +179,13 @@ class PublicHTTPTransport(httpx.HTTPTransport):
def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
- """Return the hardened HTTP client shared by receiver and sender."""
+ """Return the hardened HTTP client shared by receiver and sender.
+
+ :param user_agent: User-Agent header sent with each request.
+ :param accept: Optional Accept header for the client.
+ :returns: HTTP client configured with public-only transport, deadlines, and bounded
+ redirects.
+ """
headers = {"User-Agent": user_agent}
if accept is not None:
headers["Accept"] = accept
@@ -199,6 +205,9 @@ def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) ->
``is_global`` alone does not reliably exclude IPv6 transition addresses
across Python versions or networks with NAT64/6to4 routing.
+
+ :param address: IPv4 or IPv6 address to classify.
+ :returns: ``True`` if the address is globally routable and safe to connect to.
"""
if isinstance(address, ipaddress.IPv6Address):
# The well-known NAT64 prefix can translate an otherwise global IPv6
@@ -237,6 +246,12 @@ def public_addresses(
entirety. IP literals are handled without DNS, including IPv6 literals.
The returned strings must be used for the actual TCP connection; checking
them and then connecting to ``hostname`` reintroduces DNS rebinding.
+
+ :param hostname: Hostname or IP literal to validate.
+ :param lifetime: Optional maximum time allotted for DNS resolution, in seconds.
+ :returns: Checked numeric IP addresses to use for the TCP connection.
+ :raises AddressResolutionError: If DNS resolution fails or returns no addresses.
+ :raises UnsafeAddressError: If any resolved address is not publicly routable.
"""
try:
addresses = (ipaddress.ip_address(hostname),)
@@ -253,7 +268,11 @@ def public_addresses(
def hostname_of(url: str) -> str | None:
- """Return a normalized URL hostname, or None for malformed URLs."""
+ """Return a normalized URL hostname, or None for malformed URLs.
+
+ :param url: URL from which to extract a hostname.
+ :returns: Normalized hostname, or ``None`` when the URL is invalid or has no host.
+ """
try:
hostname = urlsplit(url).hostname
except TypeError, ValueError:
@@ -262,7 +281,11 @@ def hostname_of(url: str) -> str | None:
def content_type_of(headers: Mapping[str, str]) -> str:
- """Return a normalized media type without Content-Type parameters."""
+ """Return a normalized media type without Content-Type parameters.
+
+ :param headers: HTTP response headers to inspect.
+ :returns: Lowercase media type without parameters, or an empty string if absent.
+ """
return headers.get("Content-Type", "").partition(";")[0].strip().lower()
@@ -270,7 +293,7 @@ def is_http_url(url: str) -> bool:
"""Check whether a URL is an absolute HTTP or HTTPS URL.
:param url: URL to validate.
- :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ :returns: Whether the URL uses HTTP or HTTPS and contains a hostname.
"""
try:
parsed = urlsplit(url)
@@ -281,7 +304,15 @@ def is_http_url(url: str) -> bool:
def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
- """Read a streaming response, rejecting oversized declared or actual bodies."""
+ """Read a streaming response, rejecting oversized declared or actual bodies.
+
+ :param response: Streaming HTTP response to consume.
+ :param max_bytes: Maximum permitted response body size in bytes.
+ :param label: Description of the response for size-limit errors.
+ :returns: Response body if it fits within the configured byte limit.
+ :raises ResponseTooLargeError: If declared or streamed content exceeds the byte
+ limit.
+ """
if (content_length := response.headers.get("Content-Length")) is not None:
try:
declared_length = int(content_length)
@@ -300,7 +331,11 @@ def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -
def is_temporary_status(status: int) -> bool:
- """Return whether an HTTP status is normally retryable."""
+ """Return whether an HTTP status is normally retryable.
+
+ :param status: HTTP response status code to classify.
+ :returns: ``True`` if the HTTP status indicates a retryable failure.
+ """
return (
status
in {
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 362ebe8..13334bc 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -17,7 +17,7 @@ def create_consumer() -> Huey:
The application is initialized and the periodic scanner, notification,
verification recovery and retention tasks are registered.
- :return: Configured Huey instance.
+ :returns: Configured Huey instance.
"""
app = create_app()
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index becac58..6d01e53 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -41,9 +41,8 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str:
:param response: Response from which the endpoint was discovered.
:param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
+ :returns: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL.
"""
try:
endpoint = urljoin(str(response.url), href.strip())
@@ -60,7 +59,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None:
"""Parse a Link header value into its target and relations.
:param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
+ :returns: Link target and relation names, or ``None`` if the value is invalid.
"""
value = value.strip()
@@ -89,8 +88,8 @@ def endpoint_from_headers(response: httpx.Response) -> str | None:
"""Find a Webmention endpoint in the response Link headers.
:param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
+ :returns: First valid advertised Webmention endpoint, or ``None`` if none is found.
+ Invalid candidates are ignored.
"""
for header in response.headers.get_list("Link"):
for value in LINK_SPLIT.split(header):
@@ -115,8 +114,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
:param response: HTTP response from which the document was retrieved.
:param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
+ :returns: First valid advertised Webmention endpoint, or ``None`` if none is found.
+ Invalid candidates are ignored.
"""
document = BeautifulSoup(body, "html5lib")
@@ -132,7 +131,12 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
def read_target_body(response: httpx.Response) -> bytes:
- """Return a bounded target body, using the shared HTTP size guard."""
+ """Return a bounded target body, using the shared HTTP size guard.
+
+ :param response: HTTP response to read with the configured target-size limit.
+ :returns: Response body, provided it is within the target-size limit.
+ :raises PermanentSenderError: If the target document is too large.
+ """
try:
return read_limited_body(
response, current_app.config["MAX_TARGET_BYTES"], label="Target"
@@ -150,7 +154,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
:param client: HTTP client to use for discovery requests.
:param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :returns: Discovered Webmention endpoint, or ``None`` if none is advertised.
:raises TemporarySenderError: If the target returns a temporary HTTP failure.
:raises PermanentSenderError: If GET returns a permanent HTTP failure.
"""
@@ -191,7 +195,7 @@ def post_webmention(
:param endpoint: Webmention endpoint URL.
:param source: Source URL of the Webmention.
:param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
+ :returns: HTTP response status code and optional status URL.
"""
with client.stream(
"POST", endpoint, data={"source": source, "target": target}
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index b77a140..58e0036 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -19,7 +19,12 @@ from .json_serializer import JSONSerializer
def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Select a Redis Huey flavour while retaining the underlying URL scheme."""
+ """Select a Redis Huey flavour while retaining the underlying URL scheme.
+
+ :param parsed: Parsed Redis backend URL, including any flavour suffix.
+ :returns: Huey class and constructor options for the requested Redis flavour.
+ :raises RuntimeError: If the Redis backend flavour is unsupported.
+ """
from huey import (
PriorityRedisExpireHuey,
PriorityRedisHuey,
@@ -43,7 +48,13 @@ def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]
def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]:
- """Extract a relative or absolute SQLite/file storage path."""
+ """Extract a relative or absolute SQLite/file storage path.
+
+ :param url: Backend URL containing the storage filename or path.
+ :param scheme: Selected file or SQLite backend scheme.
+ :returns: Huey class and constructor options for the file-backed queue.
+ :raises RuntimeError: If the storage URL is malformed or omits the path.
+ """
from huey import FileHuey, SqliteHuey
if scheme == "sqlite":
@@ -101,7 +112,11 @@ class _LazyTask:
update_wrapper(self, func)
def register(self) -> None:
- """Bind the original function to the latest initialized backend."""
+ """Bind the original function to the latest initialized backend.
+
+ Register the task decorator again so its stable handle uses the currently
+ initialized Huey instance.
+ """
backend = self._extension.huey
decorator = backend.periodic_task if self._periodic else backend.task
self._registered = decorator(*self._args, **self._kwargs)(
@@ -116,11 +131,20 @@ class _LazyTask:
return self._registered
def __call__(self, *args: Any, **kwargs: Any) -> Any:
- """Enqueue a task using the active backend."""
+ """Enqueue a task using the active backend.
+
+ :param args: Positional arguments passed to the queued task.
+ :param kwargs: Keyword arguments passed to the queued task.
+ :returns: Result of enqueueing the task on the current backend.
+ """
return self._current()(*args, **kwargs)
def __getattr__(self, name: str) -> Any:
- """Delegate Huey task operations (call_local, schedule, s, etc.)."""
+ """Delegate Huey task operations (call_local, schedule, s, etc.).
+
+ :param name: Huey task attribute to delegate.
+ :returns: Requested attribute of the currently bound Huey task.
+ """
return getattr(self._current(), name)
@@ -150,8 +174,8 @@ class Huey:
resulting extension is registered with the application.
:param app: Flask application to initialize.
- :raises RuntimeError: If the configured Huey backend URL is invalid or uses
- an unsupported scheme.
+ :raises RuntimeError: If the configured Huey backend URL is invalid or uses an
+ unsupported scheme.
"""
config: dict[str, Any] = {
"name": app.import_name,
@@ -190,6 +214,7 @@ class Huey:
def huey(self) -> BaseHuey:
"""Return the initialized Huey instance.
+ :returns: Active, initialized Huey backend.
:raises RuntimeError: If the extension has not been initialized.
"""
if self._huey is None:
@@ -200,7 +225,12 @@ class Huey:
return self._huey
def _in_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]:
- """Return a task wrapper that executes with the configured Flask app."""
+ """Return a task wrapper that executes with the configured Flask app.
+
+ :param func: Task function to wrap in a Flask application context.
+ :returns: Task wrapper that enters the configured Flask application context.
+ :raises RuntimeError: If no Flask application has been registered.
+ """
@wraps(func)
def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
@@ -228,7 +258,12 @@ class Huey:
def task[**P, R](
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, R]], _LazyTask]:
- """Declare a task even before :meth:`init_app` has run."""
+ """Declare a task even before :meth:`init_app` has run.
+
+ :param task_args: Positional arguments for the Huey task decorator.
+ :param task_kwargs: Keyword options for the Huey task decorator.
+ :returns: Decorator that registers and returns a lazily bound task.
+ """
def decorator(func: Callable[P, R]) -> _LazyTask:
return self._decorate_task(func, task_args, task_kwargs, periodic=False)
@@ -238,7 +273,12 @@ class Huey:
def periodic_task[**P, R](
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, R]], _LazyTask]:
- """Declare a periodic task against the current or next backend."""
+ """Declare a periodic task against the current or next backend.
+
+ :param task_args: Positional arguments for the Huey periodic-task decorator.
+ :param task_kwargs: Keyword options for the Huey periodic-task decorator.
+ :returns: Decorator that registers and returns a lazily bound periodic task.
+ """
def decorator(func: Callable[P, R]) -> _LazyTask:
return self._decorate_task(func, task_args, task_kwargs, periodic=True)
@@ -249,7 +289,7 @@ class Huey:
"""Forward an unknown attribute to the underlying Huey instance.
:param name: Name of the attribute to retrieve.
- :return: Attribute from the initialized Huey instance.
+ :returns: Attribute from the initialized Huey instance.
:raises RuntimeError: If the extension has not been initialized.
"""
return getattr(self.huey, name)
@@ -258,12 +298,12 @@ class Huey:
def backend_from_url(url: str) -> tuple[type[BaseHuey], dict[str, Any]]:
"""Determine the Huey backend and storage options from a URL.
- :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's
- slash convention: three slashes for relative paths and four for
- absolute paths (not RFC 8089 file-URI semantics).
- :return: Huey backend class and keyword arguments for its storage backend.
- :raises RuntimeError: If the URL is malformed for the selected backend or
- uses an unsupported scheme.
+ :param url: Huey backend URL. SQLite and file paths use SQLAlchemy's slash
+ convention: three slashes for relative paths and four for absolute
+ paths (not RFC 8089 file-URI semantics).
+ :returns: Huey backend class and keyword arguments for its storage backend.
+ :raises RuntimeError: If the URL is malformed for the selected backend or uses
+ an unsupported scheme.
"""
parsed = urlsplit(url)
scheme = parsed.scheme.lower()
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py
index 8f41902..4c3918b 100644
--- a/webmentions_ssg/tasks/mentions.py
+++ b/webmentions_ssg/tasks/mentions.py
@@ -35,7 +35,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
:param body: HTML document body.
:param source_url: URL from which the document was retrieved.
:param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
+ :returns: Whether a supported HTML URL attribute references the target URL.
"""
document = BeautifulSoup(body, "html.parser")
base_url = source_url
@@ -63,7 +63,7 @@ def text_mentions_target(body: str, target_url: str) -> bool:
:param body: Plain-text document body.
:param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
+ :returns: Whether the target IRI occurs in the document.
"""
for match in IRI_PATTERN.finditer(body):
candidate = match.group()
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index fc1b475..b1762d5 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -15,14 +15,22 @@ from ..models import ReceivedStatus, ReceivedWebmention
def send_webmention_digest() -> None:
- """Send one digest of verified, previously unreported Webmentions."""
+ """Send one digest of verified, previously unreported Webmentions.
+
+ Acquire the digest lock before notifying recipients about previously unreported
+ verified mentions.
+ """
# Resolve the lock against the current Huey backend, not the import-time one.
with huey.lock_task("send-webmention-digest"):
_send_webmention_digest()
def _send_webmention_digest() -> None:
- """Do the work while the current backend's digest lock is held."""
+ """Do the work while the current backend's digest lock is held.
+
+ Collect verified mentions, send the configured message, and persist notification
+ timestamps.
+ """
recipient = current_app.config.get("NOTIFICATION_EMAIL")
if not recipient:
return
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 31fb64c..eb228b3 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -51,11 +51,11 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
redirect limit, and maximum response size.
:param source_url: URL of the source document.
- :return: HTTP response and response body.
+ :returns: HTTP response and response body.
:raises SourceGoneError: If the source returns HTTP 410.
:raises TemporaryFetchError: If the source returns a temporary HTTP error.
- :raises VerificationError: If the source returns another unsuccessful HTTP
- status or exceeds the configured maximum response size.
+ :raises VerificationError: If the source returns another unsuccessful HTTP status or
+ exceeds the configured maximum response size.
"""
try:
with (
@@ -97,9 +97,8 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
:param source_url: URL of the source document.
:param target_url: Exact target URL to look for.
- :return: Whether the source mentions the target URL.
- :raises VerificationError: If the source has an unsupported or missing
- content type.
+ :returns: Whether the source mentions the target URL.
+ :raises VerificationError: If the source has an unsupported or missing content type.
:raises SourceGoneError: If the source explicitly reports that it is gone.
:raises TemporaryFetchError: If fetching the source fails temporarily.
"""
@@ -135,7 +134,11 @@ class VerificationClaim:
def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
- """Claim an eligible row, returning its source, target, and lease version."""
+ """Claim an eligible row, returning its source, target, and lease version.
+
+ :param webmention_uuid: Identifier of the received Webmention to claim.
+ :returns: Verification claim, or ``None`` if the row cannot be claimed.
+ """
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
if webmention is None:
@@ -182,7 +185,14 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None:
def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
- """Map verification results and permanent errors to a final status/reason."""
+ """Map verification results and permanent errors to a final status/reason.
+
+ :param claim: Claim containing the source and target URLs to verify.
+ :returns: Final verification status and optional reason.
+ :raises TemporaryFetchError: If the source fetch fails transiently and should be
+ retried.
+ :raises httpx.RequestError: If a transient HTTP request error should be retried.
+ """
try:
mentions_target = source_mentions_target(claim.source, claim.target)
except SourceGoneError as exc:
@@ -206,7 +216,13 @@ def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]:
def release_claim(
claim: VerificationClaim, status: ReceivedStatus, reason: str | None
) -> bool:
- """Persist an outcome only if this worker still owns the verification lease."""
+ """Persist an outcome only if this worker still owns the verification lease.
+
+ :param claim: Verification lease acquired by this worker.
+ :param status: Final status to persist for the claimed Webmention.
+ :param reason: Optional explanation for a failed or deleted Webmention.
+ :returns: ``True`` if the worker still owned the lease and persisted the outcome.
+ """
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
@@ -226,6 +242,9 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
Temporary fetch failures are recorded and re-raised for Huey's retries.
+ Retry transient fetch errors; persist only results belonging to the currently held
+ lease.
+
:param webmention_uuid: Identifier of the received webmention to verify.
:raises TemporaryFetchError: If fetching the source fails temporarily.
:raises httpx.RequestError: If an HTTP request fails while fetching.
@@ -255,7 +274,10 @@ def recover_stale_webmentions() -> int:
Persist a recovery count so repeatedly lost or crashing tasks eventually
terminate rather than being requeued forever.
- :return: Number of webmentions requeued.
+ Eligible expired verification leases are returned to the queue, subject to recovery
+ limits.
+
+ :returns: Number of webmentions requeued.
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 86283b7..2c35cc7 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -20,6 +20,11 @@ def purge_old_received_webmentions() -> int:
The status/time predicate is checked by the database in one DELETE, so a
row recently requeued for verification is not purged based on a stale read.
Outgoing sources and Webmentions are intentionally unaffected.
+
+ Use the configured retention period and leave verified or in-progress rows
+ untouched.
+
+ :returns: Number of expired terminal Webmentions removed.
"""
cutoff = datetime.now(UTC) - timedelta(
days=current_app.config["RECEIVED_RETENTION_DAYS"]
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index ce03185..13d2ceb 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -28,7 +28,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
:param scanned: Scanned source data.
:param scan_time: Time at which the source was discovered.
- :return: Newly created source model.
+ :returns: Newly created source model.
"""
source = Source(
path=scanned.path,
@@ -60,7 +60,7 @@ def update_source(
:param source: Existing source model to update.
:param scanned: Newly scanned source data.
:param scan_time: Time at which the source was scanned.
- :return: Updated source model.
+ :returns: Updated source model.
:raises SourceScanError: If the public URL of the source has changed.
"""
if source.url != scanned.url:
@@ -106,8 +106,10 @@ def scan_sources() -> None:
successfully processes no sources is treated as inconclusive: it neither
marks sources deleted nor queues Webmentions.
- :raises RuntimeError: If the configured source directory or base URL is
- invalid.
+ Require a configured source root and public base URL before scanning generated
+ pages.
+
+ :raises RuntimeError: If the configured source directory or base URL is invalid.
"""
# Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
@@ -129,7 +131,15 @@ def scan_tree(
paths_by_url: dict[str, str],
scan_time: datetime,
) -> ScanResult:
- """Reconcile found pages and return the paths seen, even on parse failures."""
+ """Reconcile found pages and return the paths seen, even on parse failures.
+
+ :param root: Root of the generated static-site output tree.
+ :param base_url: Optional public base URL used for source URL resolution.
+ :param sources_by_path: Known source records indexed by relative path.
+ :param paths_by_url: Known relative paths indexed by public source URL.
+ :param scan_time: Timestamp associated with the current scan.
+ :returns: Scan result containing discovered paths and processing counts.
+ """
seen_paths: set[str] = set()
scanned_count = 0
@@ -189,7 +199,13 @@ def scan_tree(
def mark_missing(
sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
) -> int:
- """Mark absent, previously active sources deleted and return their count."""
+ """Mark absent, previously active sources deleted and return their count.
+
+ :param sources_by_path: Known source records indexed by relative path.
+ :param seen_paths: Paths found during this scan, including failed parses.
+ :param scan_time: Timestamp to record for sources newly marked missing.
+ :returns: Number of previously active sources marked as deleted.
+ """
marked = 0
for path, source in sources_by_path.items():
if source.deleted_at is not None or path in seen_paths:
@@ -210,7 +226,11 @@ def mark_missing(
def queue_pending(scan_time: datetime) -> int:
- """Enqueue due Webmentions and return the number of tasks submitted."""
+ """Enqueue due Webmentions and return the number of tasks submitted.
+
+ :param scan_time: Current scan timestamp used to evaluate retry deadlines.
+ :returns: Number of due Webmention tasks enqueued.
+ """
queued = 0
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
@@ -230,7 +250,13 @@ def queue_pending(scan_time: datetime) -> int:
def _scan_locked() -> None:
- """Scan sources while the caller holds the shared scan lock."""
+ """Scan sources while the caller holds the shared scan lock.
+
+ Reconcile the generated source tree, persist changes, and enqueue eligible outgoing
+ mentions.
+
+ :raises RuntimeError: If the source directory or configured base URL is invalid.
+ """
directory = current_app.config.get("SOURCE_DIRECTORY")
if directory is None:
return
@@ -279,7 +305,13 @@ def _scan_locked() -> None:
@huey.task()
def manual_scan_sources() -> None:
- """Run a source scan, deferring it while another scan holds the lock."""
+ """Run a source scan, deferring it while another scan holds the lock.
+
+ If another scan holds the lock, arrange for the scan to run later instead of
+ overlapping it.
+
+ :raises RetryTask: If another worker currently holds the scan lock.
+ """
try:
scan_sources()
except TaskLockedException as exc:
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 339dfe6..cf8ab9d 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -40,7 +40,7 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
:param webmention: Sent Webmention being processed.
:param revision: Source revision represented by the current attempt.
- :return: Whether the attempt is still current and requires processing.
+ :returns: Whether the attempt is still current and requires processing.
"""
db.session.refresh(webmention)
@@ -48,7 +48,11 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
def retry_delay(attempt: int) -> timedelta:
- """Calculate the capped exponential delay after a numbered send attempt."""
+ """Calculate the capped exponential delay after a numbered send attempt.
+
+ :param attempt: One-based send attempt number.
+ :returns: Delay until the next attempt, capped by the configured maximum.
+ """
base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"]))
maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"]))
return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum))
@@ -63,6 +67,8 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
scanner requeues them when due; Huey does not retry them independently.
The retry budget is per desired source revision.
+ Run the delivery workflow under a lock associated with the outgoing Webmention.
+
:param webmention_uuid: Identifier of the sent Webmention to process.
"""
# Huey locks have static names when used as decorators. Acquire one keyed by
@@ -107,7 +113,12 @@ class DeliveryResult:
def is_due(webmention: SentWebmention, now: datetime) -> bool:
- """Return whether an unprocessed Webmention has reached its retry deadline."""
+ """Return whether an unprocessed Webmention has reached its retry deadline.
+
+ :param webmention: Outgoing Webmention whose retry eligibility is checked.
+ :param now: Current timestamp against which the retry deadline is compared.
+ :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed.
+ """
if not webmention.pending:
return False
@@ -124,6 +135,10 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
An attempt's next retry deadline is persisted *before* the HTTP request so
a lost worker cannot cause an immediate duplicate send.
+
+ :param webmention: Outgoing Webmention to reserve for sending.
+ :param now: Timestamp at which the attempt is claimed.
+ :returns: Reserved attempt details, or ``None`` if no attempt can be started.
"""
revision = webmention.desired_revision
max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
@@ -199,7 +214,13 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No
def deliver_webmention(
source: str, target: str, *, webmention_uuid: uuid.UUID
) -> DeliveryResult:
- """Discover and POST a Webmention, returning an outcome without DB writes."""
+ """Discover and POST a Webmention, returning an outcome without DB writes.
+
+ :param source: Public source URL advertising the target link.
+ :param target: Target URL to notify.
+ :param webmention_uuid: Identifier used for delivery logging.
+ :returns: Delivery status, endpoint, response details, and any failure reason.
+ """
endpoint: str | None = None
response_status: int | None = None
status_url: str | None = None
@@ -275,7 +296,14 @@ def record_outcome(
*,
terminal: bool,
) -> bool:
- """Record an outcome only while the claimed source revision is current."""
+ """Record an outcome only while the claimed source revision is current.
+
+ :param webmention: Outgoing Webmention being updated.
+ :param attempt: Claimed revision and attempt details.
+ :param result: Result of the delivery attempt.
+ :param terminal: Whether this outcome ends attempts for the current revision.
+ :returns: ``True`` if the outcome was recorded for the current revision.
+ """
# A scanner may have reopened or superseded the revision during I/O.
if not attempt_is_current(webmention, attempt.revision):
return False
@@ -305,7 +333,8 @@ def record_outcome(
def _send_locked(webmention_uuid: uuid.UUID) -> bool:
"""Process a due Webmention while holding its per-UUID task lock.
- :return: Whether this worker recorded a delivery outcome.
+ :param webmention_uuid: Identifier of the outgoing Webmention being processed.
+ :returns: Whether this worker recorded a delivery outcome.
"""
webmention = db.session.get(SentWebmention, webmention_uuid)
if webmention is None:
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 294402c..31d0c98 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -76,7 +76,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
:param relative_path: Path of the source document relative to the source root.
:param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
+ :returns: Public URL corresponding to the source document.
"""
directory = relative_path.parent.as_posix()
@@ -93,9 +93,9 @@ def canonical_url(
:param document: Parsed HTML source document.
:param relative_path: Path of the source document relative to the source root.
:param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
+ :returns: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved, or does
+ not resolve to an HTTP or HTTPS URL.
"""
if (link := document.select_one('link[rel~="canonical"][href]')) is None:
return None
@@ -129,7 +129,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
:param entry: Parsed microformats entry.
:param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
+ :returns: Iterator over string values of the property.
"""
properties = entry.get("properties")
@@ -149,7 +149,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
:param element: HTML element containing the h-entry.
:param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
+ :returns: Parsed h-entry.
:raises SourceScanError: If parsing does not produce exactly one valid h-entry.
"""
parsed = mf2py.parse(doc=str(element), url=base_url)
@@ -185,9 +185,9 @@ def primary_entry(
:param document: Parsed HTML source document.
:param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
+ :returns: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry or its
+ ``u-url`` does not match the source URL.
"""
entries = document.find_all(class_="h-entry")
@@ -211,9 +211,9 @@ def content_element(entry: Tag) -> Tag:
"""Return the e-content element of an h-entry.
:param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
+ :returns: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one e-content
+ element.
"""
contents = entry.find_all(class_="e-content")
@@ -232,8 +232,8 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non
:param value: URL reference to normalize.
:param base_url: Base URL against which relative references are resolved.
:param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
+ :returns: Normalized target URL, or ``None`` if the value is not a valid Webmention
+ target.
"""
if not (value := value.strip()):
return None
@@ -268,7 +268,7 @@ def iter_targets(
:param base_url: Base URL against which relative targets are resolved.
:param source_url: URL of the source document.
:param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
+ :returns: Iterator over outgoing Webmention target URLs.
"""
hrefs = (
href
@@ -289,7 +289,11 @@ def iter_targets(
def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
+ """Extract stable reader-facing content without generated HTML attributes.
+
+ :param content: Parsed e-content element to normalize.
+ :returns: Normalized text used to fingerprint reader-facing content.
+ """
def extract(node: PageElement) -> str:
if isinstance(node, NavigableString):
@@ -326,9 +330,9 @@ def scan_source_file(
:param root: Root directory containing generated source documents.
:param base_url: Configured source base URL, if available.
:param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
+ :returns: Scanned source metadata and discovered targets.
:raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
+ Webmention source.
:raises OSError: If the source document cannot be read.
"""
# Use the path as published for its URL, but read only files whose
diff --git a/webmentions_ssg/validation.py b/webmentions_ssg/validation.py
index e39d4d5..d1d8129 100644
--- a/webmentions_ssg/validation.py
+++ b/webmentions_ssg/validation.py
@@ -8,5 +8,9 @@ import unicodedata
def is_control_character(character: str) -> bool:
- """Identify Unicode control characters, including C0, DEL, and C1."""
+ """Identify Unicode control characters, including C0, DEL, and C1.
+
+ :param character: Unicode character to classify.
+ :returns: ``True`` for C0, DEL, or C1 control characters.
+ """
return unicodedata.category(character) == "Cc"
diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py
index f095de8..b9ab53e 100644
--- a/webmentions_ssg/views/admin.py
+++ b/webmentions_ssg/views/admin.py
@@ -22,7 +22,10 @@ from .blueprint import root_page
def validate_admin_action() -> None:
- """Reject invalid administrative action forms with HTTP 400."""
+ """Reject invalid administrative action forms with HTTP 400.
+
+ Abort the request when the posted form does not pass CSRF or other validation.
+ """
if not forms.AdminActionForm().validate_on_submit():
abort(400)
@@ -32,7 +35,7 @@ def validate_admin_action() -> None:
def received() -> ResponseReturnValue:
"""Display received Webmentions.
- :return: Rendered page containing the paginated received Webmentions.
+ :returns: Rendered page containing the paginated received Webmentions.
"""
webmentions = db.paginate(
sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()),
@@ -53,7 +56,7 @@ def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
"""Delete a received Webmention.
:param identifier: Identifier of the Webmention to delete.
- :return: Redirect response to the received Webmentions view.
+ :returns: Redirect response to the received Webmentions view.
"""
validate_admin_action()
@@ -77,7 +80,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
The Webmention status is reset before a new verification task is queued.
:param identifier: Identifier of the Webmention to reverify.
- :return: Redirect response to the received Webmentions view.
+ :returns: Redirect response to the received Webmentions view.
"""
validate_admin_action()
@@ -101,7 +104,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
def sent() -> ResponseReturnValue:
"""Display sources with sent Webmentions.
- :return: Rendered page containing the paginated source list.
+ :returns: Rendered page containing the paginated source list.
"""
sources = db.paginate(
sa.select(Source)
@@ -122,7 +125,7 @@ def sent() -> ResponseReturnValue:
def rescan_sent_sources() -> ResponseReturnValue:
"""Queue a manual scan of sent Webmention sources.
- :return: Redirect response to the sent Webmentions view.
+ :returns: Redirect response to the sent Webmentions view.
"""
validate_admin_action()
@@ -138,7 +141,7 @@ def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
"""Display the sent Webmentions associated with a source.
:param source_identifier: Identifier of the source to display.
- :return: Rendered source details page.
+ :returns: Rendered source details page.
"""
source = db.get_or_404(Source, source_identifier)
@@ -172,7 +175,7 @@ def resend_sent_webmention(
:param source_identifier: Identifier of the source owning the Webmention.
:param webmention_identifier: Identifier of the sent Webmention to resend.
- :return: Redirect response to the source details view.
+ :returns: Redirect response to the source details view.
"""
validate_admin_action()
diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py
index a7badfd..85ec398 100644
--- a/webmentions_ssg/views/endpoint.py
+++ b/webmentions_ssg/views/endpoint.py
@@ -26,7 +26,14 @@ class UrlDigestCollisionError(Exception):
def find_webmention_by_digest(source: str, target: str) -> ReceivedWebmention | None:
- """Return the matching Webmention, or raise on a SHA-256 collision."""
+ """Return the matching Webmention, or raise on a SHA-256 collision.
+
+ :param source: Exact source URL to look up.
+ :param target: Exact target URL to look up.
+ :returns: Matching Webmention, or ``None`` if none is stored.
+ :raises UrlDigestCollisionError: If the digest matches a different source/target
+ pair.
+ """
webmention = db.session.execute(
sa.select(ReceivedWebmention).where(
ReceivedWebmention.url_digest == received_url_digest(source, target)
@@ -43,7 +50,14 @@ def find_webmention_by_digest(source: str, target: str) -> ReceivedWebmention |
def insert_webmention(source: str, target: str) -> ReceivedWebmention | None:
- """Insert and return a Webmention, or None if another request won the race."""
+ """Insert and return a Webmention, or None if another request won the race.
+
+ Use an atomic insert and treat uniqueness conflicts as concurrent submissions.
+
+ :param source: Source URL of the new Webmention.
+ :param target: Target URL of the new Webmention.
+ :returns: New Webmention, or ``None`` if a concurrent insert won.
+ """
webmention = ReceivedWebmention(uuid=uuid.uuid7(), source=source, target=target)
db.session.add(webmention)
@@ -57,7 +71,13 @@ def insert_webmention(source: str, target: str) -> ReceivedWebmention | None:
def reopen_if_cooled_down(webmention: ReceivedWebmention) -> bool:
- """Atomically reopen an old Webmention and return whether it was reopened."""
+ """Atomically reopen an old Webmention and return whether it was reopened.
+
+ Only a terminal Webmention beyond the cooldown interval can be reset.
+
+ :param webmention: Existing Webmention to conditionally reopen.
+ :returns: ``True`` when the Webmention was reopened for verification.
+ """
now = datetime.now(UTC)
cutoff = now - timedelta(
seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
@@ -80,6 +100,13 @@ def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None:
"""Return (identifier, enqueue flag), or None on a digest conflict.
A new submission is committed before it can be queued for verification.
+
+ Commit new submissions before returning whether a verification task should be
+ queued.
+
+ :param source: Submitted Webmention source URL.
+ :param target: Submitted Webmention target URL.
+ :returns: Webmention identifier and enqueue flag, or ``None`` on a digest collision.
"""
try:
if (webmention := find_webmention_by_digest(source, target)) is None:
@@ -108,8 +135,8 @@ def endpoint() -> ResponseReturnValue:
reverification. Concurrent insertion of the same Webmention is handled by
retrieving and updating the row created by the competing request.
- :return: HTTP 201 response with the Webmention status URL in the
- ``Location`` header, or validation errors with HTTP 400.
+ :returns: HTTP 201 response with the Webmention status URL in the ``Location``
+ header, or validation errors with HTTP 400.
"""
form = forms.EndpointForm(meta={"csrf": False})
@@ -139,7 +166,7 @@ def status(identifier: uuid.UUID) -> ResponseReturnValue:
"""Display the verification status of a received Webmention.
:param identifier: Identifier of the Webmention to display.
- :return: Rendered Webmention status page.
+ :returns: Rendered Webmention status page.
"""
webmention = db.get_or_404(ReceivedWebmention, identifier)
diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py
index 870c967..371997d 100644
--- a/webmentions_ssg/views/session.py
+++ b/webmentions_ssg/views/session.py
@@ -21,7 +21,7 @@ from .blueprint import root_page
def index() -> ResponseReturnValue:
"""Redirect to the received Webmentions view.
- :return: Redirect response to the received Webmentions view.
+ :returns: Redirect response to the received Webmentions view.
"""
return redirect(url_for("root.received"))
@@ -45,7 +45,7 @@ def login() -> ResponseReturnValue:
successful login, the user is redirected to the requested local URL when
provided.
- :return: Rendered login page or redirect response.
+ :returns: Rendered login page or redirect response.
"""
if current_user.is_authenticated:
return redirect(url_for("root.index"))
@@ -77,7 +77,7 @@ def login() -> ResponseReturnValue:
def logout() -> ResponseReturnValue:
"""End the current user's login session.
- :return: Redirect response to the application index.
+ :returns: Redirect response to the application index.
"""
logout_user()
return redirect(url_for("root.index"))