aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
-rw-r--r--tests/cli/test_database.py267
-rw-r--r--tests/cli/test_init.py12
-rw-r--r--tests/cli/test_mail.py77
-rw-r--r--tests/cli/test_users.py320
-rw-r--r--tests/forms/__init__.py3
-rw-r--r--tests/forms/test_validators.py166
-rw-r--r--tests/tasks/scanner_helpers.py72
-rw-r--r--tests/tasks/test_discovery.py951
-rw-r--r--tests/tasks/test_json_serializer.py2
-rw-r--r--tests/tasks/test_mentions.py176
-rw-r--r--tests/tasks/test_receiver.py160
-rw-r--r--tests/tasks/test_scanner.py1035
-rw-r--r--tests/tasks/test_sender.py946
-rw-r--r--tests/tasks/test_source_parser.py1008
-rw-r--r--tests/test_auth.py33
-rw-r--r--tests/test_cli.py650
-rw-r--r--tests/test_config.py12
-rw-r--r--tests/test_forms.py (renamed from tests/forms/test_init.py)115
-rw-r--r--tests/test_init.py4
-rw-r--r--tests/test_models.py147
-rw-r--r--tests/test_net.py (renamed from tests/test_public_transport.py)124
-rw-r--r--tests/test_packaging.py (renamed from tests/test_optional_dependencies.py)1
-rw-r--r--tests/test_url_digest.py100
-rw-r--r--tests/test_url_security.py193
-rw-r--r--tests/test_views.py1347
-rw-r--r--tests/views/__init__.py (renamed from tests/cli/__init__.py)0
-rw-r--r--tests/views/helpers.py210
-rw-r--r--tests/views/test_admin.py362
-rw-r--r--tests/views/test_endpoint.py422
-rw-r--r--tests/views/test_session.py431
-rw-r--r--webmentions_ssg/__init__.py117
-rw-r--r--webmentions_ssg/auth.py53
-rw-r--r--webmentions_ssg/cli.py (renamed from webmentions_ssg/cli/users.py)94
-rw-r--r--webmentions_ssg/cli/__init__.py5
-rw-r--r--webmentions_ssg/cli/database.py49
-rw-r--r--webmentions_ssg/cli/mail.py53
-rw-r--r--webmentions_ssg/config.py139
-rw-r--r--webmentions_ssg/extensions.py29
-rw-r--r--webmentions_ssg/forms.py (renamed from webmentions_ssg/forms/validators.py)107
-rw-r--r--webmentions_ssg/forms/__init__.py78
-rw-r--r--webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py4
-rw-r--r--webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py4
-rw-r--r--webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py4
-rw-r--r--webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py2
-rw-r--r--webmentions_ssg/models.py68
-rw-r--r--webmentions_ssg/net.py (renamed from webmentions_ssg/public_transport.py)135
-rw-r--r--webmentions_ssg/security.py51
-rw-r--r--webmentions_ssg/tasks/consumer.py2
-rw-r--r--webmentions_ssg/tasks/discovery.py231
-rw-r--r--webmentions_ssg/tasks/extension.py53
-rw-r--r--webmentions_ssg/tasks/json_serializer.py133
-rw-r--r--webmentions_ssg/tasks/mentions.py99
-rw-r--r--webmentions_ssg/tasks/notifications.py11
-rw-r--r--webmentions_ssg/tasks/receiver.py281
-rw-r--r--webmentions_ssg/tasks/retention.py11
-rw-r--r--webmentions_ssg/tasks/scanner.py414
-rw-r--r--webmentions_ssg/tasks/sender.py405
-rw-r--r--webmentions_ssg/tasks/source_parser.py390
-rw-r--r--webmentions_ssg/templates/received.html51
-rw-r--r--webmentions_ssg/templates/sent_source.html42
-rw-r--r--webmentions_ssg/url_digest.py21
-rw-r--r--webmentions_ssg/url_security.py133
-rw-r--r--webmentions_ssg/views.py454
-rw-r--r--webmentions_ssg/views/__init__.py16
-rw-r--r--webmentions_ssg/views/admin.py217
-rw-r--r--webmentions_ssg/views/endpoint.py150
-rw-r--r--webmentions_ssg/views/session.py93
67 files changed, 6795 insertions, 6750 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
deleted file mode 100644
index 7d325aa..0000000
--- a/tests/cli/test_database.py
+++ /dev/null
@@ -1,267 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Tests for the simplified database CLI and Flask-Migrate integration."""
-
-import logging
-from collections.abc import Iterator
-from pathlib import Path
-
-import pytest
-import sqlalchemy as sa
-from alembic.script import ScriptDirectory
-from flask import Flask
-from flask_migrate import upgrade as migrate_upgrade
-from sqlalchemy import inspect
-
-from webmentions_ssg import DATABASE, MIGRATE, create_app
-from webmentions_ssg.config import TestingConfig
-
-
-@pytest.fixture
-def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]:
- monkeypatch.chdir(tmp_path)
-
- class DatabaseTestingConfig(TestingConfig):
- LOG_FILE = str(tmp_path / "database-cli.log")
- SQLALCHEMY_DATABASE_URI = f"sqlite:///{tmp_path / 'database.sqlite'}"
-
- app = create_app(DatabaseTestingConfig)
- yield app
-
- with app.app_context():
- DATABASE.session.remove()
- DATABASE.engine.dispose()
-
-
-def _script_directory() -> ScriptDirectory:
- """Open the migration scripts configured for this application."""
- return ScriptDirectory(str(MIGRATE.directory))
-
-
-def _head_revision(app: Flask) -> str:
- with app.app_context():
- revision = _script_directory().get_current_head()
- assert revision is not None, "Expected a migration head"
- return revision
-
-
-def _database_revision(app: Flask) -> str:
- with app.app_context(), DATABASE.engine.connect() as connection:
- return connection.execute(
- sa.text("SELECT version_num FROM alembic_version")
- ).scalar_one()
-
-
-def test_database_commands_are_registered_without_hiding_flask_migrate(
- database_app: Flask,
-) -> None:
- runner = database_app.test_cli_runner()
-
- result = runner.invoke(args=["db", "--help"])
- assert result.exit_code == 0, result.output
- assert "create-db" in result.output
- assert "upgrade" in result.output
-
- result = runner.invoke(args=["migrate", "--help"])
- assert result.exit_code == 0, result.output
- assert "downgrade" in result.output
- assert "revision" in result.output
-
-
-def test_create_db_builds_schema_and_records_latest_migration(
- database_app: Flask,
-) -> None:
- result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
-
- assert result.exit_code == 0, result.output
- assert "Database created" in result.output
- assert _database_revision(database_app) == _head_revision(database_app)
-
- with database_app.app_context():
- inspector = inspect(DATABASE.engine)
- table_names = set(inspector.get_table_names())
- columns = {
- column["name"] for column in inspector.get_columns("sent_webmentions")
- }
- received_columns = {
- column["name"] for column in inspector.get_columns("received_webmentions")
- }
- received_indexes = {
- index["name"] for index in inspector.get_indexes("received_webmentions")
- }
- assert {"alembic_version", "users", "received_webmentions"} <= table_names
- assert {"attempt_count", "next_retry_at"} <= columns
- assert "recovery_attempts" in received_columns
- assert "ix_received_webmentions_status_updated_at" in received_indexes
-
-
-def test_create_db_refuses_existing_database_without_modifying_it(
- database_app: Flask,
-) -> None:
- runner = database_app.test_cli_runner()
- assert runner.invoke(args=["db", "create-db"]).exit_code == 0
- revision_before = _database_revision(database_app)
-
- result = runner.invoke(args=["db", "create-db"])
-
- assert result.exit_code != 0
- assert "already contains tables" in result.output
- assert _database_revision(database_app) == revision_before
-
-
-def test_create_db_refuses_unversioned_existing_tables(database_app: Flask) -> None:
- with database_app.app_context(), DATABASE.engine.begin() as connection:
- connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)"))
-
- result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
-
- assert result.exit_code != 0
- assert "already contains tables" in result.output
- with database_app.app_context():
- assert "legacy_data" in inspect(DATABASE.engine).get_table_names()
- assert "alembic_version" not in inspect(DATABASE.engine).get_table_names()
-
-
-def test_upgrade_requires_an_initialized_database(database_app: Flask) -> None:
- result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
-
- assert result.exit_code != 0
- assert "flask db create-db" in result.output
-
-
-def test_upgrade_refuses_unversioned_existing_database(database_app: Flask) -> None:
- with database_app.app_context(), DATABASE.engine.begin() as connection:
- connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)"))
-
- result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
-
- assert result.exit_code != 0
- assert "No Flask-Migrate revision table" in result.output
- with database_app.app_context():
- assert "legacy_data" in inspect(DATABASE.engine).get_table_names()
-
-
-def test_upgrade_applies_pending_migrations_and_preserves_data(
- database_app: Flask,
-) -> None:
- # Start with the first migration, as if this were an older deployment.
- with database_app.app_context():
- base_revision = _script_directory().get_base()
- assert base_revision is not None
- assert base_revision != _head_revision(database_app)
- migrate_upgrade(revision=base_revision)
-
- with DATABASE.engine.begin() as connection:
- connection.execute(
- sa.text(
- "INSERT INTO users (username, password_hash) "
- "VALUES (:username, :password_hash)"
- ),
- {"username": "existing-user", "password_hash": "existing-hash"},
- )
-
- result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
-
- assert result.exit_code == 0, result.output
- assert "Database upgraded" in result.output
- assert _database_revision(database_app) == _head_revision(database_app)
-
- with database_app.app_context(), DATABASE.engine.connect() as connection:
- assert (
- connection.execute(
- sa.text("SELECT password_hash FROM users WHERE username = :username"),
- {"username": "existing-user"},
- ).scalar_one()
- == "existing-hash"
- )
- assert "sent_webmentions" in inspect(connection).get_table_names()
-
-
-def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None:
- runner = database_app.test_cli_runner()
- assert runner.invoke(args=["db", "create-db"]).exit_code == 0
-
- result = runner.invoke(args=["db", "upgrade"])
-
- assert result.exit_code == 0, result.output
- assert _database_revision(database_app) == _head_revision(database_app)
-
-
-def test_create_db_does_not_report_success_when_migration_fails(
- database_app: Flask, monkeypatch: pytest.MonkeyPatch
-) -> None:
- from webmentions_ssg.cli import database as database_cli
-
- def fail_upgrade() -> None:
- raise RuntimeError("simulated migration failure")
-
- monkeypatch.setattr(database_cli, "migrate_upgrade", fail_upgrade)
-
- result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
-
- assert result.exit_code != 0
- assert isinstance(result.exception, RuntimeError)
- assert "Database created" not in result.output
-
-
-def test_create_db_preserves_logging(
- database_app: Flask, caplog: pytest.LogCaptureFixture
-) -> None:
- with caplog.at_level(logging.WARNING):
- result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
-
- assert result.exit_code == 0, result.output
-
- database_app.logger.warning("Logging still works after migration")
-
- assert "Logging still works after migration" in caplog.text
-
-
-def test_upgrade_backfills_portable_digest_for_existing_received_rows(
- database_app: Flask,
-) -> None:
- """Upgrade a pre-fingerprint SQLite database without losing historical rows."""
- from webmentions_ssg.url_digest import received_url_digest
-
- with database_app.app_context():
- migrate_upgrade(revision="c0272f25ce17")
- with DATABASE.engine.begin() as connection:
- connection.execute(
- sa.text(
- "INSERT INTO received_webmentions "
- "(uuid, source, target, status, updated_at) "
- "VALUES (:uuid, :source, :target, :status, :updated_at)"
- ),
- {
- "uuid": "0" * 31 + "1",
- "source": "https://source.example/old",
- "target": "https://dennisfink.me/target",
- "status": "verified",
- "updated_at": "2026-01-01 00:00:00",
- },
- )
- migrate_upgrade()
- with DATABASE.engine.connect() as connection:
- result = connection.execute(
- sa.text(
- "SELECT source, target, status, url_digest "
- "FROM received_webmentions"
- )
- ).one()
- assert result.status == "verified"
- recovery_attempts = connection.execute(
- sa.text("SELECT recovery_attempts FROM received_webmentions")
- ).scalar_one()
- assert recovery_attempts == 0
- assert result.url_digest == received_url_digest(
- result.source, result.target
- )
- constraints = inspect(connection).get_unique_constraints(
- "received_webmentions"
- )
- assert any(
- constraint["column_names"] == ["url_digest"]
- for constraint in constraints
- )
diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py
deleted file mode 100644
index cbdab8c..0000000
--- a/tests/cli/test_init.py
+++ /dev/null
@@ -1,12 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Test that all application CLI command groups are exposed."""
-
-from flask import Flask
-
-
-def test_cli_groups_are_registered(app: Flask) -> None:
- commands = app.cli.commands
- assert {"users", "mail"}.issubset(commands)
diff --git a/tests/cli/test_mail.py b/tests/cli/test_mail.py
deleted file mode 100644
index 0a2afce..0000000
--- a/tests/cli/test_mail.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-from unittest.mock import patch
-
-from flask import Flask
-
-
-def test_mail_cli_sends_to_configured_notification_email(app: Flask) -> None:
- app.config.update(
- NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
- )
-
- with patch("webmentions_ssg.MAIL.send") as send:
- result = app.test_cli_runner().invoke(args=["mail", "test"])
-
- assert result.exit_code == 0, result.output
- send.assert_called_once()
- message = send.call_args.args[0]
- assert message.sender == "sender@example.com"
- assert message.recipients == ["owner@example.com"]
- assert "test email" in message.subject.lower()
- assert "outgoing mail configuration" in message.body
- assert "Test email sent to owner@example.com." in result.output
-
-
-def test_mail_cli_overrides_configured_recipient(app: Flask) -> None:
- app.config.update(
- NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
- )
-
- with patch("webmentions_ssg.MAIL.send") as send:
- result = app.test_cli_runner().invoke(
- args=["mail", "test", "--to", "other@example.com"]
- )
-
- assert result.exit_code == 0, result.output
- send.assert_called_once()
- assert send.call_args.args[0].recipients == ["other@example.com"]
-
-
-def test_mail_cli_requires_recipient(app: Flask) -> None:
- app.config.update(NOTIFICATION_EMAIL=None, MAIL_DEFAULT_SENDER="sender@example.com")
-
- with patch("webmentions_ssg.MAIL.send") as send:
- result = app.test_cli_runner().invoke(args=["mail", "test"])
-
- assert result.exit_code != 0
- assert "Set NOTIFICATION_EMAIL or pass --to" in result.output
- send.assert_not_called()
-
-
-def test_mail_cli_requires_default_sender(app: Flask) -> None:
- app.config.update(NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER=None)
-
- with patch("webmentions_ssg.MAIL.send") as send:
- result = app.test_cli_runner().invoke(args=["mail", "test"])
-
- assert result.exit_code != 0
- assert "MAIL_DEFAULT_SENDER is not configured" in result.output
- send.assert_not_called()
-
-
-def test_mail_cli_reports_smtp_failure(app: Flask) -> None:
- app.config.update(
- NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
- )
-
- with patch(
- "webmentions_ssg.MAIL.send", side_effect=ConnectionError("SMTP unavailable")
- ) as send:
- result = app.test_cli_runner().invoke(args=["mail", "test"])
-
- assert result.exit_code != 0
- assert "Could not send test email: SMTP unavailable" in result.output
- send.assert_called_once()
diff --git a/tests/cli/test_users.py b/tests/cli/test_users.py
deleted file mode 100644
index 4764c85..0000000
--- a/tests/cli/test_users.py
+++ /dev/null
@@ -1,320 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-from unittest.mock import patch
-
-import pytest
-import sqlalchemy as sa
-from flask import Flask
-from sqlalchemy.exc import IntegrityError
-
-from webmentions_ssg import DATABASE
-from webmentions_ssg.models import User
-
-
-def _add_user(app: Flask, username: str, password: str = "original") -> None:
- """Persist a user using the same model as the CLI."""
- with app.app_context():
- DATABASE.session.add(User(username=username, password=password))
- DATABASE.session.commit()
-
-
-def _get_user(app: Flask, username: str) -> User | None:
- with app.app_context():
- return DATABASE.session.scalar(sa.select(User).where(User.username == username))
-
-
-def test_users_cli_help_lists_management_commands(app: Flask) -> None:
- result = app.test_cli_runner().invoke(args=["users", "--help"])
-
- assert result.exit_code == 0, result.output
- for command in ("create", "list", "set-password", "delete"):
- assert command in result.output
-
-
-def test_users_cli_create_with_prompts_hashes_password(app: Flask) -> None:
- result = app.test_cli_runner().invoke(
- args=["users", "create"],
- input="alice\nmy new secret passphrase\nmy new secret passphrase\n",
- )
-
- assert result.exit_code == 0, result.output
- assert "Created user 'alice'." in result.output
- assert "my new secret passphrase" not in result.output
- user = _get_user(app, "alice")
- assert user is not None
- assert user.check_password("my new secret passphrase")
- assert user.password_hash != "my new secret passphrase"
-
-
-def test_users_cli_create_with_username_option(app: Flask) -> None:
- result = app.test_cli_runner().invoke(
- args=["users", "create", "--username", "alice"],
- input="my new secret passphrase\nmy new secret passphrase\n",
- )
-
- assert result.exit_code == 0, result.output
- assert _get_user(app, "alice") is not None
-
-
-def test_users_cli_create_rejects_duplicate(app: Flask) -> None:
- _add_user(app, "alice")
-
- result = app.test_cli_runner().invoke(
- args=[
- "users",
- "create",
- "--username",
- "alice",
- "--password",
- "a stronger duplicate password",
- ]
- )
-
- assert result.exit_code != 0
- assert "User 'alice' already exists." in result.output
- user = _get_user(app, "alice")
- assert user is not None
- assert user.check_password("original")
-
-
-@pytest.mark.parametrize("username", ["", " "])
-def test_users_cli_create_rejects_invalid_username(app: Flask, username: str) -> None:
- result = app.test_cli_runner().invoke(
- args=[
- "users",
- "create",
- "--username",
- username,
- "--password",
- "a strong password for testing",
- ]
- )
-
- assert result.exit_code != 0
- assert "Username must not" in result.output
-
-
-def test_users_cli_create_rejects_empty_password(app: Flask) -> None:
- result = app.test_cli_runner().invoke(
- args=["users", "create", "--username", "alice", "--password", ""]
- )
-
- assert result.exit_code != 0
- assert "Password must not be empty." in result.output
- assert _get_user(app, "alice") is None
-
-
-def test_users_cli_list_empty(app: Flask) -> None:
- result = app.test_cli_runner().invoke(args=["users", "list"])
-
- assert result.exit_code == 0, result.output
- assert "No users found." in result.output
-
-
-def test_users_cli_list_is_sorted_and_excludes_passwords(app: Flask) -> None:
- _add_user(app, "zoe")
- _add_user(app, "alice")
-
- result = app.test_cli_runner().invoke(args=["users", "list"])
-
- assert result.exit_code == 0, result.output
- lines = result.output.splitlines()
- assert lines[0] == "ID\tUSERNAME"
- assert lines[1].endswith("\talice")
- assert lines[2].endswith("\tzoe")
- assert "original" not in result.output
-
-
-def test_users_cli_set_password_updates_hash(app: Flask) -> None:
- _add_user(app, "alice")
- old_hash = _get_user(app, "alice").password_hash # type: ignore[union-attr]
-
- result = app.test_cli_runner().invoke(
- args=["users", "set-password", "alice"],
- input="a long replacement passphrase\na long replacement passphrase\n",
- )
-
- assert result.exit_code == 0, result.output
- assert "Updated password for user 'alice'." in result.output
- assert "a long replacement passphrase" not in result.output
- user = _get_user(app, "alice")
- assert user is not None
- assert user.password_hash != old_hash
- assert not user.check_password("original")
- assert user.check_password("a long replacement passphrase")
-
-
-def test_users_cli_set_password_missing_user(app: Flask) -> None:
- result = app.test_cli_runner().invoke(
- args=[
- "users",
- "set-password",
- "missing",
- "--password",
- "a long replacement passphrase",
- ]
- )
-
- assert result.exit_code != 0
- assert "User 'missing' does not exist." in result.output
-
-
-def test_users_cli_set_password_rejects_empty_password(app: Flask) -> None:
- _add_user(app, "alice")
-
- result = app.test_cli_runner().invoke(
- args=["users", "set-password", "alice", "--password", ""]
- )
-
- assert result.exit_code != 0
- assert "Password must not be empty." in result.output
- user = _get_user(app, "alice")
- assert user is not None
- assert user.check_password("original")
-
-
-def test_users_cli_delete_with_confirmation(app: Flask) -> None:
- _add_user(app, "alice")
-
- result = app.test_cli_runner().invoke(
- args=["users", "delete", "alice"], input="y\n"
- )
-
- assert result.exit_code == 0, result.output
- assert "Delete user 'alice'?" in result.output
- assert "Deleted user 'alice'." in result.output
- assert _get_user(app, "alice") is None
-
-
-def test_users_cli_delete_confirmation_rejected(app: Flask) -> None:
- _add_user(app, "alice")
-
- result = app.test_cli_runner().invoke(
- args=["users", "delete", "alice"], input="n\n"
- )
-
- assert result.exit_code != 0
- assert "Aborted" in result.output
- assert _get_user(app, "alice") is not None
-
-
-def test_users_cli_delete_yes_only_deletes_target(app: Flask) -> None:
- _add_user(app, "alice")
- _add_user(app, "bob")
-
- result = app.test_cli_runner().invoke(args=["users", "delete", "alice", "--yes"])
-
- assert result.exit_code == 0, result.output
- assert "Deleted user 'alice'." in result.output
- assert _get_user(app, "alice") is None
- assert _get_user(app, "bob") is not None
-
-
-def test_users_cli_delete_missing_user(app: Flask) -> None:
- result = app.test_cli_runner().invoke(args=["users", "delete", "missing", "-y"])
-
- assert result.exit_code != 0
- assert "User 'missing' does not exist." in result.output
-
-
-def test_users_cli_delete_missing_blank_username(app: Flask) -> None:
- result = app.test_cli_runner().invoke(args=["users", "delete", " ", "--yes"])
-
- assert result.exit_code != 0
- assert "User ' ' does not exist." in result.output
-
-
-def test_users_cli_can_manage_legacy_whitespace_username(app: Flask) -> None:
- _add_user(app, " alice ")
-
- reset = app.test_cli_runner().invoke(
- args=[
- "users",
- "set-password",
- " alice ",
- "--password",
- "a long replacement passphrase",
- ]
- )
-
- assert reset.exit_code == 0, reset.output
- user = _get_user(app, " alice ")
- assert user is not None
- assert user.check_password("a long replacement passphrase")
-
- deleted = app.test_cli_runner().invoke(args=["users", "delete", " alice ", "--yes"])
- assert deleted.exit_code == 0, deleted.output
- assert _get_user(app, " alice ") is None
-
-
-def test_users_cli_create_rolls_back_on_duplicate_race(app: Flask) -> None:
- database_error = IntegrityError("INSERT", {}, Exception("duplicate"))
-
- with app.app_context():
- with (
- patch.object(DATABASE.session, "commit", side_effect=database_error),
- patch.object(DATABASE.session, "rollback") as rollback,
- ):
- result = app.test_cli_runner().invoke(
- args=[
- "users",
- "create",
- "--username",
- "alice",
- "--password",
- "a long secret for testing",
- ]
- )
- rollback.assert_called_once()
-
- assert result.exit_code != 0
- assert "User 'alice' already exists." in result.output
-
-
-@pytest.mark.parametrize(
- ("password", "error"),
- [
- ("short", "at least 8 characters"),
- ("good passphrase with\tab", "control characters"),
- ],
-)
-@pytest.mark.parametrize("operation", ["create", "set-password"])
-def test_users_cli_enforces_password_policy(
- app: Flask, password: str, error: str, operation: str
-) -> None:
- _add_user(app, "existing")
- if operation == "create":
- command = ["users", "create", "--username", "new"]
- else:
- command = ["users", "set-password", "existing"]
-
- result = app.test_cli_runner().invoke(args=[*command, "--password", password])
- assert result.exit_code != 0
- assert error in result.output
- assert _get_user(app, "new") is None
- existing = _get_user(app, "existing")
- assert existing is not None and existing.check_password("original")
-
-
-def test_users_cli_accepts_long_unicode_passphrase(app: Flask) -> None:
- passphrase = "🦊 several words of my passphrase 🪁"
- result = app.test_cli_runner().invoke(
- args=["users", "create", "--username", "alice", "--password", passphrase]
- )
- assert result.exit_code == 0, result.output
- user = _get_user(app, "alice")
- assert user is not None and user.check_password(passphrase)
-
-
-@pytest.mark.parametrize("passphrase", ["12345678", " " * 8, "🦊" + "a" * 1025])
-def test_users_cli_accepts_minimum_whitespace_and_unbounded_passwords(
- app: Flask, passphrase: str
-) -> None:
- result = app.test_cli_runner().invoke(
- args=["users", "create", "--username", "alice", "--password", passphrase]
- )
- assert result.exit_code == 0, result.output
- user = _get_user(app, "alice")
- assert user is not None and user.check_password(passphrase)
diff --git a/tests/forms/__init__.py b/tests/forms/__init__.py
deleted file mode 100644
index 25d78d5..0000000
--- a/tests/forms/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py
deleted file mode 100644
index ae4c8f2..0000000
--- a/tests/forms/test_validators.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-from unittest.mock import Mock
-
-import pytest
-from flask import Flask
-from werkzeug.datastructures import MultiDict
-from wtforms import Form, StringField, ValidationError
-
-from webmentions_ssg.forms.validators import (
- AllowedHostname,
- NotEqualTo,
- PublicIPLiteral,
- PublicURL,
- URLMaxBytes,
-)
-from webmentions_ssg.url_security import AddressResolutionError
-
-
-class ComparisonForm(Form):
- source = StringField("Source")
- target = StringField("Target")
-
-
-class URLForm(Form):
- url = StringField("URL")
-
-
-def test_not_equal_to_accepts_different_values() -> None:
- form = ComparisonForm(MultiDict({"source": "source", "target": "target"}))
- NotEqualTo("target")(form, form.source)
-
-
-def test_not_equal_to_rejects_equal_values() -> None:
- form = ComparisonForm(MultiDict({"source": "same", "target": "same"}))
- with pytest.raises(ValidationError, match="Field must not be equal to target"):
- NotEqualTo("target")(form, form.source)
-
-
-def test_not_equal_to_uses_custom_message() -> None:
- form = ComparisonForm(MultiDict({"source": "same", "target": "same"}))
- with pytest.raises(ValidationError, match="Must differ from Target"):
- NotEqualTo("target", "Must differ from %(other_label)s")(form, form.source)
-
-
-def test_not_equal_to_rejects_unknown_field() -> None:
- form = ComparisonForm(MultiDict({"source": "source", "target": "target"}))
- with pytest.raises(ValidationError, match="Invalid field name 'missing'"):
- NotEqualTo("missing")(form, form.source)
-
-
-def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None:
- app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
- form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"}))
- with app.app_context():
- AllowedHostname()(form, form.url)
-
-
-def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None:
- app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
- form = URLForm(MultiDict({"url": "https://example.com/post"}))
- with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
- AllowedHostname()(form, form.url)
-
-
-def test_allowed_hostname_uses_custom_message(app: Flask) -> None:
- app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
- form = URLForm(MultiDict({"url": "https://example.com/post"}))
- with (
- app.app_context(),
- pytest.raises(ValidationError, match="Hostname is not allowed"),
- ):
- AllowedHostname("Hostname is not allowed")(form, form.url)
-
-
-def test_public_url_accepts_public_url(monkeypatch: pytest.MonkeyPatch) -> None:
- is_public_url = Mock(return_value=True)
- monkeypatch.setattr("webmentions_ssg.forms.validators.is_public_url", is_public_url)
- form = URLForm(MultiDict({"url": "https://example.com/post"}))
- PublicURL()(form, form.url)
- is_public_url.assert_called_once_with("https://example.com/post")
-
-
-@pytest.mark.parametrize(
- "outcome",
- [
- pytest.param(False, id="non-public-address"),
- pytest.param(
- AddressResolutionError("Could not resolve hostname"), id="resolution-error"
- ),
- pytest.param(ValueError("No hostname was specified"), id="missing-hostname"),
- ],
-)
-def test_public_url_rejects_invalid_url(
- monkeypatch: pytest.MonkeyPatch, outcome: bool | Exception
-) -> None:
- is_public_url = Mock()
-
- if isinstance(outcome, Exception):
- is_public_url.side_effect = outcome
- else:
- is_public_url.return_value = outcome
-
- monkeypatch.setattr("webmentions_ssg.forms.validators.is_public_url", is_public_url)
- form = URLForm(MultiDict({"url": "https://example.com/post"}))
-
- with pytest.raises(ValidationError, match="URL must resolve to a public address"):
- PublicURL()(form, form.url)
-
-
-def test_public_url_uses_custom_message(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- "webmentions_ssg.forms.validators.is_public_url", Mock(return_value=False)
- )
- form = URLForm(MultiDict({"url": "https://example.com/post"}))
-
- with pytest.raises(ValidationError, match="Public URL required"):
- PublicURL("Public URL required")(form, form.url)
-
-
-@pytest.mark.parametrize(
- "address",
- ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"],
-)
-def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None:
- form = URLForm(MultiDict({"url": f"https://[{address}]/"}))
- with pytest.raises(ValidationError, match="URL must use a public address"):
- PublicIPLiteral()(form, form.url)
-
-
-def test_public_ip_literal_accepts_public_ipv6_address() -> None:
- form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"}))
- PublicIPLiteral()(form, form.url)
-
-
-def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None:
- form = URLForm(MultiDict({"url": "http://[::1"}))
- with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
- AllowedHostname()(form, form.url)
-
-
-def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None:
- validator = URLMaxBytes()
- form = URLForm(MultiDict({"url": "é" * 3}))
- with app.app_context():
- app.config["MAX_WEBMENTION_URL_BYTES"] = 6
- validator(form, form.url)
- app.config["MAX_WEBMENTION_URL_BYTES"] = 5
- with pytest.raises(ValidationError, match="5 bytes"):
- validator(form, form.url)
-
-
-def test_url_max_bytes_custom_message(app: Flask) -> None:
- form = URLForm(MultiDict({"url": "too-long"}))
- with app.app_context():
- app.config["MAX_WEBMENTION_URL_BYTES"] = 2
- with pytest.raises(ValidationError, match="URL too long"):
- URLMaxBytes(message="URL too long")(form, form.url)
-
-
-def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None:
- form = URLForm(MultiDict({"url": "\ud800"}))
- with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"):
- URLMaxBytes()(form, form.url)
diff --git a/tests/tasks/scanner_helpers.py b/tests/tasks/scanner_helpers.py
new file mode 100644
index 0000000..4100028
--- /dev/null
+++ b/tests/tasks/scanner_helpers.py
@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+from flask import Flask
+
+BASE_URL = "https://dennisfink.me/blog/"
+SOURCE_URL = f"{BASE_URL}example/"
+
+
+def parse_html(html: str) -> BeautifulSoup:
+ return BeautifulSoup(html, "html5lib")
+
+
+def write_post(
+ root: Path,
+ slug: str,
+ content: str,
+ *,
+ canonical: str | None = None,
+ entry_url: str | None = None,
+) -> Path:
+ directory = root / slug
+ directory.mkdir(parents=True, exist_ok=True)
+
+ source_url = f"{BASE_URL}{slug}/"
+
+ if canonical is None:
+ canonical = source_url
+
+ if entry_url is None:
+ entry_url = source_url
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{canonical}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="{entry_url}"
+ >
+ Permalink
+ </a>
+
+ <div class="e-content">
+ {content}
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ return path
+
+
+def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None:
+ app.config["SOURCE_DIRECTORY"] = str(root)
+ app.config["SOURCE_BASE_URL"] = base_url
diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py
new file mode 100644
index 0000000..2fc30b1
--- /dev/null
+++ b/tests/tasks/test_discovery.py
@@ -0,0 +1,951 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import ModuleType
+from unittest.mock import Mock
+
+import httpx
+import pytest
+from flask import Flask
+
+from webmentions_ssg.tasks import discovery
+
+SOURCE_URL = "https://dennisfink.me/blog/example/"
+TARGET_URL = "https://example.com/post"
+ENDPOINT_URL = "https://example.com/webmention"
+STATUS_URL = "https://example.com/webmention/status/123"
+
+
+@pytest.fixture
+def sender_module(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import discovery
+
+ return discovery
+
+
+@pytest.mark.parametrize(
+ ("value", "expected"),
+ (
+ (
+ '<https://example.com/webmention>; rel="webmention"',
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ "<https://example.com/webmention>; rel=webmention",
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ '<https://example.com/webmention>; rel="webmention alternate"',
+ ("https://example.com/webmention", {"webmention", "alternate"}),
+ ),
+ (
+ '<https://example.com/webmention>; REL="WebMention Alternate"',
+ ("https://example.com/webmention", {"webmention", "alternate"}),
+ ),
+ (
+ (
+ '<https://example.com/webmention>; type="text/html"; '
+ 'rel="webmention"; title="Endpoint"'
+ ),
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ '<https://example.com/stylesheet>; type="text/css"',
+ ("https://example.com/stylesheet", set()),
+ ),
+ ),
+)
+def test_parse_link_value(
+ sender_module: ModuleType, value: str, expected: tuple[str, set[str]]
+) -> None:
+ assert sender_module.parse_link_value(value) == expected
+
+
+@pytest.mark.parametrize(
+ "value", ("", "https://example.com/webmention", "<https://example.com/webmention")
+)
+def test_parse_link_value_rejects_malformed_value(
+ sender_module: ModuleType, value: str
+) -> None:
+ assert sender_module.parse_link_value(value) is None
+
+
+@pytest.mark.parametrize(
+ ("target", "headers", "html", "expected_endpoint"),
+ (
+ pytest.param(
+ "https://example.com/test/1",
+ (("Link", "</test/1/webmention?head=true>; rel=webmention"),),
+ "",
+ "https://example.com/test/1/webmention?head=true",
+ id="1-http-link-unquoted-rel-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/2",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/2/webmention?head=true>;"
+ "rel=webmention"
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/2/webmention?head=true",
+ id="2-http-link-unquoted-rel-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/3",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/3/webmention"
+ >
+ """,
+ "https://example.com/test/3/webmention",
+ id="3-html-link-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/4",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="https://example.com/test/4/webmention"
+ >
+ """,
+ "https://example.com/test/4/webmention",
+ id="4-html-link-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/5",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="/test/5/webmention"
+ >
+ Endpoint
+ </a>
+ """,
+ "https://example.com/test/5/webmention",
+ id="5-html-a-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/6",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="https://example.com/test/6/webmention"
+ >
+ Endpoint
+ </a>
+ """,
+ "https://example.com/test/6/webmention",
+ id="6-html-a-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/7",
+ (
+ (
+ "LinK",
+ (
+ "<https://example.com/test/7/webmention?head=true>; "
+ "rel=webmention"
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/7/webmention?head=true",
+ id="7-http-link-strange-casing",
+ ),
+ pytest.param(
+ "https://example.com/test/8",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/8/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/8/webmention?head=true",
+ id="8-http-link-quoted-rel",
+ ),
+ pytest.param(
+ "https://example.com/test/9",
+ (),
+ """
+ <link
+ rel="webmention somethingelse"
+ href="https://example.com/test/9/webmention"
+ >
+ """,
+ "https://example.com/test/9/webmention",
+ id="9-multiple-html-rel-values",
+ ),
+ pytest.param(
+ "https://example.com/test/10",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/10/webmention?head=true>; "
+ 'rel="webmention somethingelse"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/10/webmention?head=true",
+ id="10-multiple-link-header-rel-values",
+ ),
+ pytest.param(
+ "https://example.com/test/11",
+ (("Link", '</test/11/webmention>; rel="webmention"'),),
+ """
+ <link
+ rel="webmention"
+ href="/test/11/webmention/error"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/11/webmention/error"
+ >
+ Wrong endpoint
+ </a>
+ """,
+ "https://example.com/test/11/webmention",
+ id="11-http-link-precedence",
+ ),
+ pytest.param(
+ "https://example.com/test/12",
+ (),
+ """
+ <link
+ rel="not-webmention"
+ href="/test/12/webmention/error"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/12/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/12/webmention",
+ id="12-exact-rel-match",
+ ),
+ pytest.param(
+ "https://example.com/test/13",
+ (),
+ """
+ <!--
+ <a
+ rel="webmention"
+ href="/test/13/webmention/error"
+ >
+ False endpoint
+ </a>
+ -->
+
+ <a
+ rel="webmention"
+ href="/test/13/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/13/webmention",
+ id="13-endpoint-inside-html-comment",
+ ),
+ pytest.param(
+ "https://example.com/test/14",
+ (),
+ """
+ <code>
+ &lt;a
+ rel="webmention"
+ href="/test/14/webmention/error"
+ &gt;
+ False endpoint
+ &lt;/a&gt;
+ </code>
+
+ <a
+ rel="webmention"
+ href="/test/14/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/14/webmention",
+ id="14-endpoint-in-escaped-html",
+ ),
+ pytest.param(
+ "https://example.com/test/15",
+ (),
+ """
+ <link
+ rel="webmention"
+ href=""
+ >
+ """,
+ "https://example.com/test/15",
+ id="15-empty-href",
+ ),
+ pytest.param(
+ "https://example.com/test/16",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="/test/16/webmention"
+ >
+ First endpoint
+ </a>
+
+ <link
+ rel="webmention"
+ href="/test/16/webmention/error"
+ >
+ """,
+ "https://example.com/test/16/webmention",
+ id="16-a-before-link",
+ ),
+ pytest.param(
+ "https://example.com/test/17",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/17/webmention"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/17/webmention/error"
+ >
+ Second endpoint
+ </a>
+ """,
+ "https://example.com/test/17/webmention",
+ id="17-link-before-a",
+ ),
+ pytest.param(
+ "https://example.com/test/18",
+ (
+ ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'),
+ (
+ "Link",
+ (
+ "<https://example.com/test/18/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/18/webmention?head=true",
+ id="18-multiple-http-link-headers",
+ ),
+ pytest.param(
+ "https://example.com/test/19",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/19/webmention/error>; "
+ 'rel="other", '
+ "<https://example.com/test/19/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/19/webmention?head=true",
+ id="19-single-header-multiple-values",
+ ),
+ pytest.param(
+ "https://example.com/test/20",
+ (),
+ """
+ <link rel="webmention">
+
+ <a
+ rel="webmention"
+ href="/test/20/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/20/webmention",
+ id="20-link-without-href",
+ ),
+ pytest.param(
+ "https://example.com/test/21",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/21/webmention?query=yes"
+ >
+ """,
+ "https://example.com/test/21/webmention?query=yes",
+ id="21-endpoint-query-string",
+ ),
+ pytest.param(
+ "https://example.com/test/22",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="22/webmention"
+ >
+ """,
+ "https://example.com/test/22/webmention",
+ id="22-endpoint-relative-to-path",
+ ),
+ ),
+)
+def test_discovery(
+ app: Flask,
+ sender_module: ModuleType,
+ target: str,
+ headers: tuple[tuple[str, str], ...],
+ html: str,
+ expected_endpoint: str,
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200, headers=headers)
+
+ return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html)
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, target)
+
+ assert endpoint == expected_endpoint
+
+
+@pytest.mark.parametrize(
+ ("target", "responses", "expected_endpoint"),
+ (
+ pytest.param(
+ "https://example.com/test/23/page",
+ {
+ "/test/23/page": (302, {"Location": "page/redirect-key"}),
+ "/test/23/page/redirect-key": (
+ 200,
+ {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")},
+ ),
+ },
+ ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"),
+ id="relative-endpoint-after-redirect",
+ ),
+ ),
+)
+def test_discovery_redirect(
+ app: Flask,
+ sender_module: ModuleType,
+ target: str,
+ responses: dict[str, tuple[int, dict[str, str]]],
+ expected_endpoint: str,
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ try:
+ status, headers = responses[request.url.path]
+ except KeyError:
+ raise AssertionError(f"Unexpected request: {request.url}") from None
+
+ return httpx.Response(status, headers=headers)
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, target)
+
+ assert endpoint == expected_endpoint
+
+
+def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(
+ client, "https://example.com/post"
+ )
+
+ assert endpoint == "https://example.com/webmention"
+ assert [request.method for request in requests] == ["HEAD"]
+
+
+def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text="""
+ <link
+ rel="webmention"
+ href="/webmention"
+ >
+ """,
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(
+ client, "https://example.com/post"
+ )
+
+ assert endpoint == "https://example.com/webmention"
+ assert [request.method for request in requests] == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ "failure",
+ [
+ pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"),
+ pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"),
+ ],
+)
+def test_discovery_falls_back_to_get_after_head_request_error(
+ app: Flask, sender_module: ModuleType, failure: httpx.RequestError
+) -> None:
+ methods: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+
+ if request.method == "HEAD":
+ raise failure
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert methods == ["HEAD", "GET"]
+
+
+def test_discovery_falls_back_to_get_after_head_dns_failure(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ methods: list[str] = []
+
+ def request_hook(request: httpx.Request) -> None:
+ if request.method == "HEAD":
+ raise sender_module.TemporarySenderError("HEAD DNS lookup failed")
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler),
+ event_hooks={"request": [request_hook]},
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert methods == ["GET"]
+
+
+def test_discovery_get_error_after_head_error_is_not_suppressed(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ methods: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+ raise httpx.ConnectError(f"{request.method} failed")
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ pytest.raises(httpx.ConnectError, match="GET failed"),
+ ):
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert methods == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ "invalid_href",
+ [
+ pytest.param("javascript:alert(1)", id="unsupported-scheme"),
+ pytest.param("http://[invalid", id="malformed-url"),
+ ],
+)
+def test_discovery_skips_invalid_html_candidate(
+ app: Flask, sender_module: ModuleType, invalid_href: str
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text=(
+ f'<link rel="webmention" href="{invalid_href}">'
+ '<a rel="webmention" href="/webmention">Valid endpoint</a>'
+ ),
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+
+
+def test_discovery_skips_invalid_link_header_candidate(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ headers=[
+ ("Link", "<mailto:invalid@example.com>; rel=webmention"),
+ ("Link", "<http://[invalid>; rel=webmention"),
+ ("Link", "</webmention>; rel=webmention"),
+ ],
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+
+
+def test_discovery_returns_none_when_all_html_candidates_are_invalid(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"})
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text='<link rel="webmention" href="javascript:void(0)">',
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint is None
+
+
+def test_resolve_endpoint_rejects_invalid_url(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ is_http_url = Mock(return_value=False)
+ monkeypatch.setattr(discovery, "is_http_url", is_http_url)
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+ with pytest.raises(
+ sender_module.PermanentSenderError, match="Invalid Webmention endpoint"
+ ):
+ sender_module.resolve_endpoint(response, "/webmention")
+ is_http_url.assert_called_once_with(ENDPOINT_URL)
+
+
+def test_endpoint_from_headers_ignores_malformed_link(
+ sender_module: ModuleType,
+) -> None:
+ response = httpx.Response(
+ 200,
+ headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")],
+ request=httpx.Request("GET", TARGET_URL),
+ )
+ assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL
+
+
+def test_endpoint_from_html_accepts_string_rel(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ element = Mock()
+ element.get.side_effect = lambda name: {
+ "rel": "webmention alternate",
+ "href": "/webmention",
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [element]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL
+
+
+def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ missing_rel = Mock()
+ missing_rel.get.return_value = None
+
+ invalid_href = Mock()
+ invalid_href.get.side_effect = lambda name: {
+ "rel": ["webmention"],
+ "href": ["invalid"],
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [missing_rel, invalid_href]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") is None
+
+
+def test_read_target_body_rejects_declared_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["MAX_TARGET_BYTES"] = 10
+
+ response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_read_target_body_ignores_invalid_content_length(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ response = httpx.Response(
+ 200, headers={"Content-Length": "invalid"}, content=b"Document"
+ )
+
+ with app.app_context():
+ body = sender_module.read_target_body(response)
+
+ assert body == b"Document"
+
+
+def test_read_target_body_rejects_streamed_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["MAX_TARGET_BYTES"] = 10
+
+ class BodyStream(httpx.SyncByteStream):
+ def __iter__(self):
+ yield b"x" * 6
+ yield b"x" * 6
+
+ response = httpx.Response(200, stream=BodyStream())
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_discovery_uses_link_header_from_get(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert [request.method for request in requests] == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ ("status_code", "exception_name"),
+ [
+ pytest.param(503, "TemporarySenderError", id="temporary"),
+ pytest.param(404, "PermanentSenderError", id="permanent"),
+ ],
+)
+def test_discovery_maps_get_status_to_exception(
+ app: Flask, sender_module: ModuleType, status_code: int, exception_name: str
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(status_code)
+
+ exception_type = getattr(sender_module, exception_name)
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"),
+ ):
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+
+def test_discovery_returns_none_for_unsupported_media_type(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200, headers={"Content-Type": "application/json"}, content=b"{}"
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint is None
+
+
+@pytest.mark.parametrize(
+ ("status_code", "location", "expected_status_url"),
+ [
+ pytest.param(
+ 201, "/webmention/status/123", STATUS_URL, id="created-with-location"
+ ),
+ pytest.param(202, None, None, id="accepted"),
+ ],
+)
+def test_post_webmention(
+ sender_module: ModuleType,
+ status_code: int,
+ location: str | None,
+ expected_status_url: str | None,
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ headers = {}
+
+ if location is not None:
+ headers["Location"] = location
+
+ return httpx.Response(status_code, headers=headers)
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ result = sender_module.post_webmention(
+ client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
+ )
+
+ assert result == (status_code, expected_status_url)
+ assert len(requests) == 1
+
+ request = requests[0]
+
+ assert request.method == "POST"
+ assert str(request.url) == ENDPOINT_URL
+
+ form = httpx.QueryParams(request.content.decode())
+
+ assert form["source"] == SOURCE_URL
+ assert form["target"] == TARGET_URL
+
+
+@pytest.mark.parametrize(
+ ("code", "location", "expected"),
+ [
+ (201, "/webmention/status/123", "https://example.com/webmention/status/123"),
+ (201, "https://status.example.org/123", "https://status.example.org/123"),
+ (201, "status/123", "https://example.com/status/123"),
+ (201, "javascript:alert(1)", None),
+ (201, "data:text/html,evil", None),
+ (201, "ftp://example.com/status", None),
+ (201, "http://[invalid", None),
+ (202, "https://example.com/status", None),
+ ],
+)
+def test_post_webmention_validates_location(
+ sender_module: ModuleType, code: int, location: str, expected: str | None
+) -> None:
+ def handler(_request: httpx.Request) -> httpx.Response:
+ return httpx.Response(code, headers={"Location": location})
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ result = sender_module.post_webmention(
+ client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
+ )
+
+ assert result == (code, expected)
diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py
index 108e8a6..c733dce 100644
--- a/tests/tasks/test_json_serializer.py
+++ b/tests/tasks/test_json_serializer.py
@@ -10,7 +10,7 @@ from datetime import UTC, datetime, timedelta
import pytest
from flask import Flask
-from huey.utils import Error, SKIPPED
+from huey.utils import SKIPPED, Error
from webmentions_ssg.tasks.extension import Huey
from webmentions_ssg.tasks.json_serializer import JSONSerializer
diff --git a/tests/tasks/test_mentions.py b/tests/tasks/test_mentions.py
new file mode 100644
index 0000000..a5da56c
--- /dev/null
+++ b/tests/tasks/test_mentions.py
@@ -0,0 +1,176 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import ModuleType
+from unittest.mock import Mock, patch
+
+import pytest
+from flask import Flask
+
+SOURCE_URL = "https://source.example/article"
+TARGET_URL = "https://dennisfink.me/blog/example/"
+
+
+@pytest.fixture
+def receiver(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import mentions
+
+ return mentions
+
+
+@pytest.mark.parametrize(
+ ("body", "expected"),
+ [
+ pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"),
+ pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"),
+ pytest.param(
+ f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href"
+ ),
+ pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"),
+ pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"),
+ pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"),
+ pytest.param(
+ f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src"
+ ),
+ pytest.param(
+ f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src"
+ ),
+ pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"),
+ pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"),
+ pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"),
+ pytest.param(
+ f'<video><track src="{TARGET_URL}"></video>', True, id="track-src"
+ ),
+ pytest.param(
+ f'<input type="image" src="{TARGET_URL}" alt="">',
+ True,
+ id="image-input-src",
+ ),
+ pytest.param(
+ f'<input type="IMAGE" src="{TARGET_URL}" alt="">',
+ True,
+ id="image-input-case-insensitive",
+ ),
+ pytest.param(
+ f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>',
+ True,
+ id="blockquote-cite",
+ ),
+ pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"),
+ pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"),
+ pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"),
+ pytest.param(
+ '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>',
+ True,
+ id="base-url",
+ ),
+ pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"),
+ pytest.param(
+ f'<blockquote src="{TARGET_URL}">Quote</blockquote>',
+ False,
+ id="blockquote-src-invalid",
+ ),
+ pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"),
+ pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"),
+ pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"),
+ pytest.param(
+ f'<input type="text" src="{TARGET_URL}">',
+ False,
+ id="text-input-src-invalid",
+ ),
+ pytest.param(
+ f'<picture><source src="{TARGET_URL}"></picture>',
+ False,
+ id="picture-source-src-invalid",
+ ),
+ pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"),
+ pytest.param(
+ f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url"
+ ),
+ pytest.param(
+ f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">',
+ False,
+ id="invalid-cite-does-not-override-valid-src",
+ ),
+ pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"),
+ pytest.param(
+ '<a href="https://example.com/">Other site</a>', False, id="missing-target"
+ ),
+ ],
+)
+def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None:
+ assert (
+ receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected
+ )
+
+
+def test_html_mentions_relative_target(receiver: ModuleType) -> None:
+ assert receiver.html_mentions_target(
+ b'<a href="../target/">Reply</a>',
+ "https://source.example/posts/article/",
+ "https://source.example/posts/target/",
+ )
+
+
+@pytest.mark.parametrize(
+ ("body", "target_url", "expected"),
+ [
+ (TARGET_URL, TARGET_URL, True),
+ (f"This post replies to {TARGET_URL}", TARGET_URL, True),
+ (
+ f"https://example.com/first {TARGET_URL} https://example.com/last",
+ TARGET_URL,
+ True,
+ ),
+ pytest.param(
+ f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma"
+ ),
+ pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"),
+ pytest.param(
+ f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation"
+ ),
+ pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"),
+ pytest.param(
+ "https://example.com/wiki/Foo_(bar)",
+ "https://example.com/wiki/Foo_(bar)",
+ True,
+ id="balanced-parentheses-in-url",
+ ),
+ pytest.param(
+ "https://example.com/wiki/Foo_(bar)",
+ "https://example.com/wiki/Foo_(bar",
+ False,
+ id="balanced-parentheses-not-trimmed",
+ ),
+ (f"{TARGET_URL}more", TARGET_URL, False),
+ pytest.param(
+ f"{TARGET_URL},another-page",
+ TARGET_URL,
+ False,
+ id="comma-within-longer-url",
+ ),
+ ("https://dennisfink.me/blog/other/", TARGET_URL, False),
+ ("/blog/example/", TARGET_URL, False),
+ ("There are no links here.", TARGET_URL, False),
+ ],
+)
+def test_text_mentions_target(
+ receiver: ModuleType, body: str, target_url: str, expected: bool
+) -> None:
+ assert receiver.text_mentions_target(body, target_url) is expected
+
+
+@patch("webmentions_ssg.tasks.mentions.BeautifulSoup")
+def test_html_mentions_target_ignores_non_string_attribute(
+ beautiful_soup: Mock, receiver: ModuleType
+) -> None:
+ document = beautiful_soup.return_value
+ document.select_one.return_value = None
+
+ element = Mock()
+ element.get.return_value = ["not", "a", "string"]
+ document.select.return_value = [element]
+
+ assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL)
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 44d1cc2..5441eb7 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -15,7 +15,7 @@ from flask import Flask
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import ReceivedWebmention
-from webmentions_ssg.url_security import AddressResolutionError
+from webmentions_ssg.net import AddressResolutionError
SOURCE_URL = "https://source.example/article"
TARGET_URL = "https://dennisfink.me/blog/example/"
@@ -68,148 +68,6 @@ def set_stream_response(httpx_client: Mock, response: httpx.Response) -> Mock:
return client
-@pytest.mark.parametrize(
- ("body", "expected"),
- [
- pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"),
- pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"),
- pytest.param(
- f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href"
- ),
- pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"),
- pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"),
- pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"),
- pytest.param(
- f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src"
- ),
- pytest.param(
- f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src"
- ),
- pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"),
- pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"),
- pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"),
- pytest.param(
- f'<video><track src="{TARGET_URL}"></video>', True, id="track-src"
- ),
- pytest.param(
- f'<input type="image" src="{TARGET_URL}" alt="">',
- True,
- id="image-input-src",
- ),
- pytest.param(
- f'<input type="IMAGE" src="{TARGET_URL}" alt="">',
- True,
- id="image-input-case-insensitive",
- ),
- pytest.param(
- f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>',
- True,
- id="blockquote-cite",
- ),
- pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"),
- pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"),
- pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"),
- pytest.param(
- '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>',
- True,
- id="base-url",
- ),
- pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"),
- pytest.param(
- f'<blockquote src="{TARGET_URL}">Quote</blockquote>',
- False,
- id="blockquote-src-invalid",
- ),
- pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"),
- pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"),
- pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"),
- pytest.param(
- f'<input type="text" src="{TARGET_URL}">',
- False,
- id="text-input-src-invalid",
- ),
- pytest.param(
- f'<picture><source src="{TARGET_URL}"></picture>',
- False,
- id="picture-source-src-invalid",
- ),
- pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"),
- pytest.param(
- f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url"
- ),
- pytest.param(
- f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">',
- False,
- id="invalid-cite-does-not-override-valid-src",
- ),
- pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"),
- pytest.param(
- '<a href="https://example.com/">Other site</a>', False, id="missing-target"
- ),
- ],
-)
-def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None:
- assert (
- receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected
- )
-
-
-def test_html_mentions_relative_target(receiver: ModuleType) -> None:
- assert receiver.html_mentions_target(
- b'<a href="../target/">Reply</a>',
- "https://source.example/posts/article/",
- "https://source.example/posts/target/",
- )
-
-
-@pytest.mark.parametrize(
- ("body", "target_url", "expected"),
- [
- (TARGET_URL, TARGET_URL, True),
- (f"This post replies to {TARGET_URL}", TARGET_URL, True),
- (
- f"https://example.com/first {TARGET_URL} https://example.com/last",
- TARGET_URL,
- True,
- ),
- pytest.param(
- f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma"
- ),
- pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"),
- pytest.param(
- f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation"
- ),
- pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"),
- pytest.param(
- "https://example.com/wiki/Foo_(bar)",
- "https://example.com/wiki/Foo_(bar)",
- True,
- id="balanced-parentheses-in-url",
- ),
- pytest.param(
- "https://example.com/wiki/Foo_(bar)",
- "https://example.com/wiki/Foo_(bar",
- False,
- id="balanced-parentheses-not-trimmed",
- ),
- (f"{TARGET_URL}more", TARGET_URL, False),
- pytest.param(
- f"{TARGET_URL},another-page",
- TARGET_URL,
- False,
- id="comma-within-longer-url",
- ),
- ("https://dennisfink.me/blog/other/", TARGET_URL, False),
- ("/blog/example/", TARGET_URL, False),
- ("There are no links here.", TARGET_URL, False),
- ],
-)
-def test_text_mentions_target(
- receiver: ModuleType, body: str, target_url: str, expected: bool
-) -> None:
- assert receiver.text_mentions_target(body, target_url) is expected
-
-
@patch("webmentions_ssg.tasks.receiver.httpx.Client")
def test_fetch_source_returns_response_and_body(
httpx_client: Mock, app: Flask, receiver: ModuleType
@@ -786,7 +644,7 @@ def test_verify_webmention_ignores_unknown_identifier(
def test_fetch_source_rejects_unsafe_destination(
httpx_client: Mock, app: Flask, receiver: ModuleType
) -> None:
- from webmentions_ssg.url_security import UnsafeAddressError
+ from webmentions_ssg.net import UnsafeAddressError
client = httpx_client.return_value.__enter__.return_value
client.stream.side_effect = UnsafeAddressError("Local address")
@@ -810,20 +668,6 @@ def test_fetch_source_treats_dns_failure_as_temporary(
receiver.fetch_source(SOURCE_URL)
-@patch("webmentions_ssg.tasks.receiver.BeautifulSoup")
-def test_html_mentions_target_ignores_non_string_attribute(
- beautiful_soup: Mock, receiver: ModuleType
-) -> None:
- document = beautiful_soup.return_value
- document.select_one.return_value = None
-
- element = Mock()
- element.get.return_value = ["not", "a", "string"]
- document.select.return_value = [element]
-
- assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL)
-
-
@patch("webmentions_ssg.tasks.receiver.httpx.Client")
def test_fetch_source_ignores_invalid_content_length(
httpx_client: Mock, app: Flask, receiver: ModuleType
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 26e81c4..7eb970c 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -18,9 +18,9 @@ from pytest import MonkeyPatch
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
+from webmentions_ssg.tasks import source_parser
-BASE_URL = "https://dennisfink.me/blog/"
-SOURCE_URL = f"{BASE_URL}example/"
+from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post
@pytest.fixture
@@ -29,6 +29,7 @@ def scanner_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
from webmentions_ssg.tasks import scanner
monkeypatch.setattr(scanner, "is_http_url", Mock(return_value=True))
+ monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True))
return scanner
@@ -80,764 +81,6 @@ def test_manual_scan_propagates_unrelated_errors(
scanner_module.manual_scan_sources.call_local()
-def parse_html(html: str) -> BeautifulSoup:
- return BeautifulSoup(html, "html5lib")
-
-
-def write_post(
- root: Path,
- slug: str,
- content: str,
- *,
- canonical: str | None = None,
- entry_url: str | None = None,
-) -> Path:
- directory = root / slug
- directory.mkdir(parents=True, exist_ok=True)
-
- source_url = f"{BASE_URL}{slug}/"
-
- if canonical is None:
- canonical = source_url
-
- if entry_url is None:
- entry_url = source_url
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{canonical}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{entry_url}"
- >
- Permalink
- </a>
-
- <div class="e-content">
- {content}
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- return path
-
-
-def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None:
- app.config["SOURCE_DIRECTORY"] = str(root)
- app.config["SOURCE_BASE_URL"] = base_url
-
-
-def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- entry = scanner_module.parse_entry(element, SOURCE_URL)
-
- assert "h-entry" in entry["type"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article>
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find("article")
-
- assert element is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one parsed h-entry"
- ):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry" id="source">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <p class="p-name">
- Example post
- </p>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element, entry = scanner_module.primary_entry(document, SOURCE_URL)
-
- assert element["id"] == "source"
- assert entry["properties"]["name"] == ["Example post"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <main>
- <p>No microformats here.</p>
- </main>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- First
- </a>
- </article>
-
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Second
- </a>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/nested/"
- >
- Nested
- </a>
- </article>
- </div>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Other
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_content_element_returns_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content" id="content">
- Hello world
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- content = scanner_module.content_element(entry)
-
- assert content["id"] == "content"
-
-
-def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 0"
- ):
- scanner_module.content_element(entry)
-
-
-def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- First
- </div>
-
- <div class="e-content">
- Second
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 2"
- ):
- scanner_module.content_element(entry)
-
-
-def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result == SOURCE_URL
-
-
-def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <title>Example</title>
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result is None
-
-
-def test_canonical_url_resolves_relative_url_with_base_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.side_effect = [False, True]
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
- assert result == ("https://dennisfink.me/blog/example/canonical/")
-
-
-def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="relative canonical"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
-
-def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href=" "
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_scan_source_uses_canonical_without_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
- assert len(scanned.content_hash) == 32
-
-
-def test_scan_source_ignores_markup_noise_in_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- "<p>Hello <strong>world</strong>!</p>"
- '<a href="https://example.com/target">Target</a>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- write_post(
- tmp_path,
- "example",
- '<p class="generated" data-build="123"> Hello world! </p>'
- '<a class="external" href="https://example.com/target"> Target </a>',
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_uses_stable_time_value_inside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_hash_changes_when_image_changes(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_includes_reaction_targets_outside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- path.write_text(
- html.replace(
- '<div class="e-content">',
- '<a class="u-like-of" href="https://example.com/first">Like</a>'
- '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- "https://example.com/first", "https://example.com/second"
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == frozenset({"https://example.com/first"})
- assert second.targets == frozenset({"https://example.com/second"})
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_ignores_reaction_target_order(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
- second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
- path.write_text(
- html.replace(
- '<div class="e-content">',
- first_link + second_link + '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- first_link + second_link, second_link + first_link
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_falls_back_to_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <a href="https://example.com/target">
- Target
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL)
-
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-def test_scan_source_requires_canonical_or_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <p>Hello world.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"):
- scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
-
-def test_scan_source_extracts_only_content_links(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a href="https://example.com/metadata">
- Metadata link
- </a>
-
- <div class="e-content">
- <a href="https://example.com/content">
- Content link
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/content"})
-
-
-def test_scan_source_extracts_relative_content_link(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="../other/">
- Other post
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"})
-
-
-@pytest.mark.parametrize(
- "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-)
-def test_scan_source_extracts_reaction_properties(
- scanner_module: ModuleType, tmp_path: Path, property_name: str
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- target = f"https://example.com/{property_name}"
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a
- class="u-{property_name}"
- href="{target}"
- >
- Reaction target
- </a>
-
- <div class="e-content">
- <p>Post content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({target})
-
-
-@pytest.mark.parametrize("tag", ("link", "area"))
-def test_scan_source_ignores_non_anchor_href_elements(
- scanner_module: ModuleType, tmp_path: Path, tag: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <{tag} href="https://example.com/target">
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset()
-
-
-def test_scan_source_ignores_self_fragment(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="#section">
- Section
- </a>
-
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_creates_source_and_webmentions(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1885,23 +1128,23 @@ def test_unexpected_parser_error_does_not_abort_other_sources(
write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>')
if parser == "bs4":
- original_soup = scanner_module.BeautifulSoup
+ original_soup = source_parser.BeautifulSoup
def failing_soup(markup: bytes, features: str) -> BeautifulSoup:
if b"/bad/" in markup:
raise failure
return original_soup(markup, features)
- monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup)
+ monkeypatch.setattr(source_parser, "BeautifulSoup", failing_soup)
else:
- original_parse = scanner_module.mf2py.parse
+ original_parse = source_parser.mf2py.parse
def failing_parse(*, doc: str, url: str) -> dict[str, Any]:
if "/bad/" in doc:
raise failure
return original_parse(doc=doc, url=url)
- monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse)
+ monkeypatch.setattr(source_parser.mf2py, "parse", failing_parse)
queued: list[UUID] = []
monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
@@ -1979,46 +1222,6 @@ def test_unexpected_persistence_error_is_not_suppressed(
run_scan(app, scanner_module)
-def test_scan_source_rejects_symlink_outside_root_before_read(
- scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- external = write_post(tmp_path / "outside", "private", "Sensitive content")
- linked = root / "escaped" / "index.html"
- linked.parent.mkdir()
- linked.symlink_to(external)
-
- read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
- monkeypatch.setattr(Path, "read_bytes", read_bytes)
-
- with pytest.raises(
- scanner_module.SourceScanError, match="escapes source directory"
- ):
- scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- read_bytes.assert_not_called()
-
-
-def test_scan_source_accepts_symlink_within_root(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- linked = write_post(
- root, "example", '<a href="https://example.com/target">Target</a>'
- )
- shared = root / "shared.html"
- linked.rename(shared)
- linked.symlink_to(shared)
-
- scanned = scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_sources_skips_symlink_outside_root(
app: Flask,
scanner_module: ModuleType,
@@ -2043,16 +1246,6 @@ def test_scan_sources_skips_symlink_outside_root(
assert "escapes source directory" in caplog.text
-def test_scan_sources_rejects_malformed_runtime_ignored_hostnames(
- app: Flask, scanner_module: ModuleType, tmp_path: Path
-) -> None:
- configure_scanner(app, tmp_path)
- app.config["IGNORED_HOSTNAMES"] = "example.com"
-
- with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"):
- run_scan(app, scanner_module)
-
-
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -2104,220 +1297,6 @@ def test_scan_sources_rejects_invalid_base_url(
run_scan(app, scanner_module)
-@pytest.mark.parametrize(
- "href",
- (
- "https://dennisfink.me/blog/other/",
- "https://www.dennisfink.me/blog/other/",
- "https://webmentions.dennisfink.me/endpoint",
- "https://foo.bar.dennisfink.me/example",
- ),
-)
-def test_scan_source_ignores_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Ignored
- </a>
-
- <a href="https://example.com/target">
- External
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-@pytest.mark.parametrize(
- "href",
- (
- "https://notdennisfink.me/example",
- "https://dennisfink.me.example.com/example",
- "https://example.com/example",
- ),
-)
-def test_scan_source_keeps_nonmatching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({href})
-
-
-def test_scan_source_ignores_reaction_to_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{SOURCE_URL}"
- >
- Permalink
- </a>
-
- <a
- class="u-in-reply-to"
- href="https://www.dennisfink.me/blog/other/"
- >
- Reply target
- </a>
-
- <div class="e-content">
- <p>Reply content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset()
-
-
-def test_canonical_url_rejects_non_http_resolved_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="mailto:example@example.com"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError,
- match="Canonical URL is not a valid HTTP or HTTPS URL",
- ):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_property_urls_ignores_non_mapping_properties(
- scanner_module: ModuleType,
-) -> None:
- entry = {"properties": "invalid"}
-
- assert tuple(scanner_module.property_urls(entry, "url")) == ()
-
-
-@pytest.mark.parametrize(
- ("parsed", "message"),
- [
- pytest.param(
- [],
- "Microformats parser did not return a document object",
- id="non-object-document",
- ),
- pytest.param(
- {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
- ),
- pytest.param(
- {"items": [{"type": ["h-card"]}]},
- "Parsed microformats item is not an h-entry",
- id="wrong-item-type",
- ),
- ],
-)
-def test_parse_entry_rejects_invalid_parser_output(
- scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str
-) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed)
-
- with pytest.raises(scanner_module.SourceScanError, match=message):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_normalize_target_ignores_empty_target(scanner_module: ModuleType) -> None:
- assert (
- scanner_module.normalize_target(" ", base_url=SOURCE_URL, source_url=SOURCE_URL)
- is None
- )
-
-
-def test_normalize_target_ignores_url_rejected_by_validator(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- assert (
- scanner_module.normalize_target(
- "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL
- )
- is None
- )
-
-
def test_changed_source_url_is_rejected(
app: Flask,
scanner_module: ModuleType,
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index 7ebe083..c2f6e02 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -14,7 +14,6 @@ from flask import Flask
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
-from webmentions_ssg.url_security import AddressResolutionError
SOURCE_URL = "https://dennisfink.me/blog/example/"
TARGET_URL = "https://example.com/post"
@@ -101,25 +100,6 @@ def sender_module(app: Flask) -> ModuleType:
@pytest.mark.parametrize(
- ("status", "expected"),
- [
- pytest.param(200, False, id="success"),
- pytest.param(400, False, id="client-error"),
- pytest.param(408, True, id="request-timeout"),
- pytest.param(425, True, id="too-early"),
- pytest.param(429, True, id="too-many-requests"),
- pytest.param(500, True, id="server-error-lower-bound"),
- pytest.param(599, True, id="server-error-upper-bound"),
- pytest.param(600, False, id="outside-server-error-range"),
- ],
-)
-def test_temporary_http_status(
- sender_module: ModuleType, status: int, expected: bool
-) -> None:
- assert sender_module.temporary_http_status(status) is expected
-
-
-@pytest.mark.parametrize(
("desired_revision", "processed_revision", "revision", "expected"),
[
pytest.param(2, None, 2, True, id="unprocessed-current"),
@@ -147,671 +127,6 @@ def test_attempt_is_current(
assert sender_module.attempt_is_current(webmention, revision) is expected
-@pytest.mark.parametrize(
- ("value", "expected"),
- (
- (
- '<https://example.com/webmention>; rel="webmention"',
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- "<https://example.com/webmention>; rel=webmention",
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- '<https://example.com/webmention>; rel="webmention alternate"',
- ("https://example.com/webmention", {"webmention", "alternate"}),
- ),
- (
- '<https://example.com/webmention>; REL="WebMention Alternate"',
- ("https://example.com/webmention", {"webmention", "alternate"}),
- ),
- (
- (
- '<https://example.com/webmention>; type="text/html"; '
- 'rel="webmention"; title="Endpoint"'
- ),
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- '<https://example.com/stylesheet>; type="text/css"',
- ("https://example.com/stylesheet", set()),
- ),
- ),
-)
-def test_parse_link_value(
- sender_module: ModuleType, value: str, expected: tuple[str, set[str]]
-) -> None:
- assert sender_module.parse_link_value(value) == expected
-
-
-@pytest.mark.parametrize(
- "value", ("", "https://example.com/webmention", "<https://example.com/webmention")
-)
-def test_parse_link_value_rejects_malformed_value(
- sender_module: ModuleType, value: str
-) -> None:
- assert sender_module.parse_link_value(value) is None
-
-
-@pytest.mark.parametrize(
- ("target", "headers", "html", "expected_endpoint"),
- (
- pytest.param(
- "https://example.com/test/1",
- (("Link", "</test/1/webmention?head=true>; rel=webmention"),),
- "",
- "https://example.com/test/1/webmention?head=true",
- id="1-http-link-unquoted-rel-relative-url",
- ),
- pytest.param(
- "https://example.com/test/2",
- (
- (
- "Link",
- (
- "<https://example.com/test/2/webmention?head=true>;"
- "rel=webmention"
- ),
- ),
- ),
- "",
- "https://example.com/test/2/webmention?head=true",
- id="2-http-link-unquoted-rel-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/3",
- (),
- """
- <link
- rel="webmention"
- href="/test/3/webmention"
- >
- """,
- "https://example.com/test/3/webmention",
- id="3-html-link-relative-url",
- ),
- pytest.param(
- "https://example.com/test/4",
- (),
- """
- <link
- rel="webmention"
- href="https://example.com/test/4/webmention"
- >
- """,
- "https://example.com/test/4/webmention",
- id="4-html-link-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/5",
- (),
- """
- <a
- rel="webmention"
- href="/test/5/webmention"
- >
- Endpoint
- </a>
- """,
- "https://example.com/test/5/webmention",
- id="5-html-a-relative-url",
- ),
- pytest.param(
- "https://example.com/test/6",
- (),
- """
- <a
- rel="webmention"
- href="https://example.com/test/6/webmention"
- >
- Endpoint
- </a>
- """,
- "https://example.com/test/6/webmention",
- id="6-html-a-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/7",
- (
- (
- "LinK",
- (
- "<https://example.com/test/7/webmention?head=true>; "
- "rel=webmention"
- ),
- ),
- ),
- "",
- "https://example.com/test/7/webmention?head=true",
- id="7-http-link-strange-casing",
- ),
- pytest.param(
- "https://example.com/test/8",
- (
- (
- "Link",
- (
- "<https://example.com/test/8/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/8/webmention?head=true",
- id="8-http-link-quoted-rel",
- ),
- pytest.param(
- "https://example.com/test/9",
- (),
- """
- <link
- rel="webmention somethingelse"
- href="https://example.com/test/9/webmention"
- >
- """,
- "https://example.com/test/9/webmention",
- id="9-multiple-html-rel-values",
- ),
- pytest.param(
- "https://example.com/test/10",
- (
- (
- "Link",
- (
- "<https://example.com/test/10/webmention?head=true>; "
- 'rel="webmention somethingelse"'
- ),
- ),
- ),
- "",
- "https://example.com/test/10/webmention?head=true",
- id="10-multiple-link-header-rel-values",
- ),
- pytest.param(
- "https://example.com/test/11",
- (("Link", '</test/11/webmention>; rel="webmention"'),),
- """
- <link
- rel="webmention"
- href="/test/11/webmention/error"
- >
-
- <a
- rel="webmention"
- href="/test/11/webmention/error"
- >
- Wrong endpoint
- </a>
- """,
- "https://example.com/test/11/webmention",
- id="11-http-link-precedence",
- ),
- pytest.param(
- "https://example.com/test/12",
- (),
- """
- <link
- rel="not-webmention"
- href="/test/12/webmention/error"
- >
-
- <a
- rel="webmention"
- href="/test/12/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/12/webmention",
- id="12-exact-rel-match",
- ),
- pytest.param(
- "https://example.com/test/13",
- (),
- """
- <!--
- <a
- rel="webmention"
- href="/test/13/webmention/error"
- >
- False endpoint
- </a>
- -->
-
- <a
- rel="webmention"
- href="/test/13/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/13/webmention",
- id="13-endpoint-inside-html-comment",
- ),
- pytest.param(
- "https://example.com/test/14",
- (),
- """
- <code>
- &lt;a
- rel="webmention"
- href="/test/14/webmention/error"
- &gt;
- False endpoint
- &lt;/a&gt;
- </code>
-
- <a
- rel="webmention"
- href="/test/14/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/14/webmention",
- id="14-endpoint-in-escaped-html",
- ),
- pytest.param(
- "https://example.com/test/15",
- (),
- """
- <link
- rel="webmention"
- href=""
- >
- """,
- "https://example.com/test/15",
- id="15-empty-href",
- ),
- pytest.param(
- "https://example.com/test/16",
- (),
- """
- <a
- rel="webmention"
- href="/test/16/webmention"
- >
- First endpoint
- </a>
-
- <link
- rel="webmention"
- href="/test/16/webmention/error"
- >
- """,
- "https://example.com/test/16/webmention",
- id="16-a-before-link",
- ),
- pytest.param(
- "https://example.com/test/17",
- (),
- """
- <link
- rel="webmention"
- href="/test/17/webmention"
- >
-
- <a
- rel="webmention"
- href="/test/17/webmention/error"
- >
- Second endpoint
- </a>
- """,
- "https://example.com/test/17/webmention",
- id="17-link-before-a",
- ),
- pytest.param(
- "https://example.com/test/18",
- (
- ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'),
- (
- "Link",
- (
- "<https://example.com/test/18/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/18/webmention?head=true",
- id="18-multiple-http-link-headers",
- ),
- pytest.param(
- "https://example.com/test/19",
- (
- (
- "Link",
- (
- "<https://example.com/test/19/webmention/error>; "
- 'rel="other", '
- "<https://example.com/test/19/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/19/webmention?head=true",
- id="19-single-header-multiple-values",
- ),
- pytest.param(
- "https://example.com/test/20",
- (),
- """
- <link rel="webmention">
-
- <a
- rel="webmention"
- href="/test/20/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/20/webmention",
- id="20-link-without-href",
- ),
- pytest.param(
- "https://example.com/test/21",
- (),
- """
- <link
- rel="webmention"
- href="/test/21/webmention?query=yes"
- >
- """,
- "https://example.com/test/21/webmention?query=yes",
- id="21-endpoint-query-string",
- ),
- pytest.param(
- "https://example.com/test/22",
- (),
- """
- <link
- rel="webmention"
- href="22/webmention"
- >
- """,
- "https://example.com/test/22/webmention",
- id="22-endpoint-relative-to-path",
- ),
- ),
-)
-def test_discovery(
- app: Flask,
- sender_module: ModuleType,
- target: str,
- headers: tuple[tuple[str, str], ...],
- html: str,
- expected_endpoint: str,
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200, headers=headers)
-
- return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html)
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, target)
-
- assert endpoint == expected_endpoint
-
-
-@pytest.mark.parametrize(
- ("target", "responses", "expected_endpoint"),
- (
- pytest.param(
- "https://example.com/test/23/page",
- {
- "/test/23/page": (302, {"Location": "page/redirect-key"}),
- "/test/23/page/redirect-key": (
- 200,
- {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")},
- ),
- },
- ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"),
- id="relative-endpoint-after-redirect",
- ),
- ),
-)
-def test_discovery_redirect(
- app: Flask,
- sender_module: ModuleType,
- target: str,
- responses: dict[str, tuple[int, dict[str, str]]],
- expected_endpoint: str,
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- try:
- status, headers = responses[request.url.path]
- except KeyError:
- raise AssertionError(f"Unexpected request: {request.url}") from None
-
- return httpx.Response(status, headers=headers)
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, target)
-
- assert endpoint == expected_endpoint
-
-
-def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(
- client, "https://example.com/post"
- )
-
- assert endpoint == "https://example.com/webmention"
- assert [request.method for request in requests] == ["HEAD"]
-
-
-def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text="""
- <link
- rel="webmention"
- href="/webmention"
- >
- """,
- )
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(
- client, "https://example.com/post"
- )
-
- assert endpoint == "https://example.com/webmention"
- assert [request.method for request in requests] == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- "failure",
- [
- pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"),
- pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"),
- ],
-)
-def test_discovery_falls_back_to_get_after_head_request_error(
- app: Flask, sender_module: ModuleType, failure: httpx.RequestError
-) -> None:
- methods: list[str] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
-
- if request.method == "HEAD":
- raise failure
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert methods == ["HEAD", "GET"]
-
-
-def test_discovery_falls_back_to_get_after_head_dns_failure(
- app: Flask, sender_module: ModuleType
-) -> None:
- methods: list[str] = []
-
- def request_hook(request: httpx.Request) -> None:
- if request.method == "HEAD":
- raise sender_module.TemporarySenderError("HEAD DNS lookup failed")
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler),
- event_hooks={"request": [request_hook]},
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert methods == ["GET"]
-
-
-def test_discovery_get_error_after_head_error_is_not_suppressed(
- app: Flask, sender_module: ModuleType
-) -> None:
- methods: list[str] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
- raise httpx.ConnectError(f"{request.method} failed")
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- pytest.raises(httpx.ConnectError, match="GET failed"),
- ):
- sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert methods == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- "invalid_href",
- [
- pytest.param("javascript:alert(1)", id="unsupported-scheme"),
- pytest.param("http://[invalid", id="malformed-url"),
- ],
-)
-def test_discovery_skips_invalid_html_candidate(
- app: Flask, sender_module: ModuleType, invalid_href: str
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text=(
- f'<link rel="webmention" href="{invalid_href}">'
- '<a rel="webmention" href="/webmention">Valid endpoint</a>'
- ),
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
-
-
-def test_discovery_skips_invalid_link_header_candidate(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- return httpx.Response(
- 200,
- headers=[
- ("Link", "<mailto:invalid@example.com>; rel=webmention"),
- ("Link", "<http://[invalid>; rel=webmention"),
- ("Link", "</webmention>; rel=webmention"),
- ],
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
-
-
-def test_discovery_returns_none_when_all_html_candidates_are_invalid(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"})
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text='<link rel="webmention" href="javascript:void(0)">',
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint is None
-
-
def test_send_webmention_persists_success(
app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -1315,267 +630,6 @@ def test_send_webmention_ignores_unknown_identifier(
assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text
-def test_resolve_endpoint_rejects_invalid_url(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- is_http_url = Mock(return_value=False)
- monkeypatch.setattr(sender_module, "is_http_url", is_http_url)
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
- with pytest.raises(
- sender_module.PermanentSenderError, match="Invalid Webmention endpoint"
- ):
- sender_module.resolve_endpoint(response, "/webmention")
- is_http_url.assert_called_once_with(ENDPOINT_URL)
-
-
-def test_endpoint_from_headers_ignores_malformed_link(
- sender_module: ModuleType,
-) -> None:
- response = httpx.Response(
- 200,
- headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")],
- request=httpx.Request("GET", TARGET_URL),
- )
- assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL
-
-
-def test_endpoint_from_html_accepts_string_rel(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- element = Mock()
- element.get.side_effect = lambda name: {
- "rel": "webmention alternate",
- "href": "/webmention",
- }[name]
-
- document = Mock()
- document.find_all.return_value = [element]
-
- beautiful_soup = Mock(return_value=document)
- monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
-
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
-
- assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL
-
-
-def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- missing_rel = Mock()
- missing_rel.get.return_value = None
-
- invalid_href = Mock()
- invalid_href.get.side_effect = lambda name: {
- "rel": ["webmention"],
- "href": ["invalid"],
- }[name]
-
- document = Mock()
- document.find_all.return_value = [missing_rel, invalid_href]
-
- beautiful_soup = Mock(return_value=document)
- monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
-
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
-
- assert sender_module.endpoint_from_html(response, b"") is None
-
-
-def test_read_target_body_rejects_declared_oversized_body(
- app: Flask, sender_module: ModuleType
-) -> None:
- app.config["MAX_TARGET_BYTES"] = 10
-
- response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
-
- with (
- app.app_context(),
- pytest.raises(
- sender_module.PermanentSenderError, match="Target document is too large"
- ),
- ):
- sender_module.read_target_body(response)
-
-
-def test_read_target_body_ignores_invalid_content_length(
- app: Flask, sender_module: ModuleType
-) -> None:
- response = httpx.Response(
- 200, headers={"Content-Length": "invalid"}, content=b"Document"
- )
-
- with app.app_context():
- body = sender_module.read_target_body(response)
-
- assert body == b"Document"
-
-
-def test_read_target_body_rejects_streamed_oversized_body(
- app: Flask, sender_module: ModuleType
-) -> None:
- app.config["MAX_TARGET_BYTES"] = 10
-
- class BodyStream(httpx.SyncByteStream):
- def __iter__(self):
- yield b"x" * 6
- yield b"x" * 6
-
- response = httpx.Response(200, stream=BodyStream())
-
- with (
- app.app_context(),
- pytest.raises(
- sender_module.PermanentSenderError, match="Target document is too large"
- ),
- ):
- sender_module.read_target_body(response)
-
-
-def test_discovery_uses_link_header_from_get(
- app: Flask, sender_module: ModuleType
-) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert [request.method for request in requests] == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- ("status_code", "exception_name"),
- [
- pytest.param(503, "TemporarySenderError", id="temporary"),
- pytest.param(404, "PermanentSenderError", id="permanent"),
- ],
-)
-def test_discovery_maps_get_status_to_exception(
- app: Flask, sender_module: ModuleType, status_code: int, exception_name: str
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(status_code)
-
- exception_type = getattr(sender_module, exception_name)
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"),
- ):
- sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
-
-def test_discovery_returns_none_for_unsupported_media_type(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200, headers={"Content-Type": "application/json"}, content=b"{}"
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint is None
-
-
-@pytest.mark.parametrize(
- ("status_code", "location", "expected_status_url"),
- [
- pytest.param(
- 201, "/webmention/status/123", STATUS_URL, id="created-with-location"
- ),
- pytest.param(202, None, None, id="accepted"),
- ],
-)
-def test_post_webmention(
- sender_module: ModuleType,
- status_code: int,
- location: str | None,
- expected_status_url: str | None,
-) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- headers = {}
-
- if location is not None:
- headers["Location"] = location
-
- return httpx.Response(status_code, headers=headers)
-
- with httpx.Client(transport=httpx.MockTransport(handler)) as client:
- result = sender_module.post_webmention(
- client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
- )
-
- assert result == (status_code, expected_status_url)
- assert len(requests) == 1
-
- request = requests[0]
-
- assert request.method == "POST"
- assert str(request.url) == ENDPOINT_URL
-
- form = httpx.QueryParams(request.content.decode())
-
- assert form["source"] == SOURCE_URL
- assert form["target"] == TARGET_URL
-
-
-@pytest.mark.parametrize(
- ("code", "location", "expected"),
- [
- (201, "/webmention/status/123", "https://example.com/webmention/status/123"),
- (201, "https://status.example.org/123", "https://status.example.org/123"),
- (201, "status/123", "https://example.com/status/123"),
- (201, "javascript:alert(1)", None),
- (201, "data:text/html,evil", None),
- (201, "ftp://example.com/status", None),
- (201, "http://[invalid", None),
- (202, "https://example.com/status", None),
- ],
-)
-def test_post_webmention_validates_location(
- sender_module: ModuleType, code: int, location: str, expected: str | None
-) -> None:
- def handler(_request: httpx.Request) -> httpx.Response:
- return httpx.Response(code, headers={"Location": location})
-
- with httpx.Client(transport=httpx.MockTransport(handler)) as client:
- result = sender_module.post_webmention(
- client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
- )
-
- assert result == (code, expected)
-
-
@pytest.mark.parametrize(
("status_url", "expected"),
[
diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py
new file mode 100644
index 0000000..3c3a93e
--- /dev/null
+++ b/tests/tasks/test_source_parser.py
@@ -0,0 +1,1008 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from pathlib import Path
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+from flask import Flask
+from pytest import MonkeyPatch
+
+from webmentions_ssg.tasks import source_parser
+
+from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post
+
+
+@pytest.fixture
+def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import source_parser
+
+ monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True))
+ return source_parser
+
+
+def test_parse_entry_returns_microformats_entry(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ element = document.find(class_="h-entry")
+
+ assert element is not None
+
+ entry = source_parser_module.parse_entry(element, SOURCE_URL)
+
+ assert "h-entry" in entry["type"]
+ assert entry["properties"]["url"] == [SOURCE_URL]
+
+
+def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article>
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ element = document.find("article")
+
+ assert element is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one parsed h-entry",
+ ):
+ source_parser_module.parse_entry(element, SOURCE_URL)
+
+
+def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry" id="source">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <p class="p-name">
+ Example post
+ </p>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ element, entry = source_parser_module.primary_entry(document, SOURCE_URL)
+
+ assert element["id"] == "source"
+ assert entry["properties"]["name"] == ["Example post"]
+ assert entry["properties"]["url"] == [SOURCE_URL]
+
+
+def test_primary_entry_rejects_missing_h_entry(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <main>
+ <p>No microformats here.</p>
+ </main>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 0",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_multiple_h_entries(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ First
+ </a>
+ </article>
+
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/other/"
+ >
+ Second
+ </a>
+ </article>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 2",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/nested/"
+ >
+ Nested
+ </a>
+ </article>
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 2",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="does not match"):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_nonmatching_u_url(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/other/"
+ >
+ Other
+ </a>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="does not match"):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content" id="content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ content = source_parser_module.content_element(entry)
+
+ assert content["id"] == "content"
+
+
+def test_content_element_rejects_missing_e_content(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one e-content, found 0",
+ ):
+ source_parser_module.content_element(entry)
+
+
+def test_content_element_rejects_multiple_e_content(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content">
+ First
+ </div>
+
+ <div class="e-content">
+ Second
+ </div>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one e-content, found 2",
+ ):
+ source_parser_module.content_element(entry)
+
+
+def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+ assert result == SOURCE_URL
+
+
+def test_canonical_url_returns_none_when_missing(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <title>Example</title>
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+ assert result is None
+
+
+def test_canonical_url_resolves_relative_url_with_base_url(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.side_effect = [False, True]
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="./canonical/"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+ assert result == ("https://dennisfink.me/blog/example/canonical/")
+
+
+def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="./canonical/"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="relative canonical"
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+
+def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href=" "
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="empty canonical URL"
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+
+def test_scan_source_uses_canonical_without_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="https://example.com/target">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.path == "example/index.html"
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+ assert len(scanned.content_hash) == 32
+
+
+def test_scan_source_ignores_markup_noise_in_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ "<p>Hello <strong>world</strong>!</p>"
+ '<a href="https://example.com/target">Target</a>',
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ write_post(
+ tmp_path,
+ "example",
+ '<p class="generated" data-build="123"> Hello world! </p>'
+ '<a class="external" href="https://example.com/target"> Target </a>',
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_uses_stable_time_value_inside_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_hash_changes_when_image_changes(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_includes_reaction_targets_outside_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<a class="u-like-of" href="https://example.com/first">Like</a>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ "https://example.com/first", "https://example.com/second"
+ ),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == frozenset({"https://example.com/first"})
+ assert second.targets == frozenset({"https://example.com/second"})
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_ignores_reaction_target_order(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
+ second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ first_link + second_link + '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ first_link + second_link, second_link + first_link
+ ),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_falls_back_to_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ """
+ <!doctype html>
+ <html>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="./">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <a href="https://example.com/target">
+ Target
+ </a>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path, root=tmp_path, base_url=BASE_URL
+ )
+
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+def test_scan_source_requires_canonical_or_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ """
+ <!doctype html>
+ <html>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="./">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <p>Hello world.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"):
+ source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+
+def test_scan_source_extracts_only_content_links(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <a href="https://example.com/metadata">
+ Metadata link
+ </a>
+
+ <div class="e-content">
+ <a href="https://example.com/content">
+ Content link
+ </a>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://example.com/content"})
+
+
+def test_scan_source_extracts_relative_content_link(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="../other/">
+ Other post
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"})
+
+
+@pytest.mark.parametrize(
+ "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+)
+def test_scan_source_extracts_reaction_properties(
+ source_parser_module: ModuleType, tmp_path: Path, property_name: str
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ target = f"https://example.com/{property_name}"
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <a
+ class="u-{property_name}"
+ href="{target}"
+ >
+ Reaction target
+ </a>
+
+ <div class="e-content">
+ <p>Post content.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({target})
+
+
+@pytest.mark.parametrize("tag", ("link", "area"))
+def test_scan_source_ignores_non_anchor_href_elements(
+ source_parser_module: ModuleType, tmp_path: Path, tag: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <{tag} href="https://example.com/target">
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset()
+
+
+def test_scan_source_ignores_self_fragment(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="#section">
+ Section
+ </a>
+
+ <a href="https://example.com/target">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+def test_scan_source_rejects_symlink_outside_root_before_read(
+ source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ external = write_post(tmp_path / "outside", "private", "Sensitive content")
+ linked = root / "escaped" / "index.html"
+ linked.parent.mkdir()
+ linked.symlink_to(external)
+
+ read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
+ monkeypatch.setattr(Path, "read_bytes", read_bytes)
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="escapes source directory"
+ ):
+ source_parser_module.scan_source_file(linked, root=root, base_url=None)
+
+ read_bytes.assert_not_called()
+
+
+def test_scan_source_accepts_symlink_within_root(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ linked = write_post(
+ root, "example", '<a href="https://example.com/target">Target</a>'
+ )
+ shared = root / "shared.html"
+ linked.rename(shared)
+ linked.symlink_to(shared)
+
+ scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None)
+
+ assert scanned.path == "example/index.html"
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+@pytest.mark.parametrize(
+ "href",
+ (
+ "https://dennisfink.me/blog/other/",
+ "https://www.dennisfink.me/blog/other/",
+ "https://webmentions.dennisfink.me/endpoint",
+ "https://foo.bar.dennisfink.me/example",
+ ),
+)
+def test_scan_source_ignores_matching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path, href: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <a href="{href}">
+ Ignored
+ </a>
+
+ <a href="https://example.com/target">
+ External
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+@pytest.mark.parametrize(
+ "href",
+ (
+ "https://notdennisfink.me/example",
+ "https://dennisfink.me.example.com/example",
+ "https://example.com/example",
+ ),
+)
+def test_scan_source_keeps_nonmatching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path, href: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <a href="{href}">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset({href})
+
+
+def test_scan_source_ignores_reaction_to_matching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="{SOURCE_URL}"
+ >
+ Permalink
+ </a>
+
+ <a
+ class="u-in-reply-to"
+ href="https://www.dennisfink.me/blog/other/"
+ >
+ Reply target
+ </a>
+
+ <div class="e-content">
+ <p>Reply content.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset()
+
+
+def test_canonical_url_rejects_non_http_resolved_url(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="mailto:example@example.com"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Canonical URL is not a valid HTTP or HTTPS URL",
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+
+def test_property_urls_ignores_non_mapping_properties(
+ source_parser_module: ModuleType,
+) -> None:
+ entry = {"properties": "invalid"}
+
+ assert tuple(source_parser_module.property_urls(entry, "url")) == ()
+
+
+@pytest.mark.parametrize(
+ ("parsed", "message"),
+ [
+ pytest.param(
+ [],
+ "Microformats parser did not return a document object",
+ id="non-object-document",
+ ),
+ pytest.param(
+ {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
+ ),
+ pytest.param(
+ {"items": [{"type": ["h-card"]}]},
+ "Parsed microformats item is not an h-entry",
+ id="wrong-item-type",
+ ),
+ ],
+)
+def test_parse_entry_rejects_invalid_parser_output(
+ source_parser_module: ModuleType,
+ monkeypatch: MonkeyPatch,
+ parsed: object,
+ message: str,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ element = document.find(class_="h-entry")
+
+ assert element is not None
+
+ monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed)
+
+ with pytest.raises(source_parser_module.SourceScanError, match=message):
+ source_parser_module.parse_entry(element, SOURCE_URL)
+
+
+def test_normalize_target_ignores_empty_target(
+ source_parser_module: ModuleType,
+) -> None:
+ assert (
+ source_parser_module.normalize_target(
+ " ", base_url=SOURCE_URL, source_url=SOURCE_URL
+ )
+ is None
+ )
+
+
+def test_normalize_target_ignores_url_rejected_by_validator(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ assert (
+ source_parser_module.normalize_target(
+ "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL
+ )
+ is None
+ )
diff --git a/tests/test_auth.py b/tests/test_auth.py
index a6117b2..3b87923 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -4,10 +4,11 @@
"""Test the session user loader."""
+import pytest
from flask import Flask
from webmentions_ssg import DATABASE as db
-from webmentions_ssg.auth import AUTH, load_user
+from webmentions_ssg.auth import AUTH, is_safe_next_url, load_user
from webmentions_ssg.models import User
@@ -89,3 +90,33 @@ def test_password_change_invalidates_existing_session(app: Flask) -> None:
)
assert response.status_code == 302
assert client.get("/received").status_code == 200
+
+
+@pytest.mark.parametrize(
+ ("next_url", "expected"),
+ [
+ pytest.param("/", True, id="root"),
+ pytest.param("/received", True, id="local-path"),
+ pytest.param("/received?page=2", True, id="query-string"),
+ pytest.param("/received#details", True, id="fragment"),
+ pytest.param(
+ "/sent?next=https://evil.example", True, id="external-url-in-query"
+ ),
+ pytest.param("/foo//bar", True, id="double-slash-within-path"),
+ pytest.param("", False, id="empty"),
+ pytest.param("received", False, id="relative-path"),
+ pytest.param(" /received", False, id="leading-space"),
+ pytest.param("https://evil.example/phishing", False, id="absolute-https"),
+ pytest.param("http://evil.example", False, id="absolute-http"),
+ pytest.param("javascript:alert(1)", False, id="non-http-scheme"),
+ pytest.param("//evil.example/phishing", False, id="scheme-relative"),
+ pytest.param("///evil.example/phishing", False, id="triple-slash"),
+ pytest.param("////evil.example", False, id="four-slashes"),
+ pytest.param("//", False, id="two-slashes"),
+ pytest.param(r"/\evil.example", False, id="slash-backslash-host"),
+ pytest.param(r"/received\evil.example", False, id="backslash-in-path"),
+ pytest.param(r"\/evil.example", False, id="backslash-slash-host"),
+ ],
+)
+def test_is_safe_next_url(next_url: str, expected: bool) -> None:
+ assert is_safe_next_url(next_url) is expected
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..f6b7051
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,650 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import logging
+from collections.abc import Iterator
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+from alembic.script import ScriptDirectory
+from flask import Flask
+from flask_migrate import upgrade as migrate_upgrade
+from sqlalchemy import inspect
+from sqlalchemy.exc import IntegrityError
+
+from webmentions_ssg import DATABASE, MIGRATE, create_app
+from webmentions_ssg.config import TestingConfig
+from webmentions_ssg.models import User
+
+
+def test_cli_groups_are_registered(app: Flask) -> None:
+ commands = app.cli.commands
+ assert {"users", "mail"}.issubset(commands)
+
+
+@pytest.fixture
+def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]:
+ monkeypatch.chdir(tmp_path)
+
+ class DatabaseTestingConfig(TestingConfig):
+ LOG_FILE = str(tmp_path / "database-cli.log")
+ SQLALCHEMY_DATABASE_URI = f"sqlite:///{tmp_path / 'database.sqlite'}"
+
+ app = create_app(DatabaseTestingConfig)
+ yield app
+
+ with app.app_context():
+ DATABASE.session.remove()
+ DATABASE.engine.dispose()
+
+
+def _script_directory() -> ScriptDirectory:
+ """Open the migration scripts configured for this application."""
+ return ScriptDirectory(str(MIGRATE.directory))
+
+
+def _head_revision(app: Flask) -> str:
+ with app.app_context():
+ revision = _script_directory().get_current_head()
+ assert revision is not None, "Expected a migration head"
+ return revision
+
+
+def _database_revision(app: Flask) -> str:
+ with app.app_context(), DATABASE.engine.connect() as connection:
+ return connection.execute(
+ sa.text("SELECT version_num FROM alembic_version")
+ ).scalar_one()
+
+
+def test_database_commands_are_registered_without_hiding_flask_migrate(
+ database_app: Flask,
+) -> None:
+ runner = database_app.test_cli_runner()
+
+ result = runner.invoke(args=["db", "--help"])
+ assert result.exit_code == 0, result.output
+ assert "create-db" in result.output
+ assert "upgrade" in result.output
+
+ result = runner.invoke(args=["migrate", "--help"])
+ assert result.exit_code == 0, result.output
+ assert "downgrade" in result.output
+ assert "revision" in result.output
+
+
+def test_create_db_builds_schema_and_records_latest_migration(
+ database_app: Flask,
+) -> None:
+ result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
+
+ assert result.exit_code == 0, result.output
+ assert "Database created" in result.output
+ assert _database_revision(database_app) == _head_revision(database_app)
+
+ with database_app.app_context():
+ inspector = inspect(DATABASE.engine)
+ table_names = set(inspector.get_table_names())
+ columns = {
+ column["name"] for column in inspector.get_columns("sent_webmentions")
+ }
+ received_columns = {
+ column["name"] for column in inspector.get_columns("received_webmentions")
+ }
+ received_indexes = {
+ index["name"] for index in inspector.get_indexes("received_webmentions")
+ }
+ assert {"alembic_version", "users", "received_webmentions"} <= table_names
+ assert {"attempt_count", "next_retry_at"} <= columns
+ assert "recovery_attempts" in received_columns
+ assert "ix_received_webmentions_status_updated_at" in received_indexes
+
+
+def test_create_db_refuses_existing_database_without_modifying_it(
+ database_app: Flask,
+) -> None:
+ runner = database_app.test_cli_runner()
+ assert runner.invoke(args=["db", "create-db"]).exit_code == 0
+ revision_before = _database_revision(database_app)
+
+ result = runner.invoke(args=["db", "create-db"])
+
+ assert result.exit_code != 0
+ assert "already contains tables" in result.output
+ assert _database_revision(database_app) == revision_before
+
+
+def test_create_db_refuses_unversioned_existing_tables(database_app: Flask) -> None:
+ with database_app.app_context(), DATABASE.engine.begin() as connection:
+ connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)"))
+
+ result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
+
+ assert result.exit_code != 0
+ assert "already contains tables" in result.output
+ with database_app.app_context():
+ assert "legacy_data" in inspect(DATABASE.engine).get_table_names()
+ assert "alembic_version" not in inspect(DATABASE.engine).get_table_names()
+
+
+def test_upgrade_requires_an_initialized_database(database_app: Flask) -> None:
+ result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
+
+ assert result.exit_code != 0
+ assert "flask db create-db" in result.output
+
+
+def test_upgrade_refuses_unversioned_existing_database(database_app: Flask) -> None:
+ with database_app.app_context(), DATABASE.engine.begin() as connection:
+ connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)"))
+
+ result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
+
+ assert result.exit_code != 0
+ assert "No Flask-Migrate revision table" in result.output
+ with database_app.app_context():
+ assert "legacy_data" in inspect(DATABASE.engine).get_table_names()
+
+
+def test_upgrade_applies_pending_migrations_and_preserves_data(
+ database_app: Flask,
+) -> None:
+ # Start with the first migration, as if this were an older deployment.
+ with database_app.app_context():
+ base_revision = _script_directory().get_base()
+ assert base_revision is not None
+ assert base_revision != _head_revision(database_app)
+ migrate_upgrade(revision=base_revision)
+
+ with DATABASE.engine.begin() as connection:
+ connection.execute(
+ sa.text(
+ "INSERT INTO users (username, password_hash) "
+ "VALUES (:username, :password_hash)"
+ ),
+ {"username": "existing-user", "password_hash": "existing-hash"},
+ )
+
+ result = database_app.test_cli_runner().invoke(args=["db", "upgrade"])
+
+ assert result.exit_code == 0, result.output
+ assert "Database upgraded" in result.output
+ assert _database_revision(database_app) == _head_revision(database_app)
+
+ with database_app.app_context(), DATABASE.engine.connect() as connection:
+ assert (
+ connection.execute(
+ sa.text("SELECT password_hash FROM users WHERE username = :username"),
+ {"username": "existing-user"},
+ ).scalar_one()
+ == "existing-hash"
+ )
+ assert "sent_webmentions" in inspect(connection).get_table_names()
+
+
+def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None:
+ runner = database_app.test_cli_runner()
+ assert runner.invoke(args=["db", "create-db"]).exit_code == 0
+
+ result = runner.invoke(args=["db", "upgrade"])
+
+ assert result.exit_code == 0, result.output
+ assert _database_revision(database_app) == _head_revision(database_app)
+
+
+def test_create_db_does_not_report_success_when_migration_fails(
+ database_app: Flask, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import webmentions_ssg.cli as database_cli
+
+ def fail_upgrade() -> None:
+ raise RuntimeError("simulated migration failure")
+
+ monkeypatch.setattr(database_cli, "migrate_upgrade", fail_upgrade)
+
+ result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
+
+ assert result.exit_code != 0
+ assert isinstance(result.exception, RuntimeError)
+ assert "Database created" not in result.output
+
+
+def test_create_db_preserves_logging(
+ database_app: Flask, caplog: pytest.LogCaptureFixture
+) -> None:
+ with caplog.at_level(logging.WARNING):
+ result = database_app.test_cli_runner().invoke(args=["db", "create-db"])
+
+ assert result.exit_code == 0, result.output
+
+ database_app.logger.warning("Logging still works after migration")
+
+ assert "Logging still works after migration" in caplog.text
+
+
+def test_upgrade_backfills_portable_digest_for_existing_received_rows(
+ database_app: Flask,
+) -> None:
+ """Upgrade a pre-fingerprint SQLite database without losing historical rows."""
+ from webmentions_ssg.models import received_url_digest
+
+ with database_app.app_context():
+ migrate_upgrade(revision="c0272f25ce17")
+ with DATABASE.engine.begin() as connection:
+ connection.execute(
+ sa.text(
+ "INSERT INTO received_webmentions "
+ "(uuid, source, target, status, updated_at) "
+ "VALUES (:uuid, :source, :target, :status, :updated_at)"
+ ),
+ {
+ "uuid": "0" * 31 + "1",
+ "source": "https://source.example/old",
+ "target": "https://dennisfink.me/target",
+ "status": "verified",
+ "updated_at": "2026-01-01 00:00:00",
+ },
+ )
+ migrate_upgrade()
+ with DATABASE.engine.connect() as connection:
+ result = connection.execute(
+ sa.text(
+ "SELECT source, target, status, url_digest "
+ "FROM received_webmentions"
+ )
+ ).one()
+ assert result.status == "verified"
+ recovery_attempts = connection.execute(
+ sa.text("SELECT recovery_attempts FROM received_webmentions")
+ ).scalar_one()
+ assert recovery_attempts == 0
+ assert result.url_digest == received_url_digest(
+ result.source, result.target
+ )
+ constraints = inspect(connection).get_unique_constraints(
+ "received_webmentions"
+ )
+ assert any(
+ constraint["column_names"] == ["url_digest"]
+ for constraint in constraints
+ )
+
+
+def test_mail_cli_sends_to_configured_notification_email(app: Flask) -> None:
+ app.config.update(
+ NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
+ )
+
+ with patch("webmentions_ssg.MAIL.send") as send:
+ result = app.test_cli_runner().invoke(args=["mail", "test"])
+
+ assert result.exit_code == 0, result.output
+ send.assert_called_once()
+ message = send.call_args.args[0]
+ assert message.sender == "sender@example.com"
+ assert message.recipients == ["owner@example.com"]
+ assert "test email" in message.subject.lower()
+ assert "outgoing mail configuration" in message.body
+ assert "Test email sent to owner@example.com." in result.output
+
+
+def test_mail_cli_overrides_configured_recipient(app: Flask) -> None:
+ app.config.update(
+ NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
+ )
+
+ with patch("webmentions_ssg.MAIL.send") as send:
+ result = app.test_cli_runner().invoke(
+ args=["mail", "test", "--to", "other@example.com"]
+ )
+
+ assert result.exit_code == 0, result.output
+ send.assert_called_once()
+ assert send.call_args.args[0].recipients == ["other@example.com"]
+
+
+def test_mail_cli_requires_recipient(app: Flask) -> None:
+ app.config.update(NOTIFICATION_EMAIL=None, MAIL_DEFAULT_SENDER="sender@example.com")
+
+ with patch("webmentions_ssg.MAIL.send") as send:
+ result = app.test_cli_runner().invoke(args=["mail", "test"])
+
+ assert result.exit_code != 0
+ assert "Set NOTIFICATION_EMAIL or pass --to" in result.output
+ send.assert_not_called()
+
+
+def test_mail_cli_requires_default_sender(app: Flask) -> None:
+ app.config.update(NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER=None)
+
+ with patch("webmentions_ssg.MAIL.send") as send:
+ result = app.test_cli_runner().invoke(args=["mail", "test"])
+
+ assert result.exit_code != 0
+ assert "MAIL_DEFAULT_SENDER is not configured" in result.output
+ send.assert_not_called()
+
+
+def test_mail_cli_reports_smtp_failure(app: Flask) -> None:
+ app.config.update(
+ NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com"
+ )
+
+ with patch(
+ "webmentions_ssg.MAIL.send", side_effect=ConnectionError("SMTP unavailable")
+ ) as send:
+ result = app.test_cli_runner().invoke(args=["mail", "test"])
+
+ assert result.exit_code != 0
+ assert "Could not send test email: SMTP unavailable" in result.output
+ send.assert_called_once()
+
+
+def _add_user(app: Flask, username: str, password: str = "original") -> None:
+ """Persist a user using the same model as the CLI."""
+ with app.app_context():
+ DATABASE.session.add(User(username=username, password=password))
+ DATABASE.session.commit()
+
+
+def _get_user(app: Flask, username: str) -> User | None:
+ with app.app_context():
+ return DATABASE.session.scalar(sa.select(User).where(User.username == username))
+
+
+def test_users_cli_help_lists_management_commands(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(args=["users", "--help"])
+
+ assert result.exit_code == 0, result.output
+ for command in ("create", "list", "set-password", "delete"):
+ assert command in result.output
+
+
+def test_users_cli_create_with_prompts_hashes_password(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(
+ args=["users", "create"],
+ input="alice\nmy new secret passphrase\nmy new secret passphrase\n",
+ )
+
+ assert result.exit_code == 0, result.output
+ assert "Created user 'alice'." in result.output
+ assert "my new secret passphrase" not in result.output
+ user = _get_user(app, "alice")
+ assert user is not None
+ assert user.check_password("my new secret passphrase")
+ assert user.password_hash != "my new secret passphrase"
+
+
+def test_users_cli_create_with_username_option(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(
+ args=["users", "create", "--username", "alice"],
+ input="my new secret passphrase\nmy new secret passphrase\n",
+ )
+
+ assert result.exit_code == 0, result.output
+ assert _get_user(app, "alice") is not None
+
+
+def test_users_cli_create_rejects_duplicate(app: Flask) -> None:
+ _add_user(app, "alice")
+
+ result = app.test_cli_runner().invoke(
+ args=[
+ "users",
+ "create",
+ "--username",
+ "alice",
+ "--password",
+ "a stronger duplicate password",
+ ]
+ )
+
+ assert result.exit_code != 0
+ assert "User 'alice' already exists." in result.output
+ user = _get_user(app, "alice")
+ assert user is not None
+ assert user.check_password("original")
+
+
+@pytest.mark.parametrize("username", ["", " "])
+def test_users_cli_create_rejects_invalid_username(app: Flask, username: str) -> None:
+ result = app.test_cli_runner().invoke(
+ args=[
+ "users",
+ "create",
+ "--username",
+ username,
+ "--password",
+ "a strong password for testing",
+ ]
+ )
+
+ assert result.exit_code != 0
+ assert "Username must not" in result.output
+
+
+def test_users_cli_create_rejects_empty_password(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(
+ args=["users", "create", "--username", "alice", "--password", ""]
+ )
+
+ assert result.exit_code != 0
+ assert "Password must not be empty." in result.output
+ assert _get_user(app, "alice") is None
+
+
+def test_users_cli_list_empty(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(args=["users", "list"])
+
+ assert result.exit_code == 0, result.output
+ assert "No users found." in result.output
+
+
+def test_users_cli_list_is_sorted_and_excludes_passwords(app: Flask) -> None:
+ _add_user(app, "zoe")
+ _add_user(app, "alice")
+
+ result = app.test_cli_runner().invoke(args=["users", "list"])
+
+ assert result.exit_code == 0, result.output
+ lines = result.output.splitlines()
+ assert lines[0] == "ID\tUSERNAME"
+ assert lines[1].endswith("\talice")
+ assert lines[2].endswith("\tzoe")
+ assert "original" not in result.output
+
+
+def test_users_cli_set_password_updates_hash(app: Flask) -> None:
+ _add_user(app, "alice")
+ old_hash = _get_user(app, "alice").password_hash # type: ignore[union-attr]
+
+ result = app.test_cli_runner().invoke(
+ args=["users", "set-password", "alice"],
+ input="a long replacement passphrase\na long replacement passphrase\n",
+ )
+
+ assert result.exit_code == 0, result.output
+ assert "Updated password for user 'alice'." in result.output
+ assert "a long replacement passphrase" not in result.output
+ user = _get_user(app, "alice")
+ assert user is not None
+ assert user.password_hash != old_hash
+ assert not user.check_password("original")
+ assert user.check_password("a long replacement passphrase")
+
+
+def test_users_cli_set_password_missing_user(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(
+ args=[
+ "users",
+ "set-password",
+ "missing",
+ "--password",
+ "a long replacement passphrase",
+ ]
+ )
+
+ assert result.exit_code != 0
+ assert "User 'missing' does not exist." in result.output
+
+
+def test_users_cli_set_password_rejects_empty_password(app: Flask) -> None:
+ _add_user(app, "alice")
+
+ result = app.test_cli_runner().invoke(
+ args=["users", "set-password", "alice", "--password", ""]
+ )
+
+ assert result.exit_code != 0
+ assert "Password must not be empty." in result.output
+ user = _get_user(app, "alice")
+ assert user is not None
+ assert user.check_password("original")
+
+
+def test_users_cli_delete_with_confirmation(app: Flask) -> None:
+ _add_user(app, "alice")
+
+ result = app.test_cli_runner().invoke(
+ args=["users", "delete", "alice"], input="y\n"
+ )
+
+ assert result.exit_code == 0, result.output
+ assert "Delete user 'alice'?" in result.output
+ assert "Deleted user 'alice'." in result.output
+ assert _get_user(app, "alice") is None
+
+
+def test_users_cli_delete_confirmation_rejected(app: Flask) -> None:
+ _add_user(app, "alice")
+
+ result = app.test_cli_runner().invoke(
+ args=["users", "delete", "alice"], input="n\n"
+ )
+
+ assert result.exit_code != 0
+ assert "Aborted" in result.output
+ assert _get_user(app, "alice") is not None
+
+
+def test_users_cli_delete_yes_only_deletes_target(app: Flask) -> None:
+ _add_user(app, "alice")
+ _add_user(app, "bob")
+
+ result = app.test_cli_runner().invoke(args=["users", "delete", "alice", "--yes"])
+
+ assert result.exit_code == 0, result.output
+ assert "Deleted user 'alice'." in result.output
+ assert _get_user(app, "alice") is None
+ assert _get_user(app, "bob") is not None
+
+
+def test_users_cli_delete_missing_user(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(args=["users", "delete", "missing", "-y"])
+
+ assert result.exit_code != 0
+ assert "User 'missing' does not exist." in result.output
+
+
+def test_users_cli_delete_missing_blank_username(app: Flask) -> None:
+ result = app.test_cli_runner().invoke(args=["users", "delete", " ", "--yes"])
+
+ assert result.exit_code != 0
+ assert "User ' ' does not exist." in result.output
+
+
+def test_users_cli_can_manage_legacy_whitespace_username(app: Flask) -> None:
+ _add_user(app, " alice ")
+
+ reset = app.test_cli_runner().invoke(
+ args=[
+ "users",
+ "set-password",
+ " alice ",
+ "--password",
+ "a long replacement passphrase",
+ ]
+ )
+
+ assert reset.exit_code == 0, reset.output
+ user = _get_user(app, " alice ")
+ assert user is not None
+ assert user.check_password("a long replacement passphrase")
+
+ deleted = app.test_cli_runner().invoke(args=["users", "delete", " alice ", "--yes"])
+ assert deleted.exit_code == 0, deleted.output
+ assert _get_user(app, " alice ") is None
+
+
+def test_users_cli_create_rolls_back_on_duplicate_race(app: Flask) -> None:
+ database_error = IntegrityError("INSERT", {}, Exception("duplicate"))
+
+ with app.app_context():
+ with (
+ patch.object(DATABASE.session, "commit", side_effect=database_error),
+ patch.object(DATABASE.session, "rollback") as rollback,
+ ):
+ result = app.test_cli_runner().invoke(
+ args=[
+ "users",
+ "create",
+ "--username",
+ "alice",
+ "--password",
+ "a long secret for testing",
+ ]
+ )
+ rollback.assert_called_once()
+
+ assert result.exit_code != 0
+ assert "User 'alice' already exists." in result.output
+
+
+@pytest.mark.parametrize(
+ ("password", "error"),
+ [
+ ("short", "at least 8 characters"),
+ ("good passphrase with\tab", "control characters"),
+ ],
+)
+@pytest.mark.parametrize("operation", ["create", "set-password"])
+def test_users_cli_enforces_password_policy(
+ app: Flask, password: str, error: str, operation: str
+) -> None:
+ _add_user(app, "existing")
+ if operation == "create":
+ command = ["users", "create", "--username", "new"]
+ else:
+ command = ["users", "set-password", "existing"]
+
+ result = app.test_cli_runner().invoke(args=[*command, "--password", password])
+ assert result.exit_code != 0
+ assert error in result.output
+ assert _get_user(app, "new") is None
+ existing = _get_user(app, "existing")
+ assert existing is not None and existing.check_password("original")
+
+
+def test_users_cli_accepts_long_unicode_passphrase(app: Flask) -> None:
+ passphrase = "🦊 several words of my passphrase 🪁"
+ result = app.test_cli_runner().invoke(
+ args=["users", "create", "--username", "alice", "--password", passphrase]
+ )
+ assert result.exit_code == 0, result.output
+ user = _get_user(app, "alice")
+ assert user is not None and user.check_password(passphrase)
+
+
+@pytest.mark.parametrize("passphrase", ["12345678", " " * 8, "🦊" + "a" * 1025])
+def test_users_cli_accepts_minimum_whitespace_and_unbounded_passwords(
+ app: Flask, passphrase: str
+) -> None:
+ result = app.test_cli_runner().invoke(
+ args=["users", "create", "--username", "alice", "--password", passphrase]
+ )
+ assert result.exit_code == 0, result.output
+ user = _get_user(app, "alice")
+ assert user is not None and user.check_password(passphrase)
diff --git a/tests/test_config.py b/tests/test_config.py
index cc58f73..7bec8b6 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -6,10 +6,9 @@
from pathlib import Path
+import pytest
from flask import request
from flask_limiter.util import get_remote_address
-
-import pytest
from wtforms import Field, ValidationError
from wtforms.form import BaseForm
@@ -20,7 +19,7 @@ from webmentions_ssg.config import (
ProductionConfig,
TestingConfig,
)
-from webmentions_ssg.forms.validators import AllowedHostname
+from webmentions_ssg.forms import AllowedHostname
@pytest.mark.parametrize(
@@ -100,6 +99,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None:
for config in (DefaultConfig, DevelopmentConfig, ProductionConfig, TestingConfig):
assert config.MAX_REDIRECTS == 20
assert config.MAX_SOURCE_BYTES == 1_000_000
+ assert config.MAX_TARGET_BYTES == 1_000_000
assert config.REQUEST_TIMEOUT == 5.0
assert config.REQUEST_TOTAL_TIMEOUT == 30.0
@@ -160,7 +160,7 @@ def test_ignored_hostnames_are_normalized_at_startup(tmp_path: Path) -> None:
IGNORED_HOSTNAMES = [" Example.COM. ", "*.EXAMPLE.com.", "example.com"]
app = create_app(MixedCaseConfig)
- assert app.config["IGNORED_HOSTNAMES"] == ("*.example.com", "example.com")
+ assert app.config["IGNORED_HOSTNAMES"] == {"*.example.com", "example.com"}
@pytest.mark.parametrize(
@@ -360,7 +360,7 @@ def test_resource_limits_have_safe_defaults() -> None:
@pytest.mark.parametrize(
"key", ["MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS"]
)
-@pytest.mark.parametrize("value", [None, "100", 0, -1, True, 1.5])
+@pytest.mark.parametrize("value", [None, "100", 0, -1, 1.5])
def test_invalid_resource_limits_fail_at_startup(
tmp_path: Path, key: str, value: object
) -> None:
@@ -425,7 +425,7 @@ def test_proxy_fix_trusts_only_configured_proxy_hops(tmp_path: Path) -> None:
assert response.json == {"ip": "203.0.113.4", "scheme": "http", "host": "localhost"}
-@pytest.mark.parametrize("hops", [-1, True, False, None, "1", 1.5])
+@pytest.mark.parametrize("hops", [-1, None, "1", 1.5])
def test_invalid_proxy_fix_hop_counts_fail_at_startup(
tmp_path: Path, hops: object
) -> None:
diff --git a/tests/forms/test_init.py b/tests/test_forms.py
index 42e6a2c..03789da 100644
--- a/tests/forms/test_init.py
+++ b/tests/test_forms.py
@@ -5,13 +5,31 @@
import pytest
from flask import Flask
from werkzeug.datastructures import MultiDict
+from wtforms import Form, StringField, ValidationError
-from webmentions_ssg.forms import AdminActionForm, EndpointForm, LoginForm
+from webmentions_ssg.forms import (
+ AdminActionForm,
+ AllowedHostname,
+ EndpointForm,
+ LoginForm,
+ NotEqualTo,
+ PublicIPLiteral,
+ URLMaxBytes,
+)
VALID_SOURCE = "https://source.example/post"
VALID_TARGET = "https://dennisfink.me/blog/example/"
+class ComparisonForm(Form):
+ source = StringField("Source")
+ target = StringField("Target")
+
+
+class URLForm(Form):
+ url = StringField("URL")
+
+
@pytest.mark.parametrize(
("form_data", "invalid_field", "expected_error"),
[
@@ -113,7 +131,7 @@ def test_endpoint_does_not_do_dns_in_web_request(
raise AssertionError("No DNS lookup belongs in form validation")
monkeypatch.setattr(
- "webmentions_ssg.url_security.dns.resolver.resolve_name", unexpected_dns_lookup
+ "webmentions_ssg.net.dns.resolver.resolve_name", unexpected_dns_lookup
)
with app.test_request_context("/endpoint", method="POST"):
for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"):
@@ -203,3 +221,96 @@ def test_endpoint_accepts_encoded_path_space(app: Flask) -> None:
meta={"csrf": False},
)
assert form.validate()
+
+
+def test_not_equal_to_accepts_different_values() -> None:
+ form = ComparisonForm(MultiDict({"source": "source", "target": "target"}))
+ NotEqualTo("target")(form, form.source)
+
+
+def test_not_equal_to_rejects_equal_values() -> None:
+ form = ComparisonForm(MultiDict({"source": "same", "target": "same"}))
+ with pytest.raises(ValidationError, match="Field must not be equal to target"):
+ NotEqualTo("target")(form, form.source)
+
+
+def test_not_equal_to_uses_custom_message() -> None:
+ form = ComparisonForm(MultiDict({"source": "same", "target": "same"}))
+ with pytest.raises(ValidationError, match="Must differ from Target"):
+ NotEqualTo("target", "Must differ from %(other_label)s")(form, form.source)
+
+
+def test_not_equal_to_rejects_unknown_field() -> None:
+ form = ComparisonForm(MultiDict({"source": "source", "target": "target"}))
+ with pytest.raises(ValidationError, match="Invalid field name 'missing'"):
+ NotEqualTo("missing")(form, form.source)
+
+
+def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None:
+ app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
+ form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"}))
+ with app.app_context():
+ AllowedHostname()(form, form.url)
+
+
+def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None:
+ app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
+ form = URLForm(MultiDict({"url": "https://example.com/post"}))
+ with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
+ AllowedHostname()(form, form.url)
+
+
+def test_allowed_hostname_uses_custom_message(app: Flask) -> None:
+ app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"]
+ form = URLForm(MultiDict({"url": "https://example.com/post"}))
+ with (
+ app.app_context(),
+ pytest.raises(ValidationError, match="Hostname is not allowed"),
+ ):
+ AllowedHostname("Hostname is not allowed")(form, form.url)
+
+
+@pytest.mark.parametrize(
+ "address",
+ ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"],
+)
+def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None:
+ form = URLForm(MultiDict({"url": f"https://[{address}]/"}))
+ with pytest.raises(ValidationError, match="URL must use a public address"):
+ PublicIPLiteral()(form, form.url)
+
+
+def test_public_ip_literal_accepts_public_ipv6_address() -> None:
+ form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"}))
+ PublicIPLiteral()(form, form.url)
+
+
+def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "http://[::1"}))
+ with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
+ AllowedHostname()(form, form.url)
+
+
+def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None:
+ validator = URLMaxBytes()
+ form = URLForm(MultiDict({"url": "é" * 3}))
+ with app.app_context():
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 6
+ validator(form, form.url)
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 5
+ with pytest.raises(ValidationError, match="5 bytes"):
+ validator(form, form.url)
+
+
+def test_url_max_bytes_custom_message(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "too-long"}))
+ with app.app_context():
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 2
+ with pytest.raises(ValidationError, match="URL too long"):
+ URLMaxBytes(message="URL too long")(form, form.url)
+
+
+def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "\ud800"}))
+ with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"):
+ URLMaxBytes()(form, form.url)
diff --git a/tests/test_init.py b/tests/test_init.py
index d690032..0a37786 100644
--- a/tests/test_init.py
+++ b/tests/test_init.py
@@ -6,8 +6,8 @@
from flask import Flask
-from webmentions_ssg import VERSION, HUEY
-from webmentions_ssg.url_security import is_http_url
+from webmentions_ssg import HUEY, VERSION
+from webmentions_ssg.net import is_http_url
def test_create_app_registers_blueprint_and_extensions(app: Flask) -> None:
diff --git a/tests/test_models.py b/tests/test_models.py
index 392b246..26439b9 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -7,12 +7,18 @@ from collections.abc import Callable
from datetime import UTC, datetime
import pytest
+import sqlalchemy as sa
+from flask import Flask
+from sqlalchemy.exc import IntegrityError
+from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import (
ReceivedWebmention,
SentWebmention,
Source,
User,
+ received_url_digest,
+ sent_target_digest,
uuid7_to_datetime,
)
@@ -162,3 +168,144 @@ def test_user_session_id_changes_when_password_hash_changes() -> None:
user.password = "a strong passphrase"
assert user.password_hash != original_hash
assert user.get_id() != original_id
+
+
+def test_received_status_remains_text_with_enum_constants() -> None:
+ from sqlalchemy import Text
+
+ from webmentions_ssg.models import ReceivedStatus
+
+ assert isinstance(ReceivedWebmention.__table__.c.status.type, Text)
+ assert ReceivedStatus.VERIFIED == "verified"
+ assert ReceivedStatus.DELETED == "deleted"
+
+
+def test_received_verification_reset_is_reusable_for_atomic_updates() -> None:
+ from webmentions_ssg.models import ReceivedStatus
+
+ mention = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source="https://source.example/post",
+ target="https://dennisfink.me/post",
+ status=ReceivedStatus.FAILED,
+ failure_reason="Old error",
+ recovery_attempts=3,
+ )
+ assert mention.reset_for_verification() is mention
+ assert mention.status == ReceivedStatus.RECEIVED
+ assert mention.failure_reason is None
+ assert mention.recovery_attempts == 0
+ assert mention.updated_at is not None
+
+
+def test_reopen_preserves_delivery_history_unless_explicit_resend() -> None:
+ from webmentions_ssg.models import SentWebmentionStatus
+
+ sent_at = datetime.now(UTC)
+ mention = SentWebmention(
+ target="https://example.com/post",
+ active=True,
+ desired_revision=2,
+ processed_revision=2,
+ sent_revision=2,
+ status=SentWebmentionStatus.SENT,
+ attempt_count=3,
+ next_retry_at=sent_at,
+ last_sent_at=sent_at,
+ )
+ assert mention.reopen(3, False) is mention
+ assert mention.desired_revision == 3
+ assert mention.active is False
+ assert mention.attempt_count == 0
+ assert mention.next_retry_at is None
+ assert mention.status == SentWebmentionStatus.SENT
+ assert mention.sent_revision == 2
+
+ assert mention.reopen(3, True, resend=True) is mention
+ assert mention.processed_revision is None
+ assert mention.status is None
+ assert mention.sent_revision == 2
+ assert mention.last_sent_at == sent_at
+
+
+def test_received_pair_fingerprint_is_unambiguous() -> None:
+ assert received_url_digest("a", "bc") != received_url_digest("ab", "c")
+ assert received_url_digest("a", "b") != received_url_digest("b", "a")
+
+
+def test_models_hash_url_fields_when_created_or_changed() -> None:
+ received = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source="https://source.example/a",
+ target="https://site.example/b",
+ )
+ assert received.url_digest == received_url_digest(received.source, received.target)
+ received.source = "https://source.example/c"
+ assert received.url_digest == received_url_digest(received.source, received.target)
+ received.target = "https://site.example/d"
+ assert received.url_digest == received_url_digest(received.source, received.target)
+
+ sent = SentWebmention(target="https://site.example/a", desired_revision=1)
+ assert sent.target_digest == sent_target_digest(sent.target)
+ sent.target = "https://site.example/b"
+ assert sent.target_digest == sent_target_digest(sent.target)
+
+
+def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None:
+ long_source = "https://source.example/" + "a" * 4000
+ long_target = "https://dennisfink.me/" + "b" * 4000
+ with app.app_context():
+ db.session.add(
+ ReceivedWebmention(
+ uuid=uuid.uuid7(), source=long_source, target=long_target
+ )
+ )
+ db.session.commit()
+ match = db.session.scalar(
+ sa.select(ReceivedWebmention).where(
+ ReceivedWebmention.url_digest
+ == received_url_digest(long_source, long_target)
+ )
+ )
+ assert match is not None
+ assert match.source == long_source
+ assert match.target == long_target
+ db.session.add(
+ ReceivedWebmention(
+ uuid=uuid.uuid7(), source=long_source, target=long_target
+ )
+ )
+ with pytest.raises(IntegrityError):
+ db.session.commit()
+ db.session.rollback()
+
+
+def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text(
+ app: Flask,
+) -> None:
+ now = datetime.now(UTC)
+ long_target = "https://destination.example/" + "a" * 4000
+ with app.app_context():
+ source = Source(
+ path="long/index.html",
+ url="https://example.com/long",
+ content_hash="0" * 32,
+ revision=1,
+ last_seen_at=now,
+ revised_at=now,
+ )
+ source.sent_webmentions.append(
+ SentWebmention(target=long_target, desired_revision=1)
+ )
+ db.session.add(source)
+ db.session.commit()
+ assert source.sent_webmentions[0].target_digest == sent_target_digest(
+ long_target
+ )
+
+ source.sent_webmentions.append(
+ SentWebmention(target=long_target, desired_revision=1)
+ )
+ with pytest.raises(IntegrityError):
+ db.session.commit()
+ db.session.rollback()
diff --git a/tests/test_public_transport.py b/tests/test_net.py
index df7a1c3..928a0d5 100644
--- a/tests/test_public_transport.py
+++ b/tests/test_net.py
@@ -4,20 +4,26 @@
"""Security regression tests for DNS-pinned network connections."""
+import ipaddress
import socket
-from unittest.mock import Mock
+from unittest.mock import Mock, PropertyMock, patch
import httpcore
import httpx
import pytest
+from flask import Flask
-from webmentions_ssg.public_transport import (
+from webmentions_ssg.net import (
+ AddressResolutionError,
DeadlineStream,
NetworkDeadline,
PublicHTTPTransport,
PublicNetworkBackend,
+ UnsafeAddressError,
+ is_http_url,
+ is_public_address,
+ public_addresses,
)
-from webmentions_ssg.url_security import UnsafeAddressError
@pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"])
@@ -34,7 +40,7 @@ def test_backend_rejects_private_ip_without_socket(
def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None:
"""The address is resolved exactly once and the socket sees only that IP."""
dns = Mock(return_value=("93.184.216.34",))
- monkeypatch.setattr("webmentions_ssg.public_transport.public_addresses", dns)
+ monkeypatch.setattr("webmentions_ssg.net.public_addresses", dns)
socket_instance = Mock()
create_socket = Mock(return_value=socket_instance)
monkeypatch.setattr(socket, "socket", create_socket)
@@ -110,9 +116,7 @@ def test_deadline_clips_each_read_and_stops_trickle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = [100.0]
- monkeypatch.setattr(
- "webmentions_ssg.public_transport.time.monotonic", lambda: clock[0]
- )
+ monkeypatch.setattr("webmentions_ssg.net.time.monotonic", lambda: clock[0])
fake_stream = Mock()
fake_stream.read.return_value = b"x"
stream = DeadlineStream(fake_stream, NetworkDeadline(5.0))
@@ -154,3 +158,109 @@ def test_redirects_go_through_same_destination_guard(
client.get("http://source.example/post")
assert checked_hosts == ["source.example", "internal.example"]
+
+
+@pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ pytest.param("http://example.com/", True, id="http"),
+ pytest.param("https://example.com/path", True, id="https"),
+ pytest.param("HTTP://EXAMPLE.COM/", True, id="scheme-case-insensitive"),
+ pytest.param("https://example.com:8443/path", True, id="port"),
+ pytest.param("http://[2001:db8::1]/", True, id="ipv6"),
+ pytest.param("ftp://example.com/", False, id="ftp"),
+ pytest.param("mailto:example@example.com", False, id="mailto"),
+ pytest.param("/relative/url", False, id="relative"),
+ pytest.param("//example.com/path", False, id="scheme-relative"),
+ pytest.param("https:///missing-host", False, id="missing-host"),
+ pytest.param("", False, id="empty"),
+ pytest.param("http://[::1", False, id="malformed"),
+ ],
+)
+def test_is_http_url(url: str, expected: bool) -> None:
+ assert is_http_url(url) is expected
+
+
+def test_ip_literal_does_not_use_dns() -> None:
+ with patch("webmentions_ssg.net.dns.resolver.resolve_name") as dns_resolve:
+ with pytest.raises(UnsafeAddressError):
+ public_addresses("169.254.169.254")
+ with pytest.raises(UnsafeAddressError):
+ public_addresses("::1")
+ assert public_addresses("8.8.8.8") == ("8.8.8.8",)
+ dns_resolve.assert_not_called()
+
+
+@pytest.mark.parametrize(
+ "address",
+ [
+ pytest.param("64:ff9b::a9fe:a9fe", id="nat64-metadata"),
+ pytest.param("64:ff9b::808:808", id="nat64-public-ipv4"),
+ pytest.param("::10.0.0.1", id="ipv4-compatible-private"),
+ pytest.param("::8.8.8.8", id="ipv4-compatible-public"),
+ pytest.param("2002:a9fe:a9fe::1", id="6to4-metadata"),
+ pytest.param("2002:0a00:0001::1", id="6to4-private"),
+ pytest.param("::ffff:10.0.0.1", id="mapped-private"),
+ pytest.param("::ffff:169.254.169.254", id="mapped-metadata"),
+ ],
+)
+def test_public_addresses_rejects_ipv6_transition_literals(address: str) -> None:
+ with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve:
+ with pytest.raises(UnsafeAddressError, match="Non-public"):
+ public_addresses(address)
+ resolve.assert_not_called()
+
+
+@pytest.mark.parametrize(
+ "address", ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1"]
+)
+def test_public_addresses_rejects_transition_addresses_in_dns(address: str) -> None:
+ with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve:
+ resolve.return_value.addresses.return_value = ["93.184.216.34", address]
+ with pytest.raises(UnsafeAddressError, match="Non-public"):
+ public_addresses("transition.example")
+
+
+@pytest.mark.parametrize("address", ["2002:0a00:0001::1", "::ffff:10.0.0.1"])
+def test_private_embedded_ipv4_rejected_even_if_ipv6_is_global(address: str) -> None:
+ # Simulate a Python version treating the outer transition address as global.
+ with patch.object(
+ ipaddress.IPv6Address, "is_global", new_callable=PropertyMock
+ ) as global_:
+ global_.return_value = True
+ assert not is_public_address(ipaddress.ip_address(address))
+
+
+def test_public_addresses_rejects_mixed_dns_results() -> None:
+ with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve:
+ resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"]
+ with pytest.raises(UnsafeAddressError, match="Non-public"):
+ public_addresses("rebinding.example", lifetime=1.5)
+ resolve.assert_called_once_with("rebinding.example", lifetime=1.5)
+
+
+def test_public_addresses_rejects_empty_dns_result() -> None:
+ with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve:
+ resolve.return_value.addresses.return_value = []
+ with pytest.raises(AddressResolutionError):
+ public_addresses("empty.example")
+
+
+@pytest.mark.parametrize(
+ ("status", "expected"),
+ [
+ pytest.param(200, False, id="success"),
+ pytest.param(400, False, id="client-error"),
+ pytest.param(408, True, id="request-timeout"),
+ pytest.param(425, True, id="too-early"),
+ pytest.param(429, True, id="too-many-requests"),
+ pytest.param(500, True, id="server-error-lower-bound"),
+ pytest.param(599, True, id="server-error-upper-bound"),
+ pytest.param(600, False, id="outside-server-error-range"),
+ ],
+)
+def test_temporary_http_status(app: Flask, status: int, expected: bool) -> None:
+ _ = app
+ from webmentions_ssg.tasks import discovery
+
+ assert discovery.is_temporary_status(status) is expected
diff --git a/tests/test_optional_dependencies.py b/tests/test_packaging.py
index 40c53c5..34c0d72 100644
--- a/tests/test_optional_dependencies.py
+++ b/tests/test_packaging.py
@@ -9,7 +9,6 @@ from pathlib import Path
import pytest
-
BACKENDS = {
"redis": ["flask-limiter[redis]>=4.1.1", "huey[redis]>=3.3.0"],
"valkey": ["flask-limiter[valkey]>=4.1.1", "huey[redis]>=3.3.0"],
diff --git a/tests/test_url_digest.py b/tests/test_url_digest.py
deleted file mode 100644
index 0c04364..0000000
--- a/tests/test_url_digest.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Test the database-independent Webmention URL fingerprints."""
-
-import uuid
-from datetime import UTC, datetime
-
-import pytest
-import sqlalchemy as sa
-from flask import Flask
-from sqlalchemy.exc import IntegrityError
-
-from webmentions_ssg import DATABASE as db
-from webmentions_ssg.models import ReceivedWebmention, SentWebmention, Source
-from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
-
-
-def test_received_pair_fingerprint_is_unambiguous() -> None:
- assert received_url_digest("a", "bc") != received_url_digest("ab", "c")
- assert received_url_digest("a", "b") != received_url_digest("b", "a")
-
-
-def test_models_hash_url_fields_when_created_or_changed() -> None:
- received = ReceivedWebmention(
- uuid=uuid.uuid7(),
- source="https://source.example/a",
- target="https://site.example/b",
- )
- assert received.url_digest == received_url_digest(received.source, received.target)
- received.source = "https://source.example/c"
- assert received.url_digest == received_url_digest(received.source, received.target)
- received.target = "https://site.example/d"
- assert received.url_digest == received_url_digest(received.source, received.target)
-
- sent = SentWebmention(target="https://site.example/a", desired_revision=1)
- assert sent.target_digest == sent_target_digest(sent.target)
- sent.target = "https://site.example/b"
- assert sent.target_digest == sent_target_digest(sent.target)
-
-
-def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None:
- long_source = "https://source.example/" + "a" * 4000
- long_target = "https://dennisfink.me/" + "b" * 4000
- with app.app_context():
- db.session.add(
- ReceivedWebmention(
- uuid=uuid.uuid7(), source=long_source, target=long_target
- )
- )
- db.session.commit()
- match = db.session.scalar(
- sa.select(ReceivedWebmention).where(
- ReceivedWebmention.url_digest
- == received_url_digest(long_source, long_target)
- )
- )
- assert match is not None
- assert match.source == long_source
- assert match.target == long_target
- db.session.add(
- ReceivedWebmention(
- uuid=uuid.uuid7(), source=long_source, target=long_target
- )
- )
- with pytest.raises(IntegrityError):
- db.session.commit()
- db.session.rollback()
-
-
-def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text(
- app: Flask,
-) -> None:
- now = datetime.now(UTC)
- long_target = "https://destination.example/" + "a" * 4000
- with app.app_context():
- source = Source(
- path="long/index.html",
- url="https://example.com/long",
- content_hash="0" * 32,
- revision=1,
- last_seen_at=now,
- revised_at=now,
- )
- source.sent_webmentions.append(
- SentWebmention(target=long_target, desired_revision=1)
- )
- db.session.add(source)
- db.session.commit()
- assert source.sent_webmentions[0].target_digest == sent_target_digest(
- long_target
- )
-
- source.sent_webmentions.append(
- SentWebmention(target=long_target, desired_revision=1)
- )
- with pytest.raises(IntegrityError):
- db.session.commit()
- db.session.rollback()
diff --git a/tests/test_url_security.py b/tests/test_url_security.py
deleted file mode 100644
index 95c2bd9..0000000
--- a/tests/test_url_security.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-import ipaddress
-from unittest.mock import Mock, PropertyMock, patch
-
-import dns.resolver
-import pytest
-
-from webmentions_ssg.url_security import (
- AddressResolutionError,
- UnsafeAddressError,
- is_http_url,
- is_public_address,
- is_public_url,
- is_safe_next_url,
- public_addresses,
-)
-
-
-@pytest.mark.parametrize(
- ("url", "expected"),
- [
- pytest.param("http://example.com/", True, id="http"),
- pytest.param("https://example.com/path", True, id="https"),
- pytest.param("HTTP://EXAMPLE.COM/", True, id="scheme-case-insensitive"),
- pytest.param("https://example.com:8443/path", True, id="port"),
- pytest.param("http://[2001:db8::1]/", True, id="ipv6"),
- pytest.param("ftp://example.com/", False, id="ftp"),
- pytest.param("mailto:example@example.com", False, id="mailto"),
- pytest.param("/relative/url", False, id="relative"),
- pytest.param("//example.com/path", False, id="scheme-relative"),
- pytest.param("https:///missing-host", False, id="missing-host"),
- pytest.param("", False, id="empty"),
- pytest.param("http://[::1", False, id="malformed"),
- ],
-)
-def test_is_http_url(url: str, expected: bool) -> None:
- assert is_http_url(url) is expected
-
-
-@pytest.mark.parametrize(
- ("next_url", "expected"),
- [
- pytest.param("/", True, id="root"),
- pytest.param("/received", True, id="local-path"),
- pytest.param("/received?page=2", True, id="query-string"),
- pytest.param("/received#details", True, id="fragment"),
- pytest.param(
- "/sent?next=https://evil.example", True, id="external-url-in-query"
- ),
- pytest.param("/foo//bar", True, id="double-slash-within-path"),
- pytest.param("", False, id="empty"),
- pytest.param("received", False, id="relative-path"),
- pytest.param(" /received", False, id="leading-space"),
- pytest.param("https://evil.example/phishing", False, id="absolute-https"),
- pytest.param("http://evil.example", False, id="absolute-http"),
- pytest.param("javascript:alert(1)", False, id="non-http-scheme"),
- pytest.param("//evil.example/phishing", False, id="scheme-relative"),
- pytest.param("///evil.example/phishing", False, id="triple-slash"),
- pytest.param("////evil.example", False, id="four-slashes"),
- pytest.param("//", False, id="two-slashes"),
- pytest.param(r"/\evil.example", False, id="slash-backslash-host"),
- pytest.param(r"/received\evil.example", False, id="backslash-in-path"),
- pytest.param(r"\/evil.example", False, id="backslash-slash-host"),
- ],
-)
-def test_is_safe_next_url(next_url: str, expected: bool) -> None:
- assert is_safe_next_url(next_url) is expected
-
-
-@patch("webmentions_ssg.url_security.dns.resolver.resolve_name")
-@pytest.mark.parametrize(
- ("url", "resolved_addresses"),
- [
- ("http://127.0.0.1/", ["127.0.0.1"]),
- ("http://127.0.0.1:8080/test", ["127.0.0.1"]),
- ("http://[::1]/", ["::1"]),
- ("http://10.0.0.1/", ["10.0.0.1"]),
- ("http://172.16.0.1/", ["172.16.0.1"]),
- ("http://192.168.1.1/", ["192.168.1.1"]),
- ("http://169.254.169.254/", ["169.254.169.254"]),
- ("http://localhost/", ["127.0.0.1", "::1"]),
- ("http://internal.example/", ["192.168.1.10"]),
- ("https://example.com/", ["93.184.216.34", "192.168.1.10"]),
- ],
-)
-def test_is_public_url_returns_false_for_non_public_addresses(
- resolve_name: Mock, url: str, resolved_addresses: list[str]
-) -> None:
- resolve_name.return_value.addresses.return_value = resolved_addresses
- assert not is_public_url(url)
-
-
-@patch("webmentions_ssg.url_security.dns.resolver.resolve_name")
-@pytest.mark.parametrize(
- ("url", "resolved_addresses"),
- [
- ("https://example.com/", ["93.184.216.34"]),
- (
- "https://example.com/",
- ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"],
- ),
- ],
-)
-def test_is_public_url_returns_true_for_public_addresses(
- resolve_name: Mock, url: str, resolved_addresses: list[str]
-) -> None:
- resolve_name.return_value.addresses.return_value = resolved_addresses
- assert is_public_url(url)
-
-
-@patch(
- "webmentions_ssg.url_security.dns.resolver.resolve_name",
- side_effect=dns.resolver.NXDOMAIN(),
-)
-def test_is_public_url_raises_for_resolution_failure(resolve_name: Mock) -> None:
- with pytest.raises(AddressResolutionError, match="Could not resolve hostname"):
- is_public_url("https://nonexistent.example/")
- resolve_name.assert_called_once_with("nonexistent.example", lifetime=None)
-
-
-@patch("webmentions_ssg.url_security.dns.resolver.resolve_name")
-def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> None:
- with pytest.raises(ValueError, match="No hostname was specified"):
- is_public_url("/relative/url")
- resolve_name.assert_not_called()
-
-
-def test_ip_literal_does_not_use_dns() -> None:
- with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as dns_resolve:
- with pytest.raises(UnsafeAddressError):
- public_addresses("169.254.169.254")
- with pytest.raises(UnsafeAddressError):
- public_addresses("::1")
- assert public_addresses("8.8.8.8") == ("8.8.8.8",)
- dns_resolve.assert_not_called()
-
-
-@pytest.mark.parametrize(
- "address",
- [
- pytest.param("64:ff9b::a9fe:a9fe", id="nat64-metadata"),
- pytest.param("64:ff9b::808:808", id="nat64-public-ipv4"),
- pytest.param("::10.0.0.1", id="ipv4-compatible-private"),
- pytest.param("::8.8.8.8", id="ipv4-compatible-public"),
- pytest.param("2002:a9fe:a9fe::1", id="6to4-metadata"),
- pytest.param("2002:0a00:0001::1", id="6to4-private"),
- pytest.param("::ffff:10.0.0.1", id="mapped-private"),
- pytest.param("::ffff:169.254.169.254", id="mapped-metadata"),
- ],
-)
-def test_public_addresses_rejects_ipv6_transition_literals(address: str) -> None:
- with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
- with pytest.raises(UnsafeAddressError, match="Non-public"):
- public_addresses(address)
- resolve.assert_not_called()
-
-
-@pytest.mark.parametrize(
- "address", ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1"]
-)
-def test_public_addresses_rejects_transition_addresses_in_dns(address: str) -> None:
- with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
- resolve.return_value.addresses.return_value = ["93.184.216.34", address]
- with pytest.raises(UnsafeAddressError, match="Non-public"):
- public_addresses("transition.example")
-
-
-@pytest.mark.parametrize("address", ["2002:0a00:0001::1", "::ffff:10.0.0.1"])
-def test_private_embedded_ipv4_rejected_even_if_ipv6_is_global(address: str) -> None:
- # Simulate a Python version treating the outer transition address as global.
- with patch.object(
- ipaddress.IPv6Address, "is_global", new_callable=PropertyMock
- ) as global_:
- global_.return_value = True
- assert not is_public_address(ipaddress.ip_address(address))
-
-
-def test_public_addresses_rejects_mixed_dns_results() -> None:
- with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
- resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"]
- with pytest.raises(UnsafeAddressError, match="Non-public"):
- public_addresses("rebinding.example", lifetime=1.5)
- resolve.assert_called_once_with("rebinding.example", lifetime=1.5)
-
-
-def test_public_addresses_rejects_empty_dns_result() -> None:
- with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
- resolve.return_value.addresses.return_value = []
- with pytest.raises(AddressResolutionError):
- public_addresses("empty.example")
diff --git a/tests/test_views.py b/tests/test_views.py
deleted file mode 100644
index 129bdbf..0000000
--- a/tests/test_views.py
+++ /dev/null
@@ -1,1347 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-import uuid
-from datetime import UTC, datetime, timedelta
-from types import ModuleType
-from unittest.mock import ANY, Mock
-
-import pytest
-import sqlalchemy as sa
-from flask import Flask
-from flask.testing import FlaskClient
-from sqlalchemy.exc import IntegrityError
-
-from webmentions_ssg import DATABASE as db
-from webmentions_ssg.models import (
- ReceivedWebmention,
- SentWebmention,
- SentWebmentionStatus,
- Source,
- User,
-)
-
-SOURCE_URL = "https://source.example/post"
-TARGET_URL = "https://dennisfink.me/blog/example/"
-
-
-@pytest.fixture
-def views_module(app: Flask) -> ModuleType:
- _ = app
- from webmentions_ssg import views
-
- return views
-
-
-def mock_endpoint_form(
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- *,
- valid: bool,
- source: str = SOURCE_URL,
- target: str = TARGET_URL,
- errors: dict[str, list[str]] | None = None,
-) -> Mock:
- form = Mock()
- form.validate_on_submit.return_value = valid
- form.source.data = source
- form.target.data = target
- form.errors = errors or {}
-
- monkeypatch.setattr(views_module.forms, "EndpointForm", Mock(return_value=form))
- return form
-
-
-def log_in(app: Flask, client: FlaskClient) -> None:
- with app.app_context():
- user = User(username="admin", password="test")
-
- db.session.add(user)
- db.session.commit()
-
- user_id = user.get_id()
-
- with client.session_transaction() as session:
- session["_user_id"] = str(user_id)
- session["_fresh"] = True
-
-
-def create_sent_source(app: Flask) -> uuid.UUID:
- now = datetime.now(UTC)
-
- with app.app_context():
- source = Source(
- path="blog/example/index.html",
- url="https://dennisfink.me/blog/example/",
- content_hash="0" * 32,
- revision=2,
- last_seen_at=now,
- revised_at=now,
- )
-
- source.sent_webmentions.extend(
- [
- SentWebmention(
- target="https://example.com/b",
- active=True,
- desired_revision=2,
- processed_revision=2,
- sent_revision=2,
- status=SentWebmentionStatus.SENT,
- endpoint="https://example.com/webmention",
- response_status=202,
- ),
- SentWebmention(
- target="https://example.com/a",
- active=True,
- desired_revision=2,
- processed_revision=2,
- sent_revision=1,
- status=SentWebmentionStatus.FAILED,
- failure_reason="Webmention endpoint returned HTTP 400",
- endpoint="https://example.com/webmention",
- response_status=400,
- ),
- ]
- )
-
- db.session.add(source)
- db.session.commit()
-
- return source.uuid
-
-
-def test_endpoint_only_accepts_post(client: FlaskClient) -> None:
- response = client.get("/endpoint")
- assert response.status_code == 405
-
-
-def test_endpoint_returns_form_errors(
- client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- errors = {"source": ["Invalid source"]}
- mock_endpoint_form(views_module, monkeypatch, valid=False, errors=errors)
-
- response = client.post("/endpoint")
-
- assert response.status_code == 400
- assert response.get_json() == errors
-
-
-def test_endpoint_creates_webmention(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- form = mock_endpoint_form(views_module, monkeypatch, valid=True)
- verify_webmention = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
-
- response = client.post("/endpoint")
-
- assert response.status_code == 201
- form.validate_on_submit.assert_called_once_with()
-
- with app.app_context():
- webmention = db.session.scalar(sa.select(ReceivedWebmention))
-
- assert webmention is not None
- assert webmention.source == SOURCE_URL
- assert webmention.target == TARGET_URL
- assert webmention.status == "received"
- assert webmention.failure_reason is None
-
- identifier = webmention.uuid
-
- assert response.headers["Location"].endswith(str(identifier))
- verify_webmention.assert_called_once_with(identifier)
-
-
-def test_endpoint_is_idempotent(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- mock_endpoint_form(views_module, monkeypatch, valid=True)
- verify_webmention = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
-
- first_response = client.post("/endpoint")
- second_response = client.post("/endpoint")
-
- assert first_response.status_code == 201
- assert second_response.status_code == 201
- assert first_response.headers["Location"] == second_response.headers["Location"]
-
- with app.app_context():
- webmentions = db.session.scalars(sa.select(ReceivedWebmention)).all()
-
- assert len(webmentions) == 1
-
- webmention = webmentions[0]
-
- assert webmention.source == SOURCE_URL
- assert webmention.target == TARGET_URL
- assert webmention.status == "received"
- assert webmention.failure_reason is None
-
- identifier = webmention.uuid
-
- verify_webmention.assert_called_once_with(identifier)
-
-
-def test_resending_resets_failure_state(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- mock_endpoint_form(views_module, monkeypatch, valid=True)
- verify_webmention = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
-
- with app.app_context():
- existing = ReceivedWebmention(
- uuid=uuid.uuid7(),
- source=SOURCE_URL,
- target=TARGET_URL,
- status="failed",
- failure_reason="Previous failure",
- updated_at=datetime.now(UTC) - timedelta(minutes=11),
- )
-
- db.session.add(existing)
- db.session.commit()
-
- identifier = existing.uuid
-
- response = client.post("/endpoint")
-
- assert response.status_code == 201
-
- with app.app_context():
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- assert webmention is not None
- assert webmention.status == "received"
- assert webmention.failure_reason is None
-
- verify_webmention.assert_called_once_with(identifier)
-
-
-@pytest.mark.parametrize("path", ["/sent", f"/sent/{uuid.uuid7()}"])
-def test_sent_views_require_login(client: FlaskClient, path: str) -> None:
- response = client.get(path)
-
- assert response.status_code == 302
- assert "/login" in response.headers["Location"]
-
-
-def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None:
- log_in(app, client)
- identifier = create_sent_source(app)
-
- response = client.get("/sent")
-
- assert response.status_code == 200
-
- html = response.get_data(as_text=True)
-
- assert "blog/example/index.html" in html
- assert "https://dennisfink.me/blog/example/" in html
- assert f"/sent/{identifier}" in html
-
-
-def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None:
- log_in(app, client)
- identifier = create_sent_source(app)
-
- response = client.get(f"/sent/{identifier}")
-
- assert response.status_code == 200
-
- html = response.get_data(as_text=True)
-
- assert "blog/example/index.html" in html
- assert "https://dennisfink.me/blog/example/" in html
-
- first_target = "https://example.com/a"
- second_target = "https://example.com/b"
-
- assert first_target in html
- assert second_target in html
- assert html.index(first_target) < html.index(second_target)
-
- assert "Webmention endpoint returned HTTP 400" in html
- assert "https://example.com/webmention" in html
-
-
-def test_sent_source_returns_404_for_unknown_source(
- app: Flask, client: FlaskClient
-) -> None:
- log_in(app, client)
- response = client.get(f"/sent/{uuid.uuid7()}")
- assert response.status_code == 404
-
-
-def mock_login_form(
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- *,
- valid: bool,
- username: str = "admin",
- password: str | None = "test",
-) -> Mock:
- form = Mock()
- form.validate_on_submit.return_value = valid
- form.username.data = username
- form.password.data = password
-
- monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form))
- return form
-
-
-def mock_admin_action_form(
- views_module: ModuleType, monkeypatch: pytest.MonkeyPatch, *, valid: bool
-) -> Mock:
- form = Mock()
- form.validate_on_submit.return_value = valid
-
- monkeypatch.setattr(views_module.forms, "AdminActionForm", Mock(return_value=form))
- return form
-
-
-def create_received_webmention(
- app: Flask,
- *,
- source: str = SOURCE_URL,
- target: str = TARGET_URL,
- status: str = "verified",
- failure_reason: str | None = None,
-) -> uuid.UUID:
- with app.app_context():
- webmention = ReceivedWebmention(
- uuid=uuid.uuid7(),
- source=source,
- target=target,
- status=status,
- failure_reason=failure_reason,
- )
-
- db.session.add(webmention)
- db.session.commit()
-
- return webmention.uuid
-
-
-def create_resendable_sent_webmention(app: Flask) -> tuple[uuid.UUID, uuid.UUID]:
- now = datetime.now(UTC)
-
- with app.app_context():
- source = Source(
- path="blog/resend/index.html",
- url="https://dennisfink.me/blog/resend/",
- content_hash="1" * 32,
- revision=3,
- last_seen_at=now,
- revised_at=now,
- )
- webmention = SentWebmention(
- target="https://example.com/resend",
- active=True,
- desired_revision=3,
- processed_revision=3,
- sent_revision=2,
- status=SentWebmentionStatus.FAILED,
- failure_reason="Previous failure",
- endpoint="https://example.com/webmention",
- response_status=500,
- status_url="https://example.com/webmention/status/123",
- )
- source.sent_webmentions.append(webmention)
-
- db.session.add(source)
- db.session.commit()
-
- return source.uuid, webmention.uuid
-
-
-def test_login_renders_form(client: FlaskClient) -> None:
- response = client.get("/login")
-
- assert response.status_code == 200
- assert "login" in response.get_data(as_text=True).lower()
-
-
-@pytest.mark.parametrize("password", ("wrong-password", None))
-def test_login_rejects_invalid_credentials(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- password: str | None,
-) -> None:
- with app.app_context():
- db.session.add(User(username="admin", password="test"))
- db.session.commit()
-
- mock_login_form(
- views_module, monkeypatch, valid=True, username="admin", password=password
- )
-
- response = client.post("/login")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/login")
-
- with client.session_transaction() as session:
- assert "_user_id" not in session
- assert ("danger", "Invalid username or password") in session["_flashes"]
-
-
-@pytest.mark.parametrize(
- "next_url",
- [
- pytest.param("/", id="root"),
- pytest.param("/received", id="local-path"),
- pytest.param("/received?page=2", id="query-string"),
- pytest.param("/received#details", id="fragment"),
- ],
-)
-def test_login_redirects_to_local_next_page(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- next_url: str,
-) -> None:
- with app.app_context():
- user = User(username="admin", password="test")
- db.session.add(user)
- db.session.commit()
- user_id = user.get_id()
-
- mock_login_form(views_module, monkeypatch, valid=True)
-
- response = client.post("/login", query_string={"next": next_url})
-
- assert response.status_code == 302
- assert response.headers["Location"] == next_url
-
- with client.session_transaction() as session:
- assert session["_user_id"] == str(user_id)
-
-
-@pytest.mark.parametrize(
- "next_url",
- [
- pytest.param(None, id="missing"),
- pytest.param("", id="empty"),
- pytest.param("received", id="relative-path"),
- pytest.param("https://evil.example/phishing", id="absolute-https"),
- pytest.param("http://evil.example", id="absolute-http"),
- pytest.param("javascript:alert(1)", id="non-http-scheme"),
- pytest.param("//evil.example/phishing", id="scheme-relative"),
- pytest.param("///evil.example/phishing", id="triple-slash"),
- pytest.param("////evil.example", id="four-slashes"),
- pytest.param(r"/\evil.example", id="slash-backslash-host"),
- pytest.param(r"/received\evil.example", id="backslash-in-path"),
- pytest.param(r"\/evil.example", id="backslash-slash-host"),
- ],
-)
-def test_login_rejects_unsafe_next_page(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- next_url: str | None,
-) -> None:
- with app.app_context():
- user = User(username="admin", password="test")
- db.session.add(user)
- db.session.commit()
- user_id = user.get_id()
-
- mock_login_form(views_module, monkeypatch, valid=True)
-
- query_string = {} if next_url is None else {"next": next_url}
- response = client.post("/login", query_string=query_string)
-
- assert response.status_code == 302
- assert response.headers["Location"] == "/"
-
- with client.session_transaction() as session:
- assert session["_user_id"] == str(user_id)
-
-
-def test_login_redirects_authenticated_user(app: Flask, client: FlaskClient) -> None:
- log_in(app, client)
-
- response = client.get("/login")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/")
-
-
-def test_received_requires_login(client: FlaskClient) -> None:
- response = client.get("/received")
-
- assert response.status_code == 302
- assert "/login" in response.headers["Location"]
-
-
-def test_received_lists_webmentions(app: Flask, client: FlaskClient) -> None:
- log_in(app, client)
- create_received_webmention(app)
-
- response = client.get("/received")
-
- assert response.status_code == 200
- html = response.get_data(as_text=True)
- assert SOURCE_URL in html
- assert TARGET_URL in html
-
-
-@pytest.mark.parametrize("action", ["delete", "reverify"])
-def test_received_admin_actions_require_login(client: FlaskClient, action: str) -> None:
- response = client.post(f"/received/{uuid.uuid7()}/{action}")
-
- assert response.status_code == 302
- assert "/login" in response.headers["Location"]
-
-
-@pytest.mark.parametrize("action", ["delete", "reverify"])
-def test_received_admin_actions_reject_invalid_form(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- action: str,
-) -> None:
- log_in(app, client)
- identifier = create_received_webmention(app)
- form = mock_admin_action_form(views_module, monkeypatch, valid=False)
-
- response = client.post(f"/received/{identifier}/{action}")
-
- assert response.status_code == 400
- form.validate_on_submit.assert_called_once_with()
-
-
-@pytest.mark.parametrize("action", ["delete", "reverify"])
-def test_received_admin_actions_return_404_for_unknown_webmention(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- action: str,
-) -> None:
- log_in(app, client)
- mock_admin_action_form(views_module, monkeypatch, valid=True)
-
- response = client.post(f"/received/{uuid.uuid7()}/{action}")
-
- assert response.status_code == 404
-
-
-def test_delete_received_webmention(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- identifier = create_received_webmention(app)
- mock_admin_action_form(views_module, monkeypatch, valid=True)
-
- response = client.post(f"/received/{identifier}/delete?page=3")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/received?page=3")
-
- with app.app_context():
- assert db.session.get(ReceivedWebmention, identifier) is None
-
-
-def test_reverify_received_webmention(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- identifier = create_received_webmention(
- app, status="failed", failure_reason="Previous failure"
- )
- with app.app_context():
- row = db.session.get(ReceivedWebmention, identifier)
- assert row is not None
- row.recovery_attempts = 3
- db.session.commit()
- mock_admin_action_form(views_module, monkeypatch, valid=True)
- verify_webmention = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
-
- response = client.post(f"/received/{identifier}/reverify?page=2")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/received?page=2")
-
- with app.app_context():
- webmention = db.session.get(ReceivedWebmention, identifier)
- assert webmention is not None
- assert webmention.status == "received"
- assert webmention.failure_reason is None
- assert webmention.recovery_attempts == 0
-
- verify_webmention.assert_called_once_with(identifier)
-
-
-def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None:
- response = client.post("/sent/rescan")
-
- assert response.status_code == 302
- assert "/login" in response.headers["Location"]
-
-
-@pytest.mark.parametrize("valid", [False, True])
-def test_rescan_sent_sources(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- valid: bool,
-) -> None:
- log_in(app, client)
- form = mock_admin_action_form(views_module, monkeypatch, valid=valid)
- manual_scan_sources = Mock()
- monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources)
-
- response = client.post("/sent/rescan")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/sent")
- form.validate_on_submit.assert_called_once_with()
-
- if valid:
- manual_scan_sources.assert_called_once_with()
- else:
- manual_scan_sources.assert_not_called()
-
-
-def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None:
- response = client.post(f"/sent/{uuid.uuid7()}/{uuid.uuid7()}/resend")
-
- assert response.status_code == 302
- assert "/login" in response.headers["Location"]
-
-
-def test_resend_sent_webmention_rejects_invalid_form(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- source_identifier, webmention_identifier = create_resendable_sent_webmention(app)
- mock_admin_action_form(views_module, monkeypatch, valid=False)
-
- response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
-
- assert response.status_code == 400
-
-
-def test_resend_sent_webmention_returns_404_for_unknown_webmention(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- source_identifier = create_sent_source(app)
- mock_admin_action_form(views_module, monkeypatch, valid=True)
-
- response = client.post(f"/sent/{source_identifier}/{uuid.uuid7()}/resend")
-
- assert response.status_code == 404
-
-
-def test_resend_sent_webmention_rejects_wrong_source(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- _, webmention_identifier = create_resendable_sent_webmention(app)
- wrong_source_identifier = create_sent_source(app)
- mock_admin_action_form(views_module, monkeypatch, valid=True)
-
- response = client.post(
- f"/sent/{wrong_source_identifier}/{webmention_identifier}/resend"
- )
-
- assert response.status_code == 404
-
-
-def test_resend_sent_webmention_reopens_current_revision(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- log_in(app, client)
- source_identifier, webmention_identifier = create_resendable_sent_webmention(app)
- mock_admin_action_form(views_module, monkeypatch, valid=True)
- send_webmention = Mock()
- monkeypatch.setattr(views_module, "send_webmention", send_webmention)
-
- with app.app_context():
- webmention = db.session.get(SentWebmention, webmention_identifier)
- assert webmention is not None
- webmention.attempt_count = 6
- webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24)
- db.session.commit()
-
- response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith(f"/sent/{source_identifier}")
-
- with app.app_context():
- webmention = db.session.get(SentWebmention, webmention_identifier)
- assert webmention is not None
- assert webmention.desired_revision == 3
- assert webmention.processed_revision is None
- assert webmention.attempt_count == 0
- assert webmention.next_retry_at is None
- assert webmention.sent_revision == 2
- assert webmention.status is None
- assert webmention.failure_reason is None
- assert webmention.endpoint is None
- assert webmention.response_status is None
- assert webmention.status_url is None
-
- send_webmention.assert_called_once_with(webmention_identifier)
-
-
-def test_status_returns_404_for_unknown_webmention(client: FlaskClient) -> None:
- response = client.get(f"/status/{uuid.uuid7()}")
-
- assert response.status_code == 404
-
-
-def test_status_displays_webmention(app: Flask, client: FlaskClient) -> None:
- identifier = create_received_webmention(
- app, status="failed", failure_reason="Verification failed"
- )
-
- response = client.get(f"/status/{identifier}")
-
- assert response.status_code == 200
- html = response.get_data(as_text=True)
- assert SOURCE_URL in html
- assert TARGET_URL in html
- assert "failed" in html
- assert "Verification failed" in html
-
-
-def test_endpoint_recovers_from_concurrent_insert(
- client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- mock_endpoint_form(views_module, monkeypatch, valid=True)
-
- competing_identifier = uuid.uuid7()
- competing_webmention = ReceivedWebmention(
- uuid=competing_identifier,
- source=SOURCE_URL,
- target=TARGET_URL,
- status="failed",
- failure_reason="Competing request failed",
- )
-
- initial_result = Mock()
- initial_result.scalar_one_or_none.return_value = None
- competing_result = Mock()
- competing_result.scalar_one_or_none.return_value = competing_webmention
-
- execute = Mock(side_effect=[initial_result, competing_result])
- add = Mock()
- rollback = Mock()
- commit = Mock(
- side_effect=[
- IntegrityError(
- "INSERT INTO received_webmention ...",
- {},
- Exception("duplicate source/target"),
- ),
- None,
- ]
- )
- verify_webmention = Mock()
-
- monkeypatch.setattr(views_module.db.session, "execute", execute)
- monkeypatch.setattr(views_module.db.session, "add", add)
- monkeypatch.setattr(views_module.db.session, "rollback", rollback)
- monkeypatch.setattr(views_module.db.session, "commit", commit)
- monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
-
- response = client.post("/endpoint")
-
- assert response.status_code == 201
- assert response.headers["Location"].endswith(f"/status/{competing_identifier}")
-
- assert execute.call_count == 2
- add.assert_called_once_with(ANY)
- rollback.assert_called_once_with()
- assert commit.call_count == 1
-
- assert competing_webmention.status == "failed"
- assert competing_webmention.failure_reason == "Competing request failed"
- verify_webmention.assert_not_called()
-
-
-def test_endpoint_rate_limits_requests_per_ip(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["ENDPOINT_RATE_LIMIT"] = "2/minute"
-
- form = mock_endpoint_form(views_module, monkeypatch, valid=True)
- verify = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify)
-
- responses = []
-
- for index in range(3):
- form.source.data = f"https://source.example/post-{index}"
- responses.append(
- client.post("/endpoint", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
- )
-
- assert [r.status_code for r in responses] == [201, 201, 429]
-
- assert verify.call_count == 2
-
- with app.app_context():
- count = db.session.scalar(
- sa.select(sa.func.count()).select_from(ReceivedWebmention)
- )
- assert count == 2
-
-
-def test_endpoint_rate_limit_distinguishes_clients_behind_trusted_proxy(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- from werkzeug.middleware.proxy_fix import ProxyFix
-
- app.config["ENDPOINT_RATE_LIMIT"] = "1/minute"
- app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
- form = mock_endpoint_form(views_module, monkeypatch, valid=True)
- monkeypatch.setattr(views_module, "verify_webmention", Mock())
-
- def attempt(client_ip: str, index: int) -> int:
- form.source.data = f"https://source.example/proxy-post-{index}"
- return client.post(
- "/endpoint",
- environ_overrides={"REMOTE_ADDR": "192.0.2.200"},
- headers={"X-Forwarded-For": client_ip},
- ).status_code
-
- assert attempt("198.51.100.11", 0) == 201
- assert attempt("198.51.100.11", 1) == 429
- assert attempt("198.51.100.12", 2) == 201
-
-
-def test_endpoint_global_rate_limit(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["ENDPOINT_GLOBAL_RATE_LIMIT"] = "2/minute"
-
- form = mock_endpoint_form(views_module, monkeypatch, valid=True)
- verify = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify)
-
- responses = []
-
- for index in range(3):
- form.source.data = f"https://source.example/post-{index}"
-
- responses.append(
- client.post(
- "/endpoint", environ_overrides={"REMOTE_ADDR": f"192.0.2.{index + 1}"}
- )
- )
-
- assert [r.status_code for r in responses] == [201, 201, 429]
- assert verify.call_count == 2
-
-
-def test_endpoint_recent_duplicate_does_not_reverify(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- mock_endpoint_form(views_module, monkeypatch, valid=True)
-
- verify = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify)
-
- with app.app_context():
- webmention = ReceivedWebmention(
- uuid=uuid.uuid7(),
- source=SOURCE_URL,
- target=TARGET_URL,
- status="failed",
- failure_reason="Previous failure",
- )
- db.session.add(webmention)
- db.session.commit()
- identifier = webmention.uuid
-
- response = client.post("/endpoint")
-
- assert response.status_code == 201
- assert response.headers["Location"].endswith(str(identifier))
-
- with app.app_context():
- stored = db.session.get(ReceivedWebmention, identifier)
- assert stored is not None
- assert stored.status == "failed"
- assert stored.failure_reason == "Previous failure"
-
- verify.assert_not_called()
-
-
-def test_endpoint_reverifies_after_cooldown(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- mock_endpoint_form(views_module, monkeypatch, valid=True)
-
- verify = Mock()
- monkeypatch.setattr(views_module, "verify_webmention", verify)
-
- with app.app_context():
- webmention = ReceivedWebmention(
- uuid=uuid.uuid7(),
- source=SOURCE_URL,
- target=TARGET_URL,
- status="failed",
- failure_reason="Previous failure",
- recovery_attempts=3,
- updated_at=datetime.now(UTC) - timedelta(minutes=11),
- )
- db.session.add(webmention)
- db.session.commit()
- identifier = webmention.uuid
-
- response = client.post("/endpoint")
-
- assert response.status_code == 201
-
- with app.app_context():
- stored = db.session.get(ReceivedWebmention, identifier)
- assert stored is not None
- assert stored.status == "received"
- assert stored.failure_reason is None
- assert stored.recovery_attempts == 0
-
- verify.assert_called_once_with(identifier)
-
-
-@pytest.mark.parametrize("username", ["admin", "unknown-user"])
-def test_login_always_checks_password(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
- username: str,
-) -> None:
- with app.app_context():
- db.session.add(User(username="admin", password="test"))
- db.session.commit()
-
- mock_login_form(
- views_module,
- monkeypatch,
- valid=True,
- username=username,
- password="wrong-password",
- )
-
- original_check_password = User.check_password
- checked_users: list[User] = []
-
- def spy_check_password(self: User, password: str) -> bool:
- checked_users.append(self)
- return original_check_password(self, password)
-
- monkeypatch.setattr(User, "check_password", spy_check_password)
-
- response = client.post("/login")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/login")
- assert len(checked_users) == 1
-
- if username == "admin":
- assert checked_users[0].username == "admin"
- else:
- assert checked_users[0] is views_module.DUMMY_USER
-
- with client.session_transaction() as session:
- assert "_user_id" not in session
- assert ("danger", "Invalid username or password") in session["_flashes"]
-
-
-def test_login_rejects_dummy_user_even_if_password_matches(
- client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- mock_login_form(
- views_module,
- monkeypatch,
- valid=True,
- username="unknown-user",
- password="anything",
- )
-
- checked_users: list[User] = []
-
- def mock_check_password(self: User, password: str) -> bool:
- checked_users.append(self)
- return True
-
- monkeypatch.setattr(User, "check_password", mock_check_password)
-
- response = client.post("/login")
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/login")
- assert checked_users == [views_module.DUMMY_USER]
-
- with client.session_transaction() as session:
- assert "_user_id" not in session
-
-
-def test_login_rate_limit_is_independent_per_ip(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["LOGIN_RATE_LIMIT"] = "1/minute"
-
- mock_login_form(
- views_module,
- monkeypatch,
- valid=True,
- username="unknown-user",
- password="wrong-password",
- )
-
- def attempt(ip: str) -> int:
- response = client.post("/login", environ_overrides={"REMOTE_ADDR": ip})
- return response.status_code
-
- assert attempt("192.0.2.10") == 302
- assert attempt("192.0.2.10") == 429
-
- # Another IP still has its own allowance.
- assert attempt("192.0.2.11") == 302
- assert attempt("192.0.2.11") == 429
-
-
-def test_login_rate_limit_uses_forwarded_client_ip_when_proxy_is_trusted(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- from werkzeug.middleware.proxy_fix import ProxyFix
-
- app.config["LOGIN_RATE_LIMIT"] = "1/minute"
- app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
- mock_login_form(
- views_module,
- monkeypatch,
- valid=True,
- username="unknown-user",
- password="wrong-password",
- )
-
- def attempt(client_ip: str) -> int:
- response = client.post(
- "/login",
- environ_overrides={"REMOTE_ADDR": "192.0.2.200"},
- headers={"X-Forwarded-For": client_ip},
- )
- return response.status_code
-
- assert attempt("198.51.100.11") == 302
- # A forged earlier X-Forwarded-For entry cannot change the client bucket.
- assert attempt("203.0.113.99, 198.51.100.11") == 429
- assert attempt("198.51.100.12") == 302
-
-
-def test_login_username_limit_throttles_across_ips(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["LOGIN_RATE_LIMIT"] = "100/minute"
- app.config["LOGIN_USERNAME_RATE_LIMIT"] = "2/minute"
-
- with monkeypatch.context() as patcher:
- mock_login_form(
- views_module,
- patcher,
- valid=True,
- username="unknown-user",
- password="wrong-password",
- )
-
- def attempt(ip: str, username: str) -> int:
- return client.post(
- "/login",
- data={"username": username, "password": "wrong-password"},
- environ_overrides={"REMOTE_ADDR": ip},
- ).status_code
-
- assert attempt("192.0.2.10", "Admin") == 302
- assert attempt("192.0.2.11", " admin ") == 302
- assert attempt("192.0.2.12", "ADMIN") == 429
- # An unrelated username is not locked by attempts against Admin.
- assert attempt("192.0.2.12", "another-user") == 302
-
- # Restore the real form: the POST-only limits must not block a GET,
- # even from the client IP that submitted the throttled attempt.
- response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.12"})
- assert response.status_code == 200
-
-
-def test_login_missing_username_uses_per_ip_bucket(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["LOGIN_RATE_LIMIT"] = "100/minute"
- app.config["LOGIN_USERNAME_RATE_LIMIT"] = "1/minute"
- mock_login_form(
- views_module,
- monkeypatch,
- valid=True,
- username="unknown-user",
- password="wrong-password",
- )
-
- def attempt(ip: str) -> int:
- return client.post("/login", environ_overrides={"REMOTE_ADDR": ip}).status_code
-
- assert attempt("192.0.2.10") == 302
- assert attempt("192.0.2.10") == 429
- assert attempt("192.0.2.11") == 302
-
-
-def test_login_rate_limits_requests_per_ip(
- app: Flask,
- client: FlaskClient,
- views_module: ModuleType,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- app.config["LOGIN_RATE_LIMIT"] = "2/minute"
-
- with monkeypatch.context() as patcher:
- form = mock_login_form(
- views_module,
- patcher,
- valid=True,
- username="unknown-user",
- password="wrong-password",
- )
-
- responses = [
- client.post("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
- for _ in range(3)
- ]
-
- assert [response.status_code for response in responses] == [302, 302, 429]
- assert form.validate_on_submit.call_count == 2
- assert "Too many login attempts" in responses[-1].get_data(as_text=True)
-
- # The real LoginForm is restored here.
- # GET requests must remain accessible after throttling.
- response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
- assert response.status_code == 200
-
- with client.session_transaction() as session:
- assert "_user_id" not in session
-
-
-def test_logout_rejects_get(app: Flask, client: FlaskClient) -> None:
- log_in(app, client)
-
- response = client.get("/logout")
-
- assert response.status_code == 405
-
- with client.session_transaction() as session:
- assert "_user_id" in session
-
-
-@pytest.mark.parametrize(
- "data",
- [
- pytest.param({}, id="missing"),
- pytest.param({"csrf_token": "invalid"}, id="invalid"),
- ],
-)
-def test_logout_rejects_invalid_csrf(
- app: Flask, client: FlaskClient, data: dict[str, str]
-) -> None:
- app.config["WTF_CSRF_ENABLED"] = True
- log_in(app, client)
-
- response = client.post("/logout", data=data)
-
- assert response.status_code == 400
-
- with client.session_transaction() as session:
- assert "_user_id" in session
-
-
-def test_logout_ends_session(app: Flask, client: FlaskClient) -> None:
- import re
-
- app.config["WTF_CSRF_ENABLED"] = True
- log_in(app, client)
-
- page = client.get("/received")
- assert page.status_code == 200
-
- html = page.get_data(as_text=True)
-
- assert 'action="/logout"' in html
- assert 'method="post"' in html
-
- match = re.search(r'name="csrf_token"\s+value="([^"]+)"', html)
- assert match is not None
-
- response = client.post("/logout", data={"csrf_token": match.group(1)})
-
- assert response.status_code == 302
- assert response.headers["Location"].endswith("/")
-
- with client.session_transaction() as session:
- assert "_user_id" not in session
-
- protected_response = client.get("/received")
- assert protected_response.status_code == 302
- assert "/login" in protected_response.headers["Location"]
-
-
-def create_outgoing(
- app: Flask, *, status_url: str | None = None
-) -> tuple[uuid.UUID, uuid.UUID]:
- now = datetime.now(UTC)
- with app.app_context():
- source = Source(
- path="blog/post/index.html",
- url="https://source.example/blog/post",
- content_hash="0" * 32,
- revision=1,
- last_seen_at=now,
- revised_at=now,
- )
- mention = SentWebmention(
- target="https://example.com/article",
- active=True,
- desired_revision=1,
- status_url=status_url,
- response_status=201 if status_url is not None else None,
- )
- source.sent_webmentions.append(mention)
- db.session.add(source)
- db.session.commit()
- return source.uuid, mention.uuid
-
-
-@pytest.mark.parametrize(
- ("status_url", "safe"),
- [
- ("https://example.com/status", True),
- ("http://example.com/status", True),
- ("javascript:alert(1)", False),
- ("data:text/html,evil", False),
- ("ftp://example.com/status", False),
- ],
-)
-def test_admin_only_links_safe_existing_status_urls(
- app: Flask, client: FlaskClient, status_url: str, safe: bool
-) -> None:
- source_identifier, _ = create_outgoing(app, status_url=status_url)
- with app.app_context():
- user = User(username="admin", password="test")
- db.session.add(user)
- db.session.commit()
- user_id = user.get_id()
-
- with client.session_transaction() as session:
- session["_user_id"] = str(user_id)
- session["_fresh"] = True
-
- response = client.get(f"/sent/{source_identifier}")
- assert response.status_code == 200
- html = response.get_data(as_text=True)
- assert (f'href="{status_url}"' in html) is safe
- assert status_url in html
-
-
-def test_endpoint_rejects_request_bodies_over_configured_maximum(
- app: Flask, client: FlaskClient
-) -> None:
- app.config["MAX_CONTENT_LENGTH"] = 128
- response = client.post(
- "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL}
- )
- assert response.status_code == 413
-
-
-def test_endpoint_rejects_bad_url_before_inserting_row(
- app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch
-) -> None:
- from webmentions_ssg import views
-
- verify_webmention = Mock()
- monkeypatch.setattr(views, "verify_webmention", verify_webmention)
-
- response = client.post(
- "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL}
- )
-
- assert response.status_code == 400
- assert "source" in response.get_json()
- verify_webmention.assert_not_called()
- with app.app_context():
- assert (
- db.session.scalar(
- sa.select(sa.func.count()).select_from(ReceivedWebmention)
- )
- == 0
- )
diff --git a/tests/cli/__init__.py b/tests/views/__init__.py
index 25d78d5..25d78d5 100644
--- a/tests/cli/__init__.py
+++ b/tests/views/__init__.py
diff --git a/tests/views/helpers.py b/tests/views/helpers.py
new file mode 100644
index 0000000..b32a066
--- /dev/null
+++ b/tests/views/helpers.py
@@ -0,0 +1,210 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import uuid
+from datetime import UTC, datetime
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+from flask import Flask
+from flask.testing import FlaskClient
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import (
+ ReceivedWebmention,
+ SentWebmention,
+ SentWebmentionStatus,
+ Source,
+ User,
+)
+
+SOURCE_URL = "https://source.example/post"
+TARGET_URL = "https://dennisfink.me/blog/example/"
+
+
+def mock_endpoint_form(
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ valid: bool,
+ source: str = SOURCE_URL,
+ target: str = TARGET_URL,
+ errors: dict[str, list[str]] | None = None,
+) -> Mock:
+ form = Mock()
+ form.validate_on_submit.return_value = valid
+ form.source.data = source
+ form.target.data = target
+ form.errors = errors or {}
+
+ monkeypatch.setattr(views_module.forms, "EndpointForm", Mock(return_value=form))
+ return form
+
+
+def log_in(app: Flask, client: FlaskClient) -> None:
+ with app.app_context():
+ user = User(username="admin", password="test")
+
+ db.session.add(user)
+ db.session.commit()
+
+ user_id = user.get_id()
+
+ with client.session_transaction() as session:
+ session["_user_id"] = str(user_id)
+ session["_fresh"] = True
+
+
+def create_sent_source(app: Flask) -> uuid.UUID:
+ now = datetime.now(UTC)
+
+ with app.app_context():
+ source = Source(
+ path="blog/example/index.html",
+ url="https://dennisfink.me/blog/example/",
+ content_hash="0" * 32,
+ revision=2,
+ last_seen_at=now,
+ revised_at=now,
+ )
+
+ source.sent_webmentions.extend(
+ [
+ SentWebmention(
+ target="https://example.com/b",
+ active=True,
+ desired_revision=2,
+ processed_revision=2,
+ sent_revision=2,
+ status=SentWebmentionStatus.SENT,
+ endpoint="https://example.com/webmention",
+ response_status=202,
+ ),
+ SentWebmention(
+ target="https://example.com/a",
+ active=True,
+ desired_revision=2,
+ processed_revision=2,
+ sent_revision=1,
+ status=SentWebmentionStatus.FAILED,
+ failure_reason="Webmention endpoint returned HTTP 400",
+ endpoint="https://example.com/webmention",
+ response_status=400,
+ ),
+ ]
+ )
+
+ db.session.add(source)
+ db.session.commit()
+
+ return source.uuid
+
+
+def mock_login_form(
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ valid: bool,
+ username: str = "admin",
+ password: str | None = "test",
+) -> Mock:
+ form = Mock()
+ form.validate_on_submit.return_value = valid
+ form.username.data = username
+ form.password.data = password
+
+ monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form))
+ return form
+
+
+def mock_admin_action_form(
+ views_module: ModuleType, monkeypatch: pytest.MonkeyPatch, *, valid: bool
+) -> Mock:
+ form = Mock()
+ form.validate_on_submit.return_value = valid
+
+ monkeypatch.setattr(views_module.forms, "AdminActionForm", Mock(return_value=form))
+ return form
+
+
+def create_received_webmention(
+ app: Flask,
+ *,
+ source: str = SOURCE_URL,
+ target: str = TARGET_URL,
+ status: str = "verified",
+ failure_reason: str | None = None,
+) -> uuid.UUID:
+ with app.app_context():
+ webmention = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source=source,
+ target=target,
+ status=status,
+ failure_reason=failure_reason,
+ )
+
+ db.session.add(webmention)
+ db.session.commit()
+
+ return webmention.uuid
+
+
+def create_resendable_sent_webmention(app: Flask) -> tuple[uuid.UUID, uuid.UUID]:
+ now = datetime.now(UTC)
+
+ with app.app_context():
+ source = Source(
+ path="blog/resend/index.html",
+ url="https://dennisfink.me/blog/resend/",
+ content_hash="1" * 32,
+ revision=3,
+ last_seen_at=now,
+ revised_at=now,
+ )
+ webmention = SentWebmention(
+ target="https://example.com/resend",
+ active=True,
+ desired_revision=3,
+ processed_revision=3,
+ sent_revision=2,
+ status=SentWebmentionStatus.FAILED,
+ failure_reason="Previous failure",
+ endpoint="https://example.com/webmention",
+ response_status=500,
+ status_url="https://example.com/webmention/status/123",
+ )
+ source.sent_webmentions.append(webmention)
+
+ db.session.add(source)
+ db.session.commit()
+
+ return source.uuid, webmention.uuid
+
+
+def create_outgoing(
+ app: Flask, *, status_url: str | None = None
+) -> tuple[uuid.UUID, uuid.UUID]:
+ now = datetime.now(UTC)
+ with app.app_context():
+ source = Source(
+ path="blog/post/index.html",
+ url="https://source.example/blog/post",
+ content_hash="0" * 32,
+ revision=1,
+ last_seen_at=now,
+ revised_at=now,
+ )
+ mention = SentWebmention(
+ target="https://example.com/article",
+ active=True,
+ desired_revision=1,
+ status_url=status_url,
+ response_status=201 if status_url is not None else None,
+ )
+ source.sent_webmentions.append(mention)
+ db.session.add(source)
+ db.session.commit()
+ return source.uuid, mention.uuid
diff --git a/tests/views/test_admin.py b/tests/views/test_admin.py
new file mode 100644
index 0000000..26a13f8
--- /dev/null
+++ b/tests/views/test_admin.py
@@ -0,0 +1,362 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+from flask import Flask
+from flask.testing import FlaskClient
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import ReceivedWebmention, SentWebmention, User
+
+from .helpers import (
+ SOURCE_URL,
+ TARGET_URL,
+ create_outgoing,
+ create_received_webmention,
+ create_resendable_sent_webmention,
+ create_sent_source,
+ log_in,
+ mock_admin_action_form,
+)
+
+
+@pytest.fixture
+def views_module(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.views import admin as module
+
+ return module
+
+
+@pytest.mark.parametrize("path", ["/sent", f"/sent/{uuid.uuid7()}"])
+def test_sent_views_require_login(client: FlaskClient, path: str) -> None:
+ response = client.get(path)
+
+ assert response.status_code == 302
+ assert "/login" in response.headers["Location"]
+
+
+def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+ identifier = create_sent_source(app)
+
+ response = client.get("/sent")
+
+ assert response.status_code == 200
+
+ html = response.get_data(as_text=True)
+
+ assert "blog/example/index.html" in html
+ assert "https://dennisfink.me/blog/example/" in html
+ assert f"/sent/{identifier}" in html
+
+
+def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+ identifier = create_sent_source(app)
+
+ response = client.get(f"/sent/{identifier}")
+
+ assert response.status_code == 200
+
+ html = response.get_data(as_text=True)
+
+ assert "blog/example/index.html" in html
+ assert "https://dennisfink.me/blog/example/" in html
+
+ first_target = "https://example.com/a"
+ second_target = "https://example.com/b"
+
+ assert first_target in html
+ assert second_target in html
+ assert html.index(first_target) < html.index(second_target)
+
+ assert "Webmention endpoint returned HTTP 400" in html
+ assert "https://example.com/webmention" in html
+
+
+def test_sent_source_returns_404_for_unknown_source(
+ app: Flask, client: FlaskClient
+) -> None:
+ log_in(app, client)
+ response = client.get(f"/sent/{uuid.uuid7()}")
+ assert response.status_code == 404
+
+
+def test_received_requires_login(client: FlaskClient) -> None:
+ response = client.get("/received")
+
+ assert response.status_code == 302
+ assert "/login" in response.headers["Location"]
+
+
+def test_received_lists_webmentions(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+ create_received_webmention(app)
+
+ response = client.get("/received")
+
+ assert response.status_code == 200
+ html = response.get_data(as_text=True)
+ assert SOURCE_URL in html
+ assert TARGET_URL in html
+
+
+@pytest.mark.parametrize("action", ["delete", "reverify"])
+def test_received_admin_actions_require_login(client: FlaskClient, action: str) -> None:
+ response = client.post(f"/received/{uuid.uuid7()}/{action}")
+
+ assert response.status_code == 302
+ assert "/login" in response.headers["Location"]
+
+
+@pytest.mark.parametrize("action", ["delete", "reverify"])
+def test_received_admin_actions_reject_invalid_form(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ action: str,
+) -> None:
+ log_in(app, client)
+ identifier = create_received_webmention(app)
+ form = mock_admin_action_form(views_module, monkeypatch, valid=False)
+
+ response = client.post(f"/received/{identifier}/{action}")
+
+ assert response.status_code == 400
+ form.validate_on_submit.assert_called_once_with()
+
+
+@pytest.mark.parametrize("action", ["delete", "reverify"])
+def test_received_admin_actions_return_404_for_unknown_webmention(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ action: str,
+) -> None:
+ log_in(app, client)
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+
+ response = client.post(f"/received/{uuid.uuid7()}/{action}")
+
+ assert response.status_code == 404
+
+
+def test_delete_received_webmention(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ identifier = create_received_webmention(app)
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+
+ response = client.post(f"/received/{identifier}/delete?page=3")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/received?page=3")
+
+ with app.app_context():
+ assert db.session.get(ReceivedWebmention, identifier) is None
+
+
+def test_reverify_received_webmention(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ identifier = create_received_webmention(
+ app, status="failed", failure_reason="Previous failure"
+ )
+ with app.app_context():
+ row = db.session.get(ReceivedWebmention, identifier)
+ assert row is not None
+ row.recovery_attempts = 3
+ db.session.commit()
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+ verify_webmention = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
+
+ response = client.post(f"/received/{identifier}/reverify?page=2")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/received?page=2")
+
+ with app.app_context():
+ webmention = db.session.get(ReceivedWebmention, identifier)
+ assert webmention is not None
+ assert webmention.status == "received"
+ assert webmention.failure_reason is None
+ assert webmention.recovery_attempts == 0
+
+ verify_webmention.assert_called_once_with(identifier)
+
+
+def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None:
+ response = client.post("/sent/rescan")
+
+ assert response.status_code == 302
+ assert "/login" in response.headers["Location"]
+
+
+@pytest.mark.parametrize("valid", [False, True])
+def test_rescan_sent_sources(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ valid: bool,
+) -> None:
+ log_in(app, client)
+ form = mock_admin_action_form(views_module, monkeypatch, valid=valid)
+ manual_scan_sources = Mock()
+ monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources)
+
+ response = client.post("/sent/rescan")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/sent")
+ form.validate_on_submit.assert_called_once_with()
+
+ if valid:
+ manual_scan_sources.assert_called_once_with()
+ else:
+ manual_scan_sources.assert_not_called()
+
+
+def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None:
+ response = client.post(f"/sent/{uuid.uuid7()}/{uuid.uuid7()}/resend")
+
+ assert response.status_code == 302
+ assert "/login" in response.headers["Location"]
+
+
+def test_resend_sent_webmention_rejects_invalid_form(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ source_identifier, webmention_identifier = create_resendable_sent_webmention(app)
+ mock_admin_action_form(views_module, monkeypatch, valid=False)
+
+ response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
+
+ assert response.status_code == 400
+
+
+def test_resend_sent_webmention_returns_404_for_unknown_webmention(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ source_identifier = create_sent_source(app)
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+
+ response = client.post(f"/sent/{source_identifier}/{uuid.uuid7()}/resend")
+
+ assert response.status_code == 404
+
+
+def test_resend_sent_webmention_rejects_wrong_source(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ _, webmention_identifier = create_resendable_sent_webmention(app)
+ wrong_source_identifier = create_sent_source(app)
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+
+ response = client.post(
+ f"/sent/{wrong_source_identifier}/{webmention_identifier}/resend"
+ )
+
+ assert response.status_code == 404
+
+
+def test_resend_sent_webmention_reopens_current_revision(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ source_identifier, webmention_identifier = create_resendable_sent_webmention(app)
+ mock_admin_action_form(views_module, monkeypatch, valid=True)
+ send_webmention = Mock()
+ monkeypatch.setattr(views_module, "send_webmention", send_webmention)
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+ assert webmention is not None
+ webmention.attempt_count = 6
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24)
+ db.session.commit()
+
+ response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith(f"/sent/{source_identifier}")
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+ assert webmention is not None
+ assert webmention.desired_revision == 3
+ assert webmention.processed_revision is None
+ assert webmention.attempt_count == 0
+ assert webmention.next_retry_at is None
+ assert webmention.sent_revision == 2
+ assert webmention.status is None
+ assert webmention.failure_reason is None
+ assert webmention.endpoint is None
+ assert webmention.response_status is None
+ assert webmention.status_url is None
+
+ send_webmention.assert_called_once_with(webmention_identifier)
+
+
+@pytest.mark.parametrize(
+ ("status_url", "safe"),
+ [
+ ("https://example.com/status", True),
+ ("http://example.com/status", True),
+ ("javascript:alert(1)", False),
+ ("data:text/html,evil", False),
+ ("ftp://example.com/status", False),
+ ],
+)
+def test_admin_only_links_safe_existing_status_urls(
+ app: Flask, client: FlaskClient, status_url: str, safe: bool
+) -> None:
+ source_identifier, _ = create_outgoing(app, status_url=status_url)
+ with app.app_context():
+ user = User(username="admin", password="test")
+ db.session.add(user)
+ db.session.commit()
+ user_id = user.get_id()
+
+ with client.session_transaction() as session:
+ session["_user_id"] = str(user_id)
+ session["_fresh"] = True
+
+ response = client.get(f"/sent/{source_identifier}")
+ assert response.status_code == 200
+ html = response.get_data(as_text=True)
+ assert (f'href="{status_url}"' in html) is safe
+ assert status_url in html
diff --git a/tests/views/test_endpoint.py b/tests/views/test_endpoint.py
new file mode 100644
index 0000000..652250f
--- /dev/null
+++ b/tests/views/test_endpoint.py
@@ -0,0 +1,422 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import ModuleType
+from unittest.mock import ANY, Mock
+
+import pytest
+import sqlalchemy as sa
+from flask import Flask
+from flask.testing import FlaskClient
+from sqlalchemy.exc import IntegrityError
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import ReceivedWebmention
+
+from .helpers import (
+ SOURCE_URL,
+ TARGET_URL,
+ create_received_webmention,
+ mock_endpoint_form,
+)
+
+
+@pytest.fixture
+def views_module(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.views import endpoint as module
+
+ return module
+
+
+def test_endpoint_only_accepts_post(client: FlaskClient) -> None:
+ response = client.get("/endpoint")
+ assert response.status_code == 405
+
+
+def test_endpoint_returns_form_errors(
+ client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ errors = {"source": ["Invalid source"]}
+ mock_endpoint_form(views_module, monkeypatch, valid=False, errors=errors)
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 400
+ assert response.get_json() == errors
+
+
+def test_endpoint_creates_webmention(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ form = mock_endpoint_form(views_module, monkeypatch, valid=True)
+ verify_webmention = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 201
+ form.validate_on_submit.assert_called_once_with()
+
+ with app.app_context():
+ webmention = db.session.scalar(sa.select(ReceivedWebmention))
+
+ assert webmention is not None
+ assert webmention.source == SOURCE_URL
+ assert webmention.target == TARGET_URL
+ assert webmention.status == "received"
+ assert webmention.failure_reason is None
+
+ identifier = webmention.uuid
+
+ assert response.headers["Location"].endswith(str(identifier))
+ verify_webmention.assert_called_once_with(identifier)
+
+
+def test_endpoint_is_idempotent(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ mock_endpoint_form(views_module, monkeypatch, valid=True)
+ verify_webmention = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
+
+ first_response = client.post("/endpoint")
+ second_response = client.post("/endpoint")
+
+ assert first_response.status_code == 201
+ assert second_response.status_code == 201
+ assert first_response.headers["Location"] == second_response.headers["Location"]
+
+ with app.app_context():
+ webmentions = db.session.scalars(sa.select(ReceivedWebmention)).all()
+
+ assert len(webmentions) == 1
+
+ webmention = webmentions[0]
+
+ assert webmention.source == SOURCE_URL
+ assert webmention.target == TARGET_URL
+ assert webmention.status == "received"
+ assert webmention.failure_reason is None
+
+ identifier = webmention.uuid
+
+ verify_webmention.assert_called_once_with(identifier)
+
+
+def test_resending_resets_failure_state(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ mock_endpoint_form(views_module, monkeypatch, valid=True)
+ verify_webmention = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
+
+ with app.app_context():
+ existing = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source=SOURCE_URL,
+ target=TARGET_URL,
+ status="failed",
+ failure_reason="Previous failure",
+ updated_at=datetime.now(UTC) - timedelta(minutes=11),
+ )
+
+ db.session.add(existing)
+ db.session.commit()
+
+ identifier = existing.uuid
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 201
+
+ with app.app_context():
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ assert webmention is not None
+ assert webmention.status == "received"
+ assert webmention.failure_reason is None
+
+ verify_webmention.assert_called_once_with(identifier)
+
+
+def test_status_returns_404_for_unknown_webmention(client: FlaskClient) -> None:
+ response = client.get(f"/status/{uuid.uuid7()}")
+
+ assert response.status_code == 404
+
+
+def test_status_displays_webmention(app: Flask, client: FlaskClient) -> None:
+ identifier = create_received_webmention(
+ app, status="failed", failure_reason="Verification failed"
+ )
+
+ response = client.get(f"/status/{identifier}")
+
+ assert response.status_code == 200
+ html = response.get_data(as_text=True)
+ assert SOURCE_URL in html
+ assert TARGET_URL in html
+ assert "failed" in html
+ assert "Verification failed" in html
+
+
+def test_endpoint_recovers_from_concurrent_insert(
+ client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ mock_endpoint_form(views_module, monkeypatch, valid=True)
+
+ competing_identifier = uuid.uuid7()
+ competing_webmention = ReceivedWebmention(
+ uuid=competing_identifier,
+ source=SOURCE_URL,
+ target=TARGET_URL,
+ status="failed",
+ failure_reason="Competing request failed",
+ )
+
+ initial_result = Mock()
+ initial_result.scalar_one_or_none.return_value = None
+ competing_result = Mock()
+ competing_result.scalar_one_or_none.return_value = competing_webmention
+
+ execute = Mock(side_effect=[initial_result, competing_result])
+ add = Mock()
+ rollback = Mock()
+ commit = Mock(
+ side_effect=[
+ IntegrityError(
+ "INSERT INTO received_webmention ...",
+ {},
+ Exception("duplicate source/target"),
+ ),
+ None,
+ ]
+ )
+ verify_webmention = Mock()
+
+ monkeypatch.setattr(views_module.db.session, "execute", execute)
+ monkeypatch.setattr(views_module.db.session, "add", add)
+ monkeypatch.setattr(views_module.db.session, "rollback", rollback)
+ monkeypatch.setattr(views_module.db.session, "commit", commit)
+ monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 201
+ assert response.headers["Location"].endswith(f"/status/{competing_identifier}")
+
+ assert execute.call_count == 2
+ add.assert_called_once_with(ANY)
+ rollback.assert_called_once_with()
+ assert commit.call_count == 1
+
+ assert competing_webmention.status == "failed"
+ assert competing_webmention.failure_reason == "Competing request failed"
+ verify_webmention.assert_not_called()
+
+
+def test_endpoint_rate_limits_requests_per_ip(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["ENDPOINT_RATE_LIMIT"] = "2/minute"
+
+ form = mock_endpoint_form(views_module, monkeypatch, valid=True)
+ verify = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify)
+
+ responses = []
+
+ for index in range(3):
+ form.source.data = f"https://source.example/post-{index}"
+ responses.append(
+ client.post("/endpoint", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
+ )
+
+ assert [r.status_code for r in responses] == [201, 201, 429]
+
+ assert verify.call_count == 2
+
+ with app.app_context():
+ count = db.session.scalar(
+ sa.select(sa.func.count()).select_from(ReceivedWebmention)
+ )
+ assert count == 2
+
+
+def test_endpoint_rate_limit_distinguishes_clients_behind_trusted_proxy(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from werkzeug.middleware.proxy_fix import ProxyFix
+
+ app.config["ENDPOINT_RATE_LIMIT"] = "1/minute"
+ app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
+ form = mock_endpoint_form(views_module, monkeypatch, valid=True)
+ monkeypatch.setattr(views_module, "verify_webmention", Mock())
+
+ def attempt(client_ip: str, index: int) -> int:
+ form.source.data = f"https://source.example/proxy-post-{index}"
+ return client.post(
+ "/endpoint",
+ environ_overrides={"REMOTE_ADDR": "192.0.2.200"},
+ headers={"X-Forwarded-For": client_ip},
+ ).status_code
+
+ assert attempt("198.51.100.11", 0) == 201
+ assert attempt("198.51.100.11", 1) == 429
+ assert attempt("198.51.100.12", 2) == 201
+
+
+def test_endpoint_global_rate_limit(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["ENDPOINT_GLOBAL_RATE_LIMIT"] = "2/minute"
+
+ form = mock_endpoint_form(views_module, monkeypatch, valid=True)
+ verify = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify)
+
+ responses = []
+
+ for index in range(3):
+ form.source.data = f"https://source.example/post-{index}"
+
+ responses.append(
+ client.post(
+ "/endpoint", environ_overrides={"REMOTE_ADDR": f"192.0.2.{index + 1}"}
+ )
+ )
+
+ assert [r.status_code for r in responses] == [201, 201, 429]
+ assert verify.call_count == 2
+
+
+def test_endpoint_recent_duplicate_does_not_reverify(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ mock_endpoint_form(views_module, monkeypatch, valid=True)
+
+ verify = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify)
+
+ with app.app_context():
+ webmention = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source=SOURCE_URL,
+ target=TARGET_URL,
+ status="failed",
+ failure_reason="Previous failure",
+ )
+ db.session.add(webmention)
+ db.session.commit()
+ identifier = webmention.uuid
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 201
+ assert response.headers["Location"].endswith(str(identifier))
+
+ with app.app_context():
+ stored = db.session.get(ReceivedWebmention, identifier)
+ assert stored is not None
+ assert stored.status == "failed"
+ assert stored.failure_reason == "Previous failure"
+
+ verify.assert_not_called()
+
+
+def test_endpoint_reverifies_after_cooldown(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ mock_endpoint_form(views_module, monkeypatch, valid=True)
+
+ verify = Mock()
+ monkeypatch.setattr(views_module, "verify_webmention", verify)
+
+ with app.app_context():
+ webmention = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source=SOURCE_URL,
+ target=TARGET_URL,
+ status="failed",
+ failure_reason="Previous failure",
+ recovery_attempts=3,
+ updated_at=datetime.now(UTC) - timedelta(minutes=11),
+ )
+ db.session.add(webmention)
+ db.session.commit()
+ identifier = webmention.uuid
+
+ response = client.post("/endpoint")
+
+ assert response.status_code == 201
+
+ with app.app_context():
+ stored = db.session.get(ReceivedWebmention, identifier)
+ assert stored is not None
+ assert stored.status == "received"
+ assert stored.failure_reason is None
+ assert stored.recovery_attempts == 0
+
+ verify.assert_called_once_with(identifier)
+
+
+def test_endpoint_rejects_request_bodies_over_configured_maximum(
+ app: Flask, client: FlaskClient
+) -> None:
+ app.config["MAX_CONTENT_LENGTH"] = 128
+ response = client.post(
+ "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL}
+ )
+ assert response.status_code == 413
+
+
+def test_endpoint_rejects_bad_url_before_inserting_row(
+ app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from webmentions_ssg.views import endpoint as views
+
+ verify_webmention = Mock()
+ monkeypatch.setattr(views, "verify_webmention", verify_webmention)
+
+ response = client.post(
+ "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL}
+ )
+
+ assert response.status_code == 400
+ assert "source" in response.get_json()
+ verify_webmention.assert_not_called()
+ with app.app_context():
+ assert (
+ db.session.scalar(
+ sa.select(sa.func.count()).select_from(ReceivedWebmention)
+ )
+ == 0
+ )
diff --git a/tests/views/test_session.py b/tests/views/test_session.py
new file mode 100644
index 0000000..6a96032
--- /dev/null
+++ b/tests/views/test_session.py
@@ -0,0 +1,431 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import ModuleType
+
+import pytest
+from flask import Flask
+from flask.testing import FlaskClient
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import User
+
+from .helpers import log_in, mock_login_form
+
+
+@pytest.fixture
+def views_module(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.views import session as module
+
+ return module
+
+
+def test_login_renders_form(client: FlaskClient) -> None:
+ response = client.get("/login")
+
+ assert response.status_code == 200
+ assert "login" in response.get_data(as_text=True).lower()
+
+
+@pytest.mark.parametrize("password", ("wrong-password", None))
+def test_login_rejects_invalid_credentials(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ password: str | None,
+) -> None:
+ with app.app_context():
+ db.session.add(User(username="admin", password="test"))
+ db.session.commit()
+
+ mock_login_form(
+ views_module, monkeypatch, valid=True, username="admin", password=password
+ )
+
+ response = client.post("/login")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/login")
+
+ with client.session_transaction() as session:
+ assert "_user_id" not in session
+ assert ("danger", "Invalid username or password") in session["_flashes"]
+
+
+@pytest.mark.parametrize(
+ "next_url",
+ [
+ pytest.param("/", id="root"),
+ pytest.param("/received", id="local-path"),
+ pytest.param("/received?page=2", id="query-string"),
+ pytest.param("/received#details", id="fragment"),
+ ],
+)
+def test_login_redirects_to_local_next_page(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ next_url: str,
+) -> None:
+ with app.app_context():
+ user = User(username="admin", password="test")
+ db.session.add(user)
+ db.session.commit()
+ user_id = user.get_id()
+
+ mock_login_form(views_module, monkeypatch, valid=True)
+
+ response = client.post("/login", query_string={"next": next_url})
+
+ assert response.status_code == 302
+ assert response.headers["Location"] == next_url
+
+ with client.session_transaction() as session:
+ assert session["_user_id"] == str(user_id)
+
+
+@pytest.mark.parametrize(
+ "next_url",
+ [
+ pytest.param(None, id="missing"),
+ pytest.param("", id="empty"),
+ pytest.param("received", id="relative-path"),
+ pytest.param("https://evil.example/phishing", id="absolute-https"),
+ pytest.param("http://evil.example", id="absolute-http"),
+ pytest.param("javascript:alert(1)", id="non-http-scheme"),
+ pytest.param("//evil.example/phishing", id="scheme-relative"),
+ pytest.param("///evil.example/phishing", id="triple-slash"),
+ pytest.param("////evil.example", id="four-slashes"),
+ pytest.param(r"/\evil.example", id="slash-backslash-host"),
+ pytest.param(r"/received\evil.example", id="backslash-in-path"),
+ pytest.param(r"\/evil.example", id="backslash-slash-host"),
+ ],
+)
+def test_login_rejects_unsafe_next_page(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ next_url: str | None,
+) -> None:
+ with app.app_context():
+ user = User(username="admin", password="test")
+ db.session.add(user)
+ db.session.commit()
+ user_id = user.get_id()
+
+ mock_login_form(views_module, monkeypatch, valid=True)
+
+ query_string = {} if next_url is None else {"next": next_url}
+ response = client.post("/login", query_string=query_string)
+
+ assert response.status_code == 302
+ assert response.headers["Location"] == "/"
+
+ with client.session_transaction() as session:
+ assert session["_user_id"] == str(user_id)
+
+
+def test_login_redirects_authenticated_user(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+
+ response = client.get("/login")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/")
+
+
+@pytest.mark.parametrize("username", ["admin", "unknown-user"])
+def test_login_always_checks_password(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ username: str,
+) -> None:
+ with app.app_context():
+ db.session.add(User(username="admin", password="test"))
+ db.session.commit()
+
+ mock_login_form(
+ views_module,
+ monkeypatch,
+ valid=True,
+ username=username,
+ password="wrong-password",
+ )
+
+ original_check_password = User.check_password
+ checked_users: list[User] = []
+
+ def spy_check_password(self: User, password: str) -> bool:
+ checked_users.append(self)
+ return original_check_password(self, password)
+
+ monkeypatch.setattr(User, "check_password", spy_check_password)
+
+ response = client.post("/login")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/login")
+ assert len(checked_users) == 1
+
+ if username == "admin":
+ assert checked_users[0].username == "admin"
+ else:
+ assert checked_users[0] is views_module.DUMMY_USER
+
+ with client.session_transaction() as session:
+ assert "_user_id" not in session
+ assert ("danger", "Invalid username or password") in session["_flashes"]
+
+
+def test_login_rejects_dummy_user_even_if_password_matches(
+ client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ mock_login_form(
+ views_module,
+ monkeypatch,
+ valid=True,
+ username="unknown-user",
+ password="anything",
+ )
+
+ checked_users: list[User] = []
+
+ def mock_check_password(self: User, password: str) -> bool:
+ checked_users.append(self)
+ return True
+
+ monkeypatch.setattr(User, "check_password", mock_check_password)
+
+ response = client.post("/login")
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/login")
+ assert checked_users == [views_module.DUMMY_USER]
+
+ with client.session_transaction() as session:
+ assert "_user_id" not in session
+
+
+def test_login_rate_limit_is_independent_per_ip(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["LOGIN_RATE_LIMIT"] = "1/minute"
+
+ mock_login_form(
+ views_module,
+ monkeypatch,
+ valid=True,
+ username="unknown-user",
+ password="wrong-password",
+ )
+
+ def attempt(ip: str) -> int:
+ response = client.post("/login", environ_overrides={"REMOTE_ADDR": ip})
+ return response.status_code
+
+ assert attempt("192.0.2.10") == 302
+ assert attempt("192.0.2.10") == 429
+
+ # Another IP still has its own allowance.
+ assert attempt("192.0.2.11") == 302
+ assert attempt("192.0.2.11") == 429
+
+
+def test_login_rate_limit_uses_forwarded_client_ip_when_proxy_is_trusted(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from werkzeug.middleware.proxy_fix import ProxyFix
+
+ app.config["LOGIN_RATE_LIMIT"] = "1/minute"
+ app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
+ mock_login_form(
+ views_module,
+ monkeypatch,
+ valid=True,
+ username="unknown-user",
+ password="wrong-password",
+ )
+
+ def attempt(client_ip: str) -> int:
+ response = client.post(
+ "/login",
+ environ_overrides={"REMOTE_ADDR": "192.0.2.200"},
+ headers={"X-Forwarded-For": client_ip},
+ )
+ return response.status_code
+
+ assert attempt("198.51.100.11") == 302
+ # A forged earlier X-Forwarded-For entry cannot change the client bucket.
+ assert attempt("203.0.113.99, 198.51.100.11") == 429
+ assert attempt("198.51.100.12") == 302
+
+
+def test_login_username_limit_throttles_across_ips(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["LOGIN_RATE_LIMIT"] = "100/minute"
+ app.config["LOGIN_USERNAME_RATE_LIMIT"] = "2/minute"
+
+ with monkeypatch.context() as patcher:
+ mock_login_form(
+ views_module,
+ patcher,
+ valid=True,
+ username="unknown-user",
+ password="wrong-password",
+ )
+
+ def attempt(ip: str, username: str) -> int:
+ return client.post(
+ "/login",
+ data={"username": username, "password": "wrong-password"},
+ environ_overrides={"REMOTE_ADDR": ip},
+ ).status_code
+
+ assert attempt("192.0.2.10", "Admin") == 302
+ assert attempt("192.0.2.11", " admin ") == 302
+ assert attempt("192.0.2.12", "ADMIN") == 429
+ # An unrelated username is not locked by attempts against Admin.
+ assert attempt("192.0.2.12", "another-user") == 302
+
+ # Restore the real form: the POST-only limits must not block a GET,
+ # even from the client IP that submitted the throttled attempt.
+ response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.12"})
+ assert response.status_code == 200
+
+
+def test_login_missing_username_uses_per_ip_bucket(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["LOGIN_RATE_LIMIT"] = "100/minute"
+ app.config["LOGIN_USERNAME_RATE_LIMIT"] = "1/minute"
+ mock_login_form(
+ views_module,
+ monkeypatch,
+ valid=True,
+ username="unknown-user",
+ password="wrong-password",
+ )
+
+ def attempt(ip: str) -> int:
+ return client.post("/login", environ_overrides={"REMOTE_ADDR": ip}).status_code
+
+ assert attempt("192.0.2.10") == 302
+ assert attempt("192.0.2.10") == 429
+ assert attempt("192.0.2.11") == 302
+
+
+def test_login_rate_limits_requests_per_ip(
+ app: Flask,
+ client: FlaskClient,
+ views_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ app.config["LOGIN_RATE_LIMIT"] = "2/minute"
+
+ with monkeypatch.context() as patcher:
+ form = mock_login_form(
+ views_module,
+ patcher,
+ valid=True,
+ username="unknown-user",
+ password="wrong-password",
+ )
+
+ responses = [
+ client.post("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
+ for _ in range(3)
+ ]
+
+ assert [response.status_code for response in responses] == [302, 302, 429]
+ assert form.validate_on_submit.call_count == 2
+ assert "Too many login attempts" in responses[-1].get_data(as_text=True)
+
+ # The real LoginForm is restored here.
+ # GET requests must remain accessible after throttling.
+ response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"})
+ assert response.status_code == 200
+
+ with client.session_transaction() as session:
+ assert "_user_id" not in session
+
+
+def test_logout_rejects_get(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+
+ response = client.get("/logout")
+
+ assert response.status_code == 405
+
+ with client.session_transaction() as session:
+ assert "_user_id" in session
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ pytest.param({}, id="missing"),
+ pytest.param({"csrf_token": "invalid"}, id="invalid"),
+ ],
+)
+def test_logout_rejects_invalid_csrf(
+ app: Flask, client: FlaskClient, data: dict[str, str]
+) -> None:
+ app.config["WTF_CSRF_ENABLED"] = True
+ log_in(app, client)
+
+ response = client.post("/logout", data=data)
+
+ assert response.status_code == 400
+
+ with client.session_transaction() as session:
+ assert "_user_id" in session
+
+
+def test_logout_ends_session(app: Flask, client: FlaskClient) -> None:
+ import re
+
+ app.config["WTF_CSRF_ENABLED"] = True
+ log_in(app, client)
+
+ page = client.get("/received")
+ assert page.status_code == 200
+
+ html = page.get_data(as_text=True)
+
+ assert 'action="/logout"' in html
+ assert 'method="post"' in html
+
+ match = re.search(r'name="csrf_token"\s+value="([^"]+)"', html)
+ assert match is not None
+
+ response = client.post("/logout", data={"csrf_token": match.group(1)})
+
+ assert response.status_code == 302
+ assert response.headers["Location"].endswith("/")
+
+ with client.session_transaction() as session:
+ assert "_user_id" not in session
+
+ protected_response = client.get("/received")
+ assert protected_response.status_code == 302
+ assert "/login" in protected_response.headers["Location"]
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index 8fed1a7..63e9f13 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -8,47 +8,20 @@ import logging
import logging.handlers
import os
from pathlib import Path
-from urllib.parse import urlsplit
import coloredlogs
from flask import Flask
-from flask_bootstrap import Bootstrap5
-from flask_limiter import Limiter
-from flask_limiter.util import get_remote_address
-from flask_mail import Mail
-from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy
-from flask_wtf.csrf import CSRFProtect
-from sqlalchemy.orm import DeclarativeBase
from werkzeug.middleware.proxy_fix import ProxyFix
-from webmentions_ssg.config import (
- DefaultConfig,
- DevelopmentConfig,
- ProductionConfig,
- normalize_allowed_hostnames,
- normalize_ignored_hostnames,
-)
-
-from .security import validate_secret_key
-from .tasks import Huey
-from .url_security import is_http_url
-
APP_NAME = "webmentions-ssg"
VERSION = "0.0.2"
-
-class Base(DeclarativeBase):
- pass
-
-
-CSRF = CSRFProtect()
-BOOTSTRAP = Bootstrap5()
-DATABASE = SQLAlchemy(model_class=Base)
-MIGRATE = Migrate()
-HUEY = Huey()
-MAIL = Mail()
-LIMITER = Limiter(key_func=get_remote_address)
+from .auth import AUTH
+from .cli import DB_CLI, MAIL_CLI, USERS_CLI
+from .config import DefaultConfig, DevelopmentConfig, ProductionConfig, validate_config
+from .extensions import BOOTSTRAP, CSRF, DATABASE, HUEY, LIMITER, MAIL, MIGRATE
+from .net import is_http_url
+from .views import register_routes
def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
@@ -91,71 +64,15 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config.from_prefixed_env(prefix="FLASK")
app.config.from_prefixed_env(prefix="WEBMENTIONS_SSG")
- # A fixed key is convenient only for local debugging. All other
- # deployments must supply their own persistent, random signing key.
- using_local_dev_key = (
- issubclass(config_class, DevelopmentConfig)
- and app.debug
- and app.config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY
- )
- if not app.testing and not using_local_dev_key:
- validate_secret_key(app.config.get("SECRET_KEY"))
+ app.config.update(validate_config(app.config, config_class=config_class))
- app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
- app.config["ALLOWED_HOSTNAMES"]
- )
- app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames(
- app.config["IGNORED_HOSTNAMES"]
- )
- proxy_hops = app.config["PROXY_FIX_TRUSTED_HOPS"]
- if type(proxy_hops) is not int or proxy_hops < 0:
- raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.")
- if proxy_hops:
- # Trust only the configured number of proxies; never trust forwarded
- # host, port or URL prefix headers for rate limiting.
+ if proxy_hops := app.config["PROXY_FIX_TRUSTED_HOPS"]:
+ # Do not trust forwarded host, port, or URL prefixes.
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=proxy_hops, x_proto=proxy_hops)
- for key in (
- "MAX_CONTENT_LENGTH",
- "MAX_WEBMENTION_URL_BYTES",
- "RECEIVED_RETENTION_DAYS",
- ):
- value = app.config[key]
- if type(value) is not int or value <= 0:
- raise ValueError(f"{key} must be a positive integer.")
- if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]:
- raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.")
-
- if (
- not app.testing
- and not app.debug
- and app.config["RATELIMIT_STORAGE_URI"] == "memory://"
- ):
- raise RuntimeError(
- "Configure RATELIMIT_STORAGE_URI with shared storage before running in production."
- )
-
- if not app.testing and not app.debug:
- # HUEY_URL overrides the legacy HUEY.url option; absent either, Huey
- # falls back to an in-process queue that cannot serve a separate worker.
- huey_url = app.config.get(
- "HUEY_URL", app.config.get("HUEY", {}).get("url", "memory://")
- )
- if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in {
- "memory",
- "blackhole",
- }:
- raise RuntimeError(
- "Configure HUEY_URL with a shared, persistent queue before "
- "running in production (memory:// and blackhole:// are unsafe)."
- )
-
- # Local development and tests should start from a fresh checkout: their
- # default log directory and development SQLite instance are generated.
- if app.testing or app.debug:
- Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True)
- if app.debug:
+ if app.debug or app.testing:
Path(app.instance_path).mkdir(parents=True, exist_ok=True)
+ Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True)
default_logging_formatter = logging.Formatter(app.config["LOG_FORMAT"])
@@ -197,8 +114,6 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
LIMITER.init_app(app)
MAIL.init_app(app)
- from .auth import AUTH
-
AUTH.init_app(app)
BOOTSTRAP.init_app(app)
HUEY.init_app(app)
@@ -214,20 +129,12 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.add_template_filter(is_http_url)
- from .views import root_page
-
- app.register_blueprint(root_page)
-
- from .cli.users import USERS_CLI
+ app.register_blueprint(register_routes())
app.cli.add_command(USERS_CLI)
- from .cli.mail import MAIL_CLI
-
app.cli.add_command(MAIL_CLI)
- from .cli.database import DB_CLI
-
app.cli.add_command(DB_CLI)
return app
diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py
index 147220e..633aa82 100644
--- a/webmentions_ssg/auth.py
+++ b/webmentions_ssg/auth.py
@@ -4,17 +4,43 @@
"""Configure user authentication and session loading."""
+import hashlib
+import secrets
from hmac import compare_digest
+from urllib.parse import urlsplit
+from flask import request
+from flask_limiter.util import get_remote_address
from flask_login import LoginManager
-from . import DATABASE as db
+from .extensions import DATABASE as db
from .models import User
AUTH = LoginManager()
AUTH.login_view = "root.login" # pyright: ignore[reportAttributeAccessIssue]
+MIN_PASSWORD_LENGTH = 8
+
+DUMMY_USER = User(username=secrets.token_urlsafe())
+DUMMY_USER.password = secrets.token_urlsafe()
+
+
+def validate_password(password: str) -> None:
+ """Validate newly chosen passwords without arbitrary composition rules.
+
+ Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases,
+ but reject empty passwords and control characters.
+ """
+ if not password:
+ raise ValueError("Password must not be empty.")
+ if len(password) < MIN_PASSWORD_LENGTH:
+ raise ValueError(
+ f"Password must be at least {MIN_PASSWORD_LENGTH} characters long."
+ )
+ if any(ord(char) < 32 or ord(char) == 127 for char in password):
+ raise ValueError("Password must not contain control characters.")
+
@AUTH.user_loader
def load_user(session_id: str) -> User | None:
@@ -44,3 +70,28 @@ def load_user(session_id: str) -> User | None:
if not compare_digest(session_id, user.get_id()):
return None
return user
+
+
+def login_username_limit_key() -> str:
+ """Group usernames across IPs without exposing plaintext to rate-limit storage."""
+ username = request.form.get("username", "").strip().casefold()
+ if not username:
+ return f"missing-username:{get_remote_address()}"
+ digest = hashlib.sha256(username.encode("utf-8")).hexdigest()
+ return f"username:{digest}"
+
+
+def is_safe_next_url(next_url: str) -> bool:
+ """Allow redirects only to paths on this site."""
+ if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"):
+ return False
+ try:
+ target = urlsplit(next_url)
+ except ValueError:
+ return False
+ return (
+ not target.scheme
+ and not target.netloc
+ and target.path.startswith("/")
+ and not target.path.startswith("//")
+ )
diff --git a/webmentions_ssg/cli/users.py b/webmentions_ssg/cli.py
index a48fc71..0387ba6 100644
--- a/webmentions_ssg/cli/users.py
+++ b/webmentions_ssg/cli.py
@@ -2,22 +2,26 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Provide commands for managing application user accounts."""
+"""Manage users, mail settings, and database migrations."""
import click
import sqlalchemy as sa
+from flask import current_app
from flask.cli import AppGroup
+from flask_mail import Message
+from flask_migrate import upgrade as migrate_upgrade
+from sqlalchemy import inspect
from sqlalchemy.exc import IntegrityError
-from .. import DATABASE
-from ..security import validate_password
+from .auth import validate_password
+from .extensions import DATABASE, MAIL
+from .models import User
USERS_CLI = AppGroup("users", help="Manage users.")
def _require_user(username: str):
"""Return a user by username or report a CLI-friendly error."""
- from ..models import User
user = DATABASE.session.scalar(sa.select(User).where(User.username == username))
if user is None:
@@ -36,7 +40,6 @@ def create_user(username: str, password: str) -> None:
:param password: Plain-text password to hash and store.
:raises click.ClickException: If the credentials are invalid or the user exists.
"""
- from ..models import User
username = username.strip()
if not username:
@@ -63,7 +66,6 @@ def create_user(username: str, password: str) -> None:
@USERS_CLI.command("list", help="List registered users.")
def list_users() -> None:
"""List user IDs and usernames in alphabetical order."""
- from ..models import User
users = DATABASE.session.scalars(sa.select(User).order_by(User.username)).all()
if not users:
@@ -115,3 +117,83 @@ def delete_user(username: str, yes: bool) -> None:
DATABASE.session.delete(user)
DATABASE.session.commit()
click.echo(f"Deleted user {username!r}.")
+
+
+MAIL_CLI = AppGroup("mail", help="Handle mail")
+
+
+@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration")
+@click.option(
+ "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)."
+)
+def send_test_mail(recipient: str | None) -> None:
+ """
+ Send a test email using the configured SMTP server.
+
+ :param recipient: Optional recipient overriding NOTIFICATION_EMAIL.
+ :raises click.ClickException: If configuration is missing or sending fails.
+ """
+ recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL")
+ if not recipient:
+ raise click.ClickException(
+ "No recipient specified. Set NOTIFICATION_EMAIL or pass --to."
+ )
+
+ sender = current_app.config.get("MAIL_DEFAULT_SENDER")
+ if not sender:
+ raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.")
+
+ message = Message(
+ subject="webmentions-ssg: test email",
+ sender=sender,
+ recipients=[recipient],
+ body=(
+ "This is a test email from webmentions-ssg.\n\n"
+ "If you received this message, the outgoing mail configuration works.\n"
+ ),
+ )
+
+ try:
+ MAIL.send(message)
+ except Exception as exc:
+ raise click.ClickException(f"Could not send test email: {exc}") from exc
+
+ click.echo(f"Test email sent to {recipient}.")
+
+
+DB_CLI = AppGroup("db", help="Handle database")
+
+
+def _table_names() -> set[str]:
+ """Return existing table names without modifying the database schema."""
+ with DATABASE.engine.connect() as connection:
+ return set(inspect(connection).get_table_names())
+
+
+@DB_CLI.command("create-db", help="Create a new database using all migrations")
+def create_db() -> None:
+ """Initialize an empty database and record its Alembic revision."""
+ if _table_names():
+ raise click.ClickException(
+ "The database already contains tables. "
+ "Use 'flask db upgrade' for an existing migrated database."
+ )
+
+ # Applying migrations (rather than create_all()) keeps alembic_version in sync.
+ migrate_upgrade()
+ click.echo("Database created at the latest migration revision.")
+
+
+@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision")
+def upgrade_db() -> None:
+ """Apply pending migrations to a previously initialized database."""
+ if "alembic_version" not in _table_names():
+ raise click.ClickException(
+ "No Flask-Migrate revision table found. "
+ "For a new, empty database run 'flask db create-db'. "
+ "For a legacy database, inspect its schema and use "
+ "'flask migrate stamp' only if appropriate."
+ )
+
+ migrate_upgrade()
+ click.echo("Database upgraded to the latest migration revision.")
diff --git a/webmentions_ssg/cli/__init__.py b/webmentions_ssg/cli/__init__.py
deleted file mode 100644
index accaaa3..0000000
--- a/webmentions_ssg/cli/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Group the application command-line management modules."""
diff --git a/webmentions_ssg/cli/database.py b/webmentions_ssg/cli/database.py
deleted file mode 100644
index e918600..0000000
--- a/webmentions_ssg/cli/database.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Simple database commands built on Flask-Migrate."""
-
-import click
-from flask.cli import AppGroup
-from flask_migrate import upgrade as migrate_upgrade
-from sqlalchemy import inspect
-
-from .. import DATABASE
-
-DB_CLI = AppGroup("db", help="Handle database")
-
-
-def _table_names() -> set[str]:
- """Return existing table names without modifying the database schema."""
- with DATABASE.engine.connect() as connection:
- return set(inspect(connection).get_table_names())
-
-
-@DB_CLI.command("create-db", help="Create a new database using all migrations")
-def create_db() -> None:
- """Initialize an empty database and record its Alembic revision."""
- if _table_names():
- raise click.ClickException(
- "The database already contains tables. "
- "Use 'flask db upgrade' for an existing migrated database."
- )
-
- # Applying migrations (rather than create_all()) keeps alembic_version in sync.
- migrate_upgrade()
- click.echo("Database created at the latest migration revision.")
-
-
-@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision")
-def upgrade_db() -> None:
- """Apply pending migrations to a previously initialized database."""
- if "alembic_version" not in _table_names():
- raise click.ClickException(
- "No Flask-Migrate revision table found. "
- "For a new, empty database run 'flask db create-db'. "
- "For a legacy database, inspect its schema and use "
- "'flask migrate stamp' only if appropriate."
- )
-
- migrate_upgrade()
- click.echo("Database upgraded to the latest migration revision.")
diff --git a/webmentions_ssg/cli/mail.py b/webmentions_ssg/cli/mail.py
deleted file mode 100644
index 47687e2..0000000
--- a/webmentions_ssg/cli/mail.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Provide a command for testing outgoing email configuration."""
-
-import click
-from flask import current_app
-from flask.cli import AppGroup
-from flask_mail import Message
-
-from .. import MAIL
-
-MAIL_CLI = AppGroup("mail", help="Handle mail")
-
-
-@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration")
-@click.option(
- "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)."
-)
-def send_test_mail(recipient: str | None) -> None:
- """
- Send a test email using the configured SMTP server.
-
- :param recipient: Optional recipient overriding NOTIFICATION_EMAIL.
- :raises click.ClickException: If configuration is missing or sending fails.
- """
- recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL")
- if not recipient:
- raise click.ClickException(
- "No recipient specified. Set NOTIFICATION_EMAIL or pass --to."
- )
-
- sender = current_app.config.get("MAIL_DEFAULT_SENDER")
- if not sender:
- raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.")
-
- message = Message(
- subject="webmentions-ssg: test email",
- sender=sender,
- recipients=[recipient],
- body=(
- "This is a test email from webmentions-ssg.\n\n"
- "If you received this message, the outgoing mail configuration works.\n"
- ),
- )
-
- try:
- MAIL.send(message)
- except Exception as exc:
- raise click.ClickException(f"Could not send test email: {exc}") from exc
-
- click.echo(f"Test email sent to {recipient}.")
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 8a3e58d..8251da3 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -5,39 +5,11 @@
"""Define application settings for production, development, and testing."""
import logging
+from urllib.parse import urlsplit
+from flask.config import Config
-def normalize_allowed_hostnames(value: object) -> set[str]:
- """Validate and normalize the configured collection of allowed hostnames."""
- if not isinstance(value, (list, tuple, set, frozenset)):
- raise TypeError(
- "ALLOWED_HOSTNAMES must be a list, tuple, or set of hostnames, not a string."
- )
-
- hostnames: set[str] = set()
- for hostname in value:
- if not isinstance(hostname, str) or not hostname.strip():
- raise ValueError("ALLOWED_HOSTNAMES must contain non-empty strings only.")
- hostnames.add(hostname.strip().lower())
- return hostnames
-
-
-def normalize_ignored_hostnames(value: object) -> tuple[str, ...]:
- """Validate and normalize hostname patterns ignored by the source scanner."""
- if not isinstance(value, (list, tuple, set, frozenset)):
- raise TypeError(
- "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, "
- "not a string."
- )
-
- patterns: set[str] = set()
- for pattern in value:
- if not isinstance(pattern, str) or not (
- normalized := pattern.strip().lower().rstrip(".")
- ):
- raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.")
- patterns.add(normalized)
- return tuple(sorted(patterns))
+MIN_SECRET_KEY_LENGTH = 32
class DefaultConfig:
@@ -69,6 +41,7 @@ class DefaultConfig:
RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
+ MAX_TARGET_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
# Absolute wall-clock budget for each verification or delivery attempt.
REQUEST_TOTAL_TIMEOUT = 30.0
@@ -144,3 +117,107 @@ class TestingConfig(DefaultConfig):
HUEY_URL = "memory://"
ALLOWED_HOSTNAMES = {"dennisfink.me"}
+
+
+def normalize_hostnames(value: object) -> set[str]:
+ """Validate and normalize a collection of hostnames."""
+ if not isinstance(value, (list, tuple, set, frozenset)):
+ raise TypeError("Expected a list, tuple, or set of hostnames.")
+
+ hostnames: set[str] = set()
+ for hostname in value:
+ if not isinstance(hostname, str) or not (
+ normalized := hostname.strip().lower().rstrip(".")
+ ):
+ raise ValueError("Hostnames must be non-empty strings.")
+
+ hostnames.add(normalized)
+
+ return hostnames
+
+
+def validate_secret_key(value: object) -> str | bytes:
+ """Require a non-placeholder, sufficiently long deployment signing key."""
+ if not isinstance(value, (str, bytes)):
+ raise TypeError(
+ "Configure a persistent SECRET_KEY of at least 32 random bytes "
+ "before starting."
+ )
+
+ # Flask accepts either text or bytes as its signing key. Reject common
+ # example/development values even when their lengths look sufficient.
+ key = value.encode("utf-8") if isinstance(value, str) else value
+ if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip():
+ raise RuntimeError(
+ "Configure a persistent SECRET_KEY of at least 32 random bytes "
+ "before starting."
+ )
+
+ normalized = key.strip().lower()
+ if normalized.startswith(
+ (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_")
+ ):
+ raise RuntimeError(
+ "SECRET_KEY is an example/development placeholder; "
+ "generate and configure a real random key."
+ )
+
+ return value
+
+
+def validate_config(
+ config: Config, *, config_class: type[DefaultConfig]
+) -> dict[str, object]:
+ """Validate runtime configuration and return normalized values.
+
+ The caller applies the returned values once; validation does not mutate
+ the Flask application or install extensions.
+ """
+ normalized: dict[str, object] = {}
+ for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"):
+ try:
+ normalized[key] = normalize_hostnames(config[key])
+ except (TypeError, ValueError) as exc:
+ raise type(exc)(f"{key}: {exc}") from exc
+
+ proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"]
+ if type(proxy_hops) is not int or proxy_hops < 0:
+ raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.")
+
+ for key in (
+ "MAX_CONTENT_LENGTH",
+ "MAX_WEBMENTION_URL_BYTES",
+ "RECEIVED_RETENTION_DAYS",
+ ):
+ value = config[key]
+ if type(value) is not int or value <= 0:
+ raise ValueError(f"{key} must be a positive integer.")
+
+ if not config["TESTING"] and not (
+ issubclass(config_class, DevelopmentConfig)
+ and config["DEBUG"]
+ and config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY
+ ):
+ normalized["SECRET_KEY"] = validate_secret_key(config.get("SECRET_KEY"))
+
+ if not config["DEBUG"] and not config["TESTING"]:
+ if not normalized["ALLOWED_HOSTNAMES"]:
+ raise RuntimeError(
+ "Configure ALLOWED_HOSTNAMES before running in production."
+ )
+ if config["RATELIMIT_STORAGE_URI"] == "memory://":
+ raise RuntimeError(
+ "Configure RATELIMIT_STORAGE_URI with shared storage before running in production."
+ )
+ huey_url = config.get(
+ "HUEY_URL", config.get("HUEY", {}).get("url", "memory://")
+ )
+ if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in {
+ "memory",
+ "blackhole",
+ }:
+ raise RuntimeError(
+ "Configure HUEY_URL with a shared, persistent queue before "
+ "running in production (memory:// and blackhole:// are unsafe)."
+ )
+ return normalized
diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py
new file mode 100644
index 0000000..0afec9c
--- /dev/null
+++ b/webmentions_ssg/extensions.py
@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Unbound extensions shared across Flask application instances."""
+
+from flask_bootstrap import Bootstrap5
+from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
+from flask_mail import Mail
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from flask_wtf.csrf import CSRFProtect
+from sqlalchemy.orm import DeclarativeBase
+
+from .tasks.extension import Huey
+
+
+class Base(DeclarativeBase):
+ pass
+
+
+CSRF = CSRFProtect()
+BOOTSTRAP = Bootstrap5()
+DATABASE = SQLAlchemy(model_class=Base)
+MIGRATE = Migrate()
+HUEY = Huey()
+MAIL = Mail()
+LIMITER = Limiter(key_func=get_remote_address)
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms.py
index 384c7a1..a721dc6 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms.py
@@ -2,19 +2,21 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Provide validators for URL safety and form field constraints."""
+"""Webmention forms and URL validation."""
import ipaddress
+import re
import unicodedata
from urllib.parse import urlsplit
import httpx
from flask import current_app
-from wtforms import Field, ValidationError
-from wtforms.validators import StopValidation
+from flask_wtf import FlaskForm
+from wtforms import Field, PasswordField, StringField, SubmitField, ValidationError
from wtforms.form import BaseForm
+from wtforms.validators import URL, InputRequired, Regexp, StopValidation
-from ..url_security import AddressResolutionError, is_public_address, is_public_url
+from .net import is_public_address
class URLMaxBytes:
@@ -143,7 +145,7 @@ class AllowedHostname:
:raises ValidationError: If the URL hostname is not configured as allowed.
"""
try:
- hostname = urlsplit(field.data).hostname
+ hostname = (urlsplit(field.data).hostname or "").rstrip(".")
except ValueError, TypeError:
hostname = None
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
@@ -151,42 +153,6 @@ class AllowedHostname:
raise ValidationError(self.message or field.gettext("Invalid input."))
-class PublicURL:
- """
- Validate that a URL resolves exclusively to public IP addresses.
-
- :param message: Optional validation error message.
- """
-
- def __init__(self, message: str | None = None) -> None:
- """
- Initialize the validator.
-
- :param message: Optional validation error message.
- """
- self.message = message
-
- def __call__(self, form: BaseForm, field: Field) -> None:
- """
- Validate that a URL resolves exclusively to public IP addresses.
-
- :param form: Form containing the field.
- :param field: Field containing the URL to validate.
- :raises ValidationError: If the URL cannot be resolved, has no hostname, or
- resolves to a non-public address.
- """
- message = self.message
-
- if message is None:
- message = field.gettext("URL must resolve to a public address.")
-
- try:
- if not is_public_url(field.data):
- raise ValidationError(message)
- except (AddressResolutionError, ValueError) as exc:
- raise ValidationError(message) from exc
-
-
class PublicIPLiteral:
"""Reject non-public IP literals without performing DNS in the web worker."""
@@ -206,3 +172,62 @@ class PublicIPLiteral:
if not is_public_address(address):
raise ValidationError(field.gettext("URL must use a public address."))
+
+
+class LoginForm(FlaskForm):
+ """
+ Form for authenticating a user.
+ """
+
+ username = StringField("Username", validators=[InputRequired()])
+ password = PasswordField("Password", validators=[InputRequired()])
+ submit = SubmitField("Sign In")
+
+
+class EndpointForm(FlaskForm):
+ """
+ Form for receiving a Webmention.
+
+ The source must be a public HTTP or HTTPS URL distinct from the target.
+ The target must be an HTTP or HTTPS URL using an allowed hostname.
+ """
+
+ source = StringField(
+ "source",
+ validators=[
+ InputRequired(),
+ URLMaxBytes(),
+ HTTPXURL(),
+ URL(),
+ Regexp(
+ "^https?://.*",
+ flags=re.IGNORECASE,
+ message="source must begin with http or https",
+ ),
+ NotEqualTo("target"),
+ PublicIPLiteral(),
+ ],
+ )
+ target = StringField(
+ "target",
+ validators=[
+ InputRequired(),
+ URLMaxBytes(),
+ HTTPXURL(),
+ URL(),
+ Regexp(
+ "^https?://.*",
+ flags=re.IGNORECASE,
+ message="target must begin with http or https",
+ ),
+ AllowedHostname(),
+ ],
+ )
+
+
+class AdminActionForm(FlaskForm):
+ """
+ Form for confirming an administrative action.
+ """
+
+ submit = SubmitField("Confirm")
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
deleted file mode 100644
index 589cbd9..0000000
--- a/webmentions_ssg/forms/__init__.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Define forms for login, Webmention submission, and administration."""
-
-import re
-
-from flask_wtf import FlaskForm
-from wtforms import PasswordField, StringField, SubmitField
-from wtforms.validators import URL, InputRequired, Regexp
-
-from .validators import (
- AllowedHostname,
- HTTPXURL,
- NotEqualTo,
- PublicIPLiteral,
- URLMaxBytes,
-)
-
-
-class LoginForm(FlaskForm):
- """
- Form for authenticating a user.
- """
-
- username = StringField("Username", validators=[InputRequired()])
- password = PasswordField("Password", validators=[InputRequired()])
- submit = SubmitField("Sign In")
-
-
-class EndpointForm(FlaskForm):
- """
- Form for receiving a Webmention.
-
- The source must be a public HTTP or HTTPS URL distinct from the target.
- The target must be an HTTP or HTTPS URL using an allowed hostname.
- """
-
- source = StringField(
- "source",
- validators=[
- InputRequired(),
- URLMaxBytes(),
- HTTPXURL(),
- URL(),
- Regexp(
- "^https?://.*",
- flags=re.IGNORECASE,
- message="source must begin with http or https",
- ),
- NotEqualTo("target"),
- PublicIPLiteral(),
- ],
- )
- target = StringField(
- "target",
- validators=[
- InputRequired(),
- URLMaxBytes(),
- HTTPXURL(),
- URL(),
- Regexp(
- "^https?://.*",
- flags=re.IGNORECASE,
- message="target must begin with http or https",
- ),
- AllowedHostname(),
- ],
- )
-
-
-class AdminActionForm(FlaskForm):
- """
- Form for confirming an administrative action.
- """
-
- submit = SubmitField("Confirm")
diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
index 905b2c2..a263a2f 100644
--- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
+++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Track webmention digest notifications
Revision ID: 408fc2eca9ce
diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
index c4eac52..ee6ad09 100644
--- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
+++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Add Webmention retry backoff
Revision ID: 85f2b226ec99
diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
index cbb40ef..08dd45f 100644
--- a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
+++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Index received webmentions for expired verification recovery.
Revision ID: c0272f25ce17
diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
index 0157df3..41c5376 100644
--- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
+++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
@@ -11,7 +11,7 @@ Revises: c0272f25ce17
import sqlalchemy as sa
from alembic import op
-from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
+from webmentions_ssg.models import received_url_digest, sent_target_digest
revision = "d845aa7fbc19"
down_revision = "c0272f25ce17"
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 31516d5..9442309 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -6,6 +6,7 @@
from __future__ import annotations
+import json
import uuid
from datetime import UTC, datetime
from enum import StrEnum
@@ -28,8 +29,17 @@ from sqlalchemy import (
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
from werkzeug.security import check_password_hash, generate_password_hash
-from . import Base
-from .url_digest import received_url_digest, sent_target_digest
+from .extensions import Base
+
+
+class ReceivedStatus(StrEnum):
+ """Persisted lifecycle state of an incoming Webmention."""
+
+ RECEIVED = "received"
+ VERIFYING = "verifying"
+ VERIFIED = "verified"
+ FAILED = "failed"
+ DELETED = "deleted"
class SentWebmentionStatus(StrEnum):
@@ -120,7 +130,9 @@ class ReceivedWebmention(Base):
self.url_digest = received_url_digest(source, target)
return value
- status: Mapped[str] = mapped_column(Text(), nullable=False, default="received")
+ status: Mapped[str] = mapped_column(
+ Text(), nullable=False, default=ReceivedStatus.RECEIVED
+ )
# Counts stale-row requeues, not Huey's bounded in-task retries. A fresh
# explicit submission resets this counter.
recovery_attempts: Mapped[int] = mapped_column(
@@ -154,6 +166,24 @@ class ReceivedWebmention(Base):
"""
return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>"
+ @staticmethod
+ def verification_reset_values(
+ *, at: datetime, recoveries: int = 0
+ ) -> dict[str, object]:
+ """Return the columns to reset when verification is explicitly reopened."""
+ return {
+ "status": ReceivedStatus.RECEIVED,
+ "failure_reason": None,
+ "recovery_attempts": recoveries,
+ "updated_at": at,
+ }
+
+ def reset_for_verification(self) -> ReceivedWebmention:
+ """Reopen an existing row for manual verification and return it."""
+ for key, value in self.verification_reset_values(at=datetime.now(UTC)).items():
+ setattr(self, key, value)
+ return self
+
@property
def verified(self) -> bool:
"""
@@ -161,7 +191,7 @@ class ReceivedWebmention(Base):
:return: Whether the Webmention status is ``verified``.
"""
- return self.status == "verified"
+ return self.status == ReceivedStatus.VERIFIED
@property
def created_at(self) -> datetime:
@@ -308,6 +338,23 @@ class SentWebmention(Base):
"""
return f"<SentWebmention {self.target!r}>"
+ def reopen(
+ self, revision: int, active: bool, *, resend: bool = False
+ ) -> SentWebmention:
+ """Reopen delivery for a revision while preserving send history."""
+ self.desired_revision = revision
+ self.active = active
+ self.attempt_count = 0
+ self.next_retry_at = None
+ if resend:
+ self.processed_revision = None
+ self.status = None
+ self.failure_reason = None
+ self.endpoint = None
+ self.response_status = None
+ self.status_url = None
+ return self
+
@property
def pending(self) -> bool:
"""
@@ -338,3 +385,16 @@ def uuid7_to_datetime(identifier: uuid.UUID) -> datetime:
:return: Timestamp as a timezone-aware UTC datetime.
"""
return datetime.fromtimestamp(identifier.time / 1000, tz=UTC)
+
+
+def received_url_digest(source: str, target: str) -> str:
+ """Fingerprint an unambiguous, ordered source/target pair."""
+ encoded = json.dumps(
+ [source, target], ensure_ascii=False, separators=(",", ":")
+ ).encode("utf-8")
+ return sha256(encoded).hexdigest()
+
+
+def sent_target_digest(target: str) -> str:
+ """Fingerprint an outgoing target (unique within each source ID)."""
+ return sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/net.py
index 131cf55..01a9a80 100644
--- a/webmentions_ssg/public_transport.py
+++ b/webmentions_ssg/net.py
@@ -2,23 +2,31 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""HTTP transport that connects only to validated DNS answers.
-
-httpcore receives the original host for Host and TLS SNI/certificate checks,
-but the socket is opened to an IP from a single, bounded DNS lookup. This
-protects the receiver and sender, including redirects and advertised URLs.
-"""
+"""Public-only HTTP transport, URL checks, and bounded response reads."""
import ipaddress
import socket
import time
from collections.abc import Iterable
+from urllib.parse import urlsplit
+import dns.exception
+import dns.resolver
import httpcore
import httpx
-from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation
+from flask import current_app
+from httpcore._backends.sync import SyncStream
+
+NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
+IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
+
+
+class AddressResolutionError(Exception):
+ """A hostname could not be resolved (potentially transient)."""
-from .url_security import public_addresses
+
+class UnsafeAddressError(ValueError):
+ """A hostname or IP literal identifies a non-public network address."""
class NetworkDeadline:
@@ -142,3 +150,114 @@ class PublicHTTPTransport(httpx.HTTPTransport):
ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
)
+
+
+def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
+ """Return the hardened HTTP client shared by receiver and sender."""
+ headers = {"User-Agent": user_agent}
+ if accept is not None:
+ headers["Accept"] = accept
+ return httpx.Client(
+ headers=headers,
+ timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]),
+ follow_redirects=True,
+ max_redirects=current_app.config["MAX_REDIRECTS"],
+ trust_env=False,
+ transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]),
+ )
+
+
+def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+ """Reject non-public addresses, including IPv4 hidden in IPv6 forms.
+
+ ``is_global`` alone does not reliably exclude IPv6 transition addresses
+ across Python versions or networks with NAT64/6to4 routing.
+ """
+ if isinstance(address, ipaddress.IPv6Address):
+ # The well-known NAT64 prefix can translate an otherwise global IPv6
+ # address into a private IPv4 destination. IPv4-compatible addresses
+ # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
+ if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
+ return False
+
+ # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
+ # destination through ipaddress; validate that destination as well.
+ for embedded in (address.sixtofour, address.ipv4_mapped):
+ if embedded is not None and not embedded.is_global:
+ return False
+
+ return address.is_global
+
+
+def public_addresses(
+ hostname: str, *, lifetime: float | None = None
+) -> tuple[str, ...]:
+ """Resolve *once* and return only checked, globally routable IP addresses.
+
+ A DNS name with a mixture of public and private answers is rejected in its
+ entirety. IP literals are handled without DNS, including IPv6 literals.
+ The returned strings must be used for the actual TCP connection; checking
+ them and then connecting to ``hostname`` reintroduces DNS rebinding.
+ """
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ addresses = (literal,)
+ else:
+ try:
+ answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
+ addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
+ except (dns.exception.DNSException, ValueError) as exc:
+ raise AddressResolutionError(
+ f"Could not resolve hostname {hostname!r}"
+ ) from exc
+
+ if not addresses:
+ raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
+
+ if any(not is_public_address(address) for address in addresses):
+ raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
+
+ return tuple(str(address) for address in addresses)
+
+
+def is_http_url(url: str) -> bool:
+ """
+ Check whether a URL is an absolute HTTP or HTTPS URL.
+
+ :param url: URL to validate.
+ :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ """
+ try:
+ parsed = urlsplit(url)
+ except ValueError:
+ return False
+
+ return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
+
+
+def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
+ """Read a streaming response, rejecting oversized declared or actual bodies."""
+ if (content_length := response.headers.get("Content-Length")) is not None:
+ try:
+ declared_length = int(content_length)
+ except ValueError:
+ pass
+ else:
+ if declared_length > max_bytes:
+ raise ValueError(f"{label} document is too large")
+
+ body = bytearray()
+ for chunk in response.iter_bytes(chunk_size=64 * 1024):
+ body.extend(chunk)
+ if len(body) > max_bytes:
+ raise ValueError(f"{label} document is too large")
+ return bytes(body)
+
+
+def is_temporary_status(status: int) -> bool:
+ """Return whether an HTTP status is normally retryable."""
+ return status in {408, 425, 429} or 500 <= status <= 599
diff --git a/webmentions_ssg/security.py b/webmentions_ssg/security.py
deleted file mode 100644
index f4841a4..0000000
--- a/webmentions_ssg/security.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Validate persistent signing keys and administrator passwords."""
-
-MIN_PASSWORD_LENGTH = 8
-MIN_SECRET_KEY_LENGTH = 32
-
-
-def validate_secret_key(value: object) -> None:
- """Require a non-placeholder, sufficiently long deployment signing key."""
- if not isinstance(value, (str, bytes)):
- raise TypeError(
- "Configure a persistent SECRET_KEY of at least 32 random bytes "
- "before starting."
- )
-
- # Flask accepts either text or bytes as its signing key. Reject common
- # example/development values even when their lengths look sufficient.
- key = value.encode("utf-8") if isinstance(value, str) else value
- if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip():
- raise RuntimeError(
- "Configure a persistent SECRET_KEY of at least 32 random bytes "
- "before starting."
- )
-
- normalized = key.strip().lower()
- if normalized.startswith(
- (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_")
- ):
- raise RuntimeError(
- "SECRET_KEY is an example/development placeholder; "
- "generate and configure a real random key."
- )
-
-
-def validate_password(password: str) -> None:
- """Validate newly chosen passwords without arbitrary composition rules.
-
- Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases,
- but reject empty passwords and control characters.
- """
- if not password:
- raise ValueError("Password must not be empty.")
- if len(password) < MIN_PASSWORD_LENGTH:
- raise ValueError(
- f"Password must be at least {MIN_PASSWORD_LENGTH} characters long."
- )
- if any(ord(char) < 32 or ord(char) == 127 for char in password):
- raise ValueError("Password must not contain control characters.")
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index fb1d46d..bf7189f 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,7 +21,7 @@ def create_consumer() -> Huey:
"""
app = create_app()
- from . import notifications, receiver, retention, scanner, sender # noqa: F401
+ from . import notifications, receiver, retention, scanner
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
new file mode 100644
index 0000000..98f3fd5
--- /dev/null
+++ b/webmentions_ssg/tasks/discovery.py
@@ -0,0 +1,231 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Discover a target Webmention endpoint and submit a POST."""
+
+import re
+from urllib.parse import urljoin
+
+import httpx
+from bs4 import BeautifulSoup
+from flask import current_app
+
+from ..net import is_http_url, is_temporary_status, read_limited_body
+
+LINK_SPLIT = re.compile(r",\s*(?=<)")
+DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
+
+
+class SenderError(Exception):
+ """A Webmention cannot be sent."""
+
+
+class TemporarySenderError(SenderError):
+ """A Webmention send failed for a potentially temporary reason."""
+
+
+class PermanentSenderError(SenderError):
+ """A Webmention send failed permanently for this source revision."""
+
+
+def resolve_endpoint(response: httpx.Response, href: str) -> str:
+ """
+ Resolve and validate a discovered Webmention endpoint.
+
+ :param response: Response from which the endpoint was discovered.
+ :param href: Endpoint reference to resolve.
+ :return: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
+ HTTPS URL.
+ """
+ try:
+ endpoint = urljoin(str(response.url), href.strip())
+ except ValueError as exc:
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
+
+ if not is_http_url(endpoint):
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
+
+ return endpoint
+
+
+def parse_link_value(value: str) -> tuple[str, set[str]] | None:
+ """
+ Parse a Link header value into its target and relations.
+
+ :param value: Link header value to parse.
+ :return: Link target and relation names, or ``None`` if the value is invalid.
+ """
+ value = value.strip()
+
+ if not value.startswith("<"):
+ return None
+
+ href, separator, parameters = value[1:].partition(">")
+
+ if not separator:
+ return None
+
+ relations: set[str] = set()
+
+ for parameter in parameters.split(";"):
+ name, separator, value = parameter.partition("=")
+
+ if separator and name.strip().lower() == "rel":
+ relations.update(
+ relation.lower() for relation in value.strip().strip("\"'").split()
+ )
+
+ return href, relations
+
+
+def endpoint_from_headers(response: httpx.Response) -> str | None:
+ """
+ Find a Webmention endpoint in the response Link headers.
+
+ :param response: HTTP response whose headers should be inspected.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ for header in response.headers.get_list("Link"):
+ for value in LINK_SPLIT.split(header):
+ if (link := parse_link_value(value)) is None:
+ continue
+
+ href, relations = link
+
+ if "webmention" in relations:
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
+ """
+ Find a Webmention endpoint in an HTML document.
+
+ ``link`` and ``a`` elements are inspected in document order.
+
+ :param response: HTTP response from which the document was retrieved.
+ :param body: HTML document body.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ document = BeautifulSoup(body, "html5lib")
+
+ for element in document.find_all(["link", "a"], href=True):
+ relations = element.get("rel")
+
+ if isinstance(relations, str):
+ relations = relations.split()
+
+ if not relations:
+ continue
+
+ if not any(
+ isinstance(relation, str) and relation.lower() == "webmention"
+ for relation in relations
+ ):
+ continue
+
+ href = element.get("href")
+
+ if not isinstance(href, str):
+ continue
+
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def read_target_body(response: httpx.Response) -> bytes:
+ """Return a bounded target body, using the shared HTTP size guard."""
+ try:
+ return read_limited_body(
+ response, current_app.config["MAX_TARGET_BYTES"], label="Target"
+ )
+ except ValueError as exc:
+ raise PermanentSenderError(str(exc)) from exc
+
+
+def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
+ """
+ Discover the Webmention endpoint advertised by a target URL.
+
+ Endpoint discovery first checks the response to a HEAD request and then falls
+ back to a GET request, inspecting both HTTP Link headers and supported HTML
+ documents.
+
+ :param client: HTTP client to use for discovery requests.
+ :param target: Target URL whose Webmention endpoint should be discovered.
+ :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :raises TemporarySenderError: If the target returns a temporary HTTP failure.
+ :raises PermanentSenderError: If GET returns a permanent HTTP failure.
+ """
+ try:
+ head_response = client.head(target, headers=DISCOVERY_HEADERS)
+ except httpx.RequestError, TemporarySenderError:
+ # HEAD is only an optimization. Servers and intermediaries may reject
+ # or drop it even when fetching the document with GET would succeed.
+ pass
+ else:
+ if endpoint := endpoint_from_headers(head_response):
+ return endpoint
+
+ with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
+ if endpoint := endpoint_from_headers(response):
+ return endpoint
+
+ if is_temporary_status(response.status_code):
+ raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
+
+ if not response.is_success:
+ raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
+
+ media_type = (
+ response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
+ )
+
+ if media_type not in {"text/html", "application/xhtml+xml"}:
+ return None
+
+ return endpoint_from_html(response, read_target_body(response))
+
+
+def post_webmention(
+ client: httpx.Client, *, endpoint: str, source: str, target: str
+) -> tuple[int, str | None]:
+ """
+ Send a Webmention to a discovered endpoint.
+
+ :param client: HTTP client to use for the request.
+ :param endpoint: Webmention endpoint URL.
+ :param source: Source URL of the Webmention.
+ :param target: Target URL of the Webmention.
+ :return: HTTP response status code and optional status URL.
+ """
+ with client.stream(
+ "POST", endpoint, data={"source": source, "target": target}
+ ) as response:
+ status_url = None
+
+ if response.status_code == 201 and (
+ location := response.headers.get("Location")
+ ):
+ try:
+ candidate = urljoin(str(response.url), location)
+ except ValueError:
+ # A malformed Location must not fail an otherwise successful send.
+ pass
+ else:
+ if is_http_url(candidate):
+ status_url = candidate
+
+ return (response.status_code, status_url)
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index df6cb21..9f53f3b 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -93,54 +93,37 @@ class Huey:
)
return self._huey
+ def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]:
+ """Return a task wrapper that executes with the configured Flask app."""
+
+ @wraps(func)
+ def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
+ if self.app is None:
+ raise RuntimeError("Flask app is not available.")
+ with self.app.app_context():
+ return func(*args, **kwargs)
+
+ return wrapped
+
def task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's task decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's task decorator.
- :return: Decorator that registers the wrapped function as a Huey task.
- """
+ """Return a Huey task decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func))
return decorator
def periodic_task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a periodic Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's periodic task
- decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's periodic task
- decorator.
- :return: Decorator that registers the wrapped function as a periodic Huey
- task.
- """
+ """Return a periodic Huey decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.periodic_task(*task_args, **task_kwargs)(
+ self._in_app_context(func)
+ )
return decorator
diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py
index 473a544..7e51358 100644
--- a/webmentions_ssg/tasks/json_serializer.py
+++ b/webmentions_ssg/tasks/json_serializer.py
@@ -11,7 +11,7 @@ from typing import Any
from huey.registry import Message
from huey.serializer import Serializer
-from huey.utils import Error, SKIPPED
+from huey.utils import SKIPPED, Error
class JSONSerializer(Serializer):
@@ -39,66 +39,87 @@ class JSONSerializer(Serializer):
@classmethod
def _encode(cls, value: Any) -> Any:
- if value is None or type(value) in {str, int, float, bool}:
- return value
- if isinstance(value, uuid.UUID):
- return {"type": "uuid", "value": str(value)}
- if isinstance(value, datetime):
- return {"type": "datetime", "value": value.isoformat()}
- if isinstance(value, Message):
- return {"type": "message", "value": [cls._encode(v) for v in value]}
- if isinstance(value, Error):
- return {"type": "error", "value": cls._encode(value.metadata)}
- if value is SKIPPED:
- return {"type": "skipped", "value": None}
- if isinstance(value, tuple):
- return {"type": "tuple", "value": [cls._encode(v) for v in value]}
- if isinstance(value, list):
- return {"type": "list", "value": [cls._encode(v) for v in value]}
- if isinstance(value, dict) and all(type(k) is str for k in value):
- return {
- "type": "dict",
- "value": [[k, cls._encode(v)] for k, v in value.items()],
- }
- raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
+ match value:
+ case _ if value is None or type(value) in {str, int, float, bool}:
+ return value
+
+ case uuid.UUID():
+ return {"type": "uuid", "value": str(value)}
+
+ case datetime():
+ return {"type": "datetime", "value": value.isoformat()}
+
+ case Message():
+ return {"type": "message", "value": [cls._encode(v) for v in value]}
+
+ case Error():
+ return {"type": "error", "value": cls._encode(value.metadata)}
+
+ case _ if value is SKIPPED:
+ return {"type": "skipped", "value": None}
+
+ case tuple():
+ return {"type": "tuple", "value": [cls._encode(v) for v in value]}
+
+ case list():
+ return {"type": "list", "value": [cls._encode(v) for v in value]}
+
+ case dict() if all(isinstance(k, str) for k in value):
+ return {
+ "type": "dict",
+ "value": [[k, cls._encode(v)] for k, v in value.items()],
+ }
+
+ case _:
+ raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
@classmethod
def _decode(cls, value: Any) -> Any:
if value is None or type(value) in {str, int, float, bool}:
return value
+
if not isinstance(value, dict) or set(value) != {"type", "value"}:
raise ValueError("Invalid Huey JSON data")
- kind, payload = value["type"], value["value"]
- if kind == "uuid" and isinstance(payload, str):
- return uuid.UUID(payload)
- if kind == "datetime" and isinstance(payload, str):
- return datetime.fromisoformat(payload)
- if kind == "message" and isinstance(payload, list):
- if len(payload) != len(Message._fields):
- raise ValueError("Invalid Huey JSON message field count")
- return Message(*(cls._decode(v) for v in payload))
- if kind == "error":
- metadata = cls._decode(payload)
- if not isinstance(metadata, dict):
- raise ValueError("Invalid Huey JSON error metadata")
- return Error(metadata)
- if kind == "skipped" and payload is None:
- return SKIPPED
- if kind in {"tuple", "list"} and isinstance(payload, list):
- items = [cls._decode(v) for v in payload]
- return tuple(items) if kind == "tuple" else items
- if kind == "dict" and isinstance(payload, list):
- result: dict[str, Any] = {}
- for pair in payload:
- if (
- not isinstance(pair, list)
- or len(pair) != 2
- or type(pair[0]) is not str
- ):
- raise ValueError("Invalid Huey JSON dictionary entry")
- if pair[0] in result:
- raise ValueError("Duplicate Huey JSON dictionary key")
- result[pair[0]] = cls._decode(pair[1])
- return result
- raise ValueError("Unsupported Huey JSON type")
+ match value["type"], value["value"]:
+ case "uuid", str() as payload:
+ return uuid.UUID(payload)
+
+ case "datetime", str() as payload:
+ return datetime.fromisoformat(payload)
+
+ case "message", list() as payload:
+ if len(payload) != len(Message._fields):
+ raise ValueError("Invalid Huey JSON message field count")
+ return Message(*(cls._decode(v) for v in payload))
+
+ case "error", payload:
+ metadata = cls._decode(payload)
+ if not isinstance(metadata, dict):
+ raise TypeError("Invalid Huey JSON error metadata")
+ return Error(metadata)
+
+ case "skipped", None:
+ return SKIPPED
+
+ case ("tuple" | "list") as kind, list() as payload:
+ items = [cls._decode(v) for v in payload]
+ return tuple(items) if kind == "tuple" else items
+
+ case "dict", list() as payload:
+ result: dict[str, Any] = {}
+
+ for pair in payload:
+ match pair:
+ case list([str() as key, encoded]):
+ if key in result:
+ raise ValueError("Duplicate Huey JSON dictionary key")
+ result[key] = cls._decode(encoded)
+
+ case _:
+ raise ValueError("Invalid Huey JSON dictionary entry")
+
+ return result
+
+ case _:
+ raise ValueError("Unsupported Huey JSON type")
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py
new file mode 100644
index 0000000..4e5a5ca
--- /dev/null
+++ b/webmentions_ssg/tasks/mentions.py
@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Pure HTML and text matching for received Webmention targets."""
+
+from urllib.parse import urljoin
+
+import rfc3987
+from bs4 import BeautifulSoup
+
+IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
+TRAILING_IRI_PUNCTUATION = ".,;:!?"
+
+HTML_URL_ATTRIBUTES = {
+ "href": {"a", "area", "link"},
+ "src": {
+ "audio",
+ "embed",
+ "iframe",
+ "img",
+ 'input[type="image" i]',
+ "script",
+ "audio source",
+ "video source",
+ "track",
+ "video",
+ },
+ "cite": {"blockquote", "del", "ins", "q"},
+}
+
+
+def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
+ """
+ Check whether an HTML document mentions the target URL.
+
+ URL references are resolved against the document's ``base`` element when
+ present, or against the source URL otherwise.
+
+ :param body: HTML document body.
+ :param source_url: URL from which the document was retrieved.
+ :param target_url: Exact URL to look for.
+ :return: Whether a supported HTML URL attribute references the target URL.
+ """
+
+ document = BeautifulSoup(body, "html.parser")
+ base_url = source_url
+
+ if (base_element := document.select_one("base[href]")) is not None and isinstance(
+ base_href := base_element.get("href"), str
+ ):
+ base_url = urljoin(source_url, base_href.strip())
+
+ for attribute, selectors in HTML_URL_ATTRIBUTES.items():
+ selector = ", ".join(
+ [f"{selector_string}[{attribute}]" for selector_string in selectors]
+ )
+
+ for element in document.select(selector):
+ if not isinstance(reference := element.get(attribute), str):
+ continue
+
+ if urljoin(base_url, reference.strip()) == target_url:
+ return True
+
+ return False
+
+
+def text_mentions_target(body: str, target_url: str) -> bool:
+ """
+ Check whether plain text contains the target IRI, ignoring trailing prose
+ punctuation but not extra URL path or query characters.
+
+ :param body: Plain-text document body.
+ :param target_url: Exact IRI to look for.
+ :return: Whether the target IRI occurs in the document.
+ """
+ for match in IRI_PATTERN.finditer(body):
+ candidate = match.group()
+
+ # A URL in prose can be followed by punctuation that the IRI pattern
+ # consumes as part of its match. Only remove trailing punctuation,
+ # leaving longer URLs (e.g. ``target/more``) unmatched.
+ if not candidate.startswith(target_url):
+ continue
+
+ while candidate != target_url:
+ if candidate[-1] in TRAILING_IRI_PUNCTUATION:
+ candidate = candidate[:-1]
+ elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
+ # Keep balanced parentheses that belong to the URL itself.
+ candidate = candidate[:-1]
+ else:
+ break
+
+ if candidate == target_url:
+ return True
+
+ return False
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index 475fad1..889030c 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -10,10 +10,10 @@ import sqlalchemy as sa
from flask import current_app
from flask_mail import Message
-from .. import DATABASE as db
-from .. import HUEY as huey
-from .. import MAIL as mail
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..extensions import MAIL as mail
+from ..models import ReceivedStatus, ReceivedWebmention
@huey.lock_task("send-webmention-digest")
@@ -27,7 +27,7 @@ def send_webmention_digest() -> None:
cutoff = datetime.now(UTC)
conditions = (
- ReceivedWebmention.status == "verified",
+ ReceivedWebmention.status == ReceivedStatus.VERIFIED,
ReceivedWebmention.notification_sent_at.is_(None),
ReceivedWebmention.updated_at <= cutoff,
)
@@ -53,7 +53,6 @@ def send_webmention_digest() -> None:
),
)
- # If SMTP fails, the records remain pending for the next digest.
mail.send(message)
db.session.execute(
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index acd66a1..bed923c 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -2,27 +2,27 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Fetch and verify incoming Webmentions against their source pages."""
+"""Fetch and verify incoming Webmentions with transactional state changes."""
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
-import rfc3987
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..models import ReceivedWebmention
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError
-
-
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..models import ReceivedStatus, ReceivedWebmention
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_temporary_status,
+ read_limited_body,
+)
+from .mentions import html_mentions_target, text_mentions_target
class VerificationError(Exception):
@@ -37,95 +37,7 @@ class TemporaryFetchError(Exception):
"""Fetching the source may succeed when retried later."""
-IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
-TRAILING_IRI_PUNCTUATION = ".,;:!?"
-
-
-HTML_URL_ATTRIBUTES = {
- "href": {"a", "area", "link"},
- "src": {
- "audio",
- "embed",
- "iframe",
- "img",
- 'input[type="image" i]',
- "script",
- "audio source",
- "video source",
- "track",
- "video",
- },
- "cite": {"blockquote", "del", "ins", "q"},
-}
-
-
-def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """
- Check whether an HTML document mentions the target URL.
-
- URL references are resolved against the document's ``base`` element when
- present, or against the source URL otherwise.
-
- :param body: HTML document body.
- :param source_url: URL from which the document was retrieved.
- :param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
- """
-
- document = BeautifulSoup(body, "html.parser")
- base_url = source_url
-
- if (base_element := document.select_one("base[href]")) is not None and isinstance(
- base_href := base_element.get("href"), str
- ):
- base_url = urljoin(source_url, base_href.strip())
-
- for attribute, selectors in HTML_URL_ATTRIBUTES.items():
- selector = ", ".join(
- [f"{selector_string}[{attribute}]" for selector_string in selectors]
- )
-
- for element in document.select(selector):
- if not isinstance(reference := element.get(attribute), str):
- continue
-
- if urljoin(base_url, reference.strip()) == target_url:
- return True
-
- return False
-
-
-def text_mentions_target(body: str, target_url: str) -> bool:
- """
- Check whether plain text contains the target IRI, ignoring trailing prose
- punctuation but not extra URL path or query characters.
-
- :param body: Plain-text document body.
- :param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
- """
- for match in IRI_PATTERN.finditer(body):
- candidate = match.group()
-
- # A URL in prose can be followed by punctuation that the IRI pattern
- # consumes as part of its match. Only remove trailing punctuation,
- # leaving longer URLs (e.g. ``target/more``) unmatched.
- if not candidate.startswith(target_url):
- continue
-
- while candidate != target_url:
- if candidate[-1] in TRAILING_IRI_PUNCTUATION:
- candidate = candidate[:-1]
- elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
- # Keep balanced parentheses that belong to the URL itself.
- candidate = candidate[:-1]
- else:
- break
-
- if candidate == target_url:
- return True
-
- return False
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
@@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
try:
- return _fetch_source(source_url)
+ with (
+ build_client(
+ f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if is_temporary_status(status):
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
+
+ try:
+ return response, read_limited_body(
+ response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
+ )
+ except ValueError as exc:
+ raise VerificationError(str(exc)) from exc
+
except UnsafeAddressError as exc:
raise VerificationError("Source resolves to a non-public address") from exc
except AddressResolutionError as exc:
raise TemporaryFetchError("Source hostname could not be resolved") from exc
-def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- with (
- httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client,
- client.stream("GET", source_url) as response,
- ):
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
-
- max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
-
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_source_bytes:
- raise VerificationError("Source document is too large")
-
- return response, bytes(body)
-
-
def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
Fetch a source and check whether it mentions the target URL.
@@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- # Failed tasks may be retried by Huey. Completed and already-running tasks
- # should not start another concurrent verification.
- if webmention.status not in {"received", "failed"}:
+ if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
return
- # A reaper-exhausted row must not be revived by a late Huey retry.
+
if (
- webmention.status == "failed"
+ webmention.status == ReceivedStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return
@@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
ReceivedWebmention.status == previous_status,
ReceivedWebmention.updated_at == previous_updated_at,
)
- .values(status="verifying", failure_reason=None, updated_at=started_at)
+ .values(
+ status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at
+ )
)
+
if claimed.rowcount != 1:
db.session.rollback()
return
- db.session.commit()
- def finish(status: str, failure_reason: str | None) -> None:
- # A timeout or a newer verification may have superseded this attempt.
- # Do not let a late worker overwrite its state.
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == "verifying",
- ReceivedWebmention.updated_at == started_at,
- )
- .values(
- status=status,
- failure_reason=failure_reason,
- updated_at=datetime.now(UTC),
- )
- )
- db.session.commit()
+ db.session.commit()
+ status = None
+ failure_reason = None
try:
mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- finish("deleted", str(exc))
+ status = ReceivedStatus.DELETED
+ failure_reason = str(exc)
except VerificationError as exc:
- finish("failed", str(exc))
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc)
except (TemporaryFetchError, httpx.RequestError) as exc:
- finish("failed", str(exc) or "Source could not be fetched")
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or "Source could not be fetched"
raise
except Exception as exc:
# InvalidURL is not a RequestError; other unexpected source errors must
@@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", webmention_uuid
)
- finish("failed", str(exc) or type(exc).__name__)
+
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or type(exc).__name__
else:
if mentions_target:
- finish("verified", None)
+ status = ReceivedStatus.VERIFIED
else:
- finish("deleted", "Source does not mention target")
+ status = ReceivedStatus.DELETED
+ failure_reason = "Source does not mention target"
+ finally:
+ if status is not None:
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
def recover_stale_webmentions() -> int:
@@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int:
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+
stale = db.session.execute(
sa.select(
ReceivedWebmention.uuid,
@@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.updated_at,
ReceivedWebmention.recovery_attempts,
).where(
- ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ ),
ReceivedWebmention.updated_at <= cutoff,
)
).all()
@@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int:
recovered = 0
exhausted = 0
max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+
for identifier, status, updated_at, recovery_attempts in stale:
at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
@@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.recovery_attempts == recovery_attempts,
)
.values(
- status="failed" if at_limit else "received",
- failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
- updated_at=now,
- recovery_attempts=(
- recovery_attempts if at_limit else recovery_attempts + 1
- ),
+ **(
+ {
+ "status": ReceivedStatus.FAILED,
+ "failure_reason": RECOVERY_EXHAUSTED_REASON,
+ "updated_at": now,
+ "recovery_attempts": recovery_attempts,
+ }
+ if at_limit
+ else ReceivedWebmention.verification_reset_values(
+ at=now, recoveries=recovery_attempts + 1
+ )
+ )
)
)
+
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+
if at_limit:
exhausted += 1
continue
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 727b186..bf79512 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -11,8 +11,8 @@ import sqlalchemy as sa
from flask import current_app
from sqlalchemy.engine import CursorResult
-from .. import DATABASE as db
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..models import ReceivedStatus, ReceivedWebmention
def purge_old_received_webmentions() -> int:
@@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int:
cutoff = datetime.now(UTC) - timedelta(
days=current_app.config["RECEIVED_RETENTION_DAYS"]
)
+
result = cast(
CursorResult[Any],
db.session.execute(
sa.delete(ReceivedWebmention).where(
- ReceivedWebmention.status.in_(("failed", "deleted")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ ),
ReceivedWebmention.updated_at < cutoff,
)
),
)
db.session.commit()
+
removed = result.rowcount
if removed:
current_app.logger.info("Purged %d old failed/deleted Webmentions", removed)
+
return removed
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 666eba7..b2d6def 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -2,404 +2,22 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Scan generated site pages to track outgoing Webmention targets."""
+"""Reconcile scanned static-site sources with the database."""
-import json
-from collections.abc import Iterator
-from dataclasses import dataclass
from datetime import UTC, datetime
-from fnmatch import fnmatchcase
-from itertools import chain
from pathlib import Path
-from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
-import mf2py
import sqlalchemy as sa
-import xxhash
-from bs4 import BeautifulSoup, NavigableString, Tag
-from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..config import normalize_ignored_hostnames
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, Source
-from ..url_security import is_http_url
+from ..net import is_http_url
from .sender import send_webmention
-
-REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-
-CONTENT_BLOCK_TAGS = frozenset(
- {
- "address",
- "article",
- "aside",
- "blockquote",
- "br",
- "dd",
- "div",
- "dl",
- "dt",
- "figcaption",
- "figure",
- "footer",
- "h1",
- "h2",
- "h3",
- "h4",
- "h5",
- "h6",
- "header",
- "hr",
- "li",
- "main",
- "ol",
- "p",
- "pre",
- "section",
- "table",
- "td",
- "th",
- "tr",
- "ul",
- }
-)
-
-
-class SourceScanError(Exception):
- """A source document cannot safely be processed."""
-
-
-@dataclass(frozen=True)
-class ScannedSource:
- path: str
- url: str
- content_hash: str
- targets: frozenset[str]
-
-
-def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """
- Derive the public source URL from a relative filesystem path.
-
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
- """
- directory = relative_path.parent.as_posix()
-
- return urljoin(base_url, f"{quote(directory, safe='/')}/")
-
-
-def canonical_url(
- document: BeautifulSoup, *, relative_path: Path, base_url: str | None
-) -> str | None:
- """
- Return the canonical URL declared by a source document.
-
- Relative canonical URLs are resolved against the configured source base URL.
-
- :param document: Parsed HTML source document.
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
- """
- if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
- href = link.get("href")
-
- if not isinstance(href, str) or not (href := href.strip()):
- raise SourceScanError("Document contains an empty canonical URL")
-
- if is_http_url(href):
- return href
-
- if base_url is None:
- raise SourceScanError(
- "Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
- )
-
- resolved = urljoin(source_url_for_path(relative_path, base_url), href)
-
- if not is_http_url(resolved):
- raise SourceScanError(
- f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
- )
-
- return resolved
-
-
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """
- Yield URL values from a microformats property.
-
- :param entry: Parsed microformats entry.
- :param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
- """
- properties = entry.get("properties")
-
- if not isinstance(properties, dict):
- return
-
- values = properties.get(property_name)
-
- if not isinstance(values, list):
- return
-
- yield from (value for value in values if isinstance(value, str))
-
-
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """
- Parse and validate an h-entry with mf2py.
-
- :param element: HTML element containing the h-entry.
- :param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
- :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
- """
- parsed = mf2py.parse(doc=str(element), url=base_url)
-
- if not isinstance(parsed, dict):
- raise SourceScanError("Microformats parser did not return a document object")
-
- items = parsed.get("items")
-
- if not isinstance(items, list) or len(items) != 1:
- raise SourceScanError("Expected exactly one parsed h-entry")
-
- entry = items[0]
-
- if not isinstance(entry, dict):
- raise SourceScanError("Parsed h-entry is not an object")
-
- types = entry.get("type")
-
- if not isinstance(types, list) or "h-entry" not in types:
- raise SourceScanError("Parsed microformats item is not an h-entry")
-
- return entry
-
-
-def primary_entry(
- document: BeautifulSoup, source_url: str
-) -> tuple[Tag, dict[str, Any]]:
- """
- Return and validate the primary h-entry of a source document.
-
- The document must contain exactly one h-entry whose ``u-url`` matches the
- source URL.
-
- :param document: Parsed HTML source document.
- :param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
- """
- entries = document.find_all(class_="h-entry")
-
- if len(entries) != 1:
- raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
-
- entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
-
- urls = tuple(property_urls(mf2_entry, "url"))
-
- if source_url not in urls:
- raise SourceScanError(
- f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
- )
-
- return entry_element, mf2_entry
-
-
-def content_element(entry: Tag) -> Tag:
- """
- Return the e-content element of an h-entry.
-
- :param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
- """
- contents = entry.find_all(class_="e-content")
-
- if len(contents) != 1:
- raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
-
- return contents[0]
-
-
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """
- Resolve and validate a possible Webmention target URL.
-
- Empty values, non-HTTP URLs, and URLs referring to the source itself are
- discarded.
-
- :param value: URL reference to normalize.
- :param base_url: Base URL against which relative references are resolved.
- :param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
- """
- value = value.strip()
-
- if not value:
- return None
-
- target = urljoin(base_url, value)
-
- if not is_http_url(target):
- return None
-
- if urldefrag(target)[0] == urldefrag(source_url)[0]:
- return None
-
- return target
-
-
-def iter_targets(
- content: Tag,
- mf2_entry: dict[str, Any],
- *,
- base_url: str,
- source_url: str,
- ignored_hostnames: tuple[str, ...] = (),
-) -> Iterator[str]:
- """
- Yield outgoing Webmention targets from an h-entry.
-
- Targets are collected from links in the entry content and supported
- microformats reaction properties. Invalid, self-referencing, and ignored
- targets are excluded.
-
- :param content: e-content element of the h-entry.
- :param mf2_entry: Parsed microformats representation of the h-entry.
- :param base_url: Base URL against which relative targets are resolved.
- :param source_url: URL of the source document.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
- """
- hrefs = (
- href
- for element in content.find_all("a", href=True)
- if isinstance(href := element.get("href"), str)
- )
-
- reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
- )
-
- for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
- if target is not None:
- hostname = urlsplit(target).hostname or ""
- if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
- yield target
-
-
-def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
-
- def extract(node: PageElement) -> str:
- if isinstance(node, NavigableString):
- return str(node)
- if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
- return ""
-
- # A relative-time label may change on every site build even though its
- # machine-readable date stays the same.
- if node.name == "time" and isinstance(value := node.get("datetime"), str):
- return value.strip()
-
- # Image-only posts also have meaningful content, despite having no text.
- if node.name == "img":
- return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
-
- text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
-
- return " ".join(extract(content).split())
-
-
-def scan_source_file(
- path: Path,
- *,
- root: Path,
- base_url: str | None,
- ignored_hostnames: tuple[str, ...] = (),
-) -> ScannedSource:
- """
- Scan one generated source document for outgoing Webmentions.
-
- :param path: Path of the generated HTML document.
- :param root: Root directory containing generated source documents.
- :param base_url: Configured source base URL, if available.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
- :raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
- :raises OSError: If the source document cannot be read.
- """
- # Use the path as published for its URL, but read only files whose
- # resolved target stays inside the configured source directory. Resolving
- # before reading also protects against symlinked index.html files.
- resolved_root = root.resolve()
- resolved_path = path.resolve()
- if not resolved_path.is_relative_to(resolved_root):
- raise SourceScanError(f"Source document escapes source directory: {path}")
-
- relative_path = path.relative_to(root)
- document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
-
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
-
- if source_url is None:
- if base_url is None:
- raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
- )
-
- source_url = source_url_for_path(relative_path, base_url)
-
- entry_element, mf2_entry = primary_entry(document, source_url)
-
- content = content_element(entry_element)
-
- targets = frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- )
-
- # Fingerprint the meaningful post text and its outgoing targets rather than
- # the full h-entry markup. Relative timestamps, classes, and other volatile
- # metadata outside e-content must not cause repeat Webmentions.
- stable_content = normalized_content(content)
- fingerprint = json.dumps(
- {"content": stable_content, "targets": sorted(targets)},
- ensure_ascii=False,
- separators=(",", ":"),
- )
-
- return ScannedSource(
- path=relative_path.as_posix(),
- url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
- targets=targets,
- )
+from .source_parser import ScannedSource, SourceScanError, scan_source_file
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
@@ -466,10 +84,9 @@ def update_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.active = webmention.target in scanned.targets
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(
+ source.revision, webmention.target in scanned.targets
+ )
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -523,10 +140,6 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = normalize_ignored_hostnames(
- current_app.config["IGNORED_HOSTNAMES"]
- )
-
sources_by_path = {
source.path: source
for source in db.session.scalars(
@@ -555,7 +168,7 @@ def scan_sources_task() -> None:
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=ignored_hostnames,
+ ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -592,10 +205,6 @@ def scan_sources_task() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- # A static-site generator can temporarily empty its output directory while
- # rebuilding. With no successfully scanned pages, absence is not evidence
- # of deletion; do not generate removals or send pending Webmentions that
- # might be verified while the site is unavailable.
if scanned_count == 0:
current_app.logger.warning(
"No source documents successfully scanned; "
@@ -612,10 +221,7 @@ def scan_sources_task() -> None:
source.deleted_at = scan_time
for webmention in source.sent_webmentions:
- webmention.active = False
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 9330247..3c4c717 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -2,272 +2,33 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Discover endpoints and deliver outgoing Webmentions."""
+"""Claim, send, and persist outgoing Webmention attempts."""
-import re
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from huey.exceptions import TaskLockedException
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
-
-LINK_SPLIT = re.compile(r",\s*(?=<)")
-
-DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
-
-
-class SenderError(Exception):
- """A Webmention cannot be sent."""
-
-
-class TemporarySenderError(SenderError):
- """A Webmention send failed for a potentially temporary reason."""
-
-
-class PermanentSenderError(SenderError):
- """A Webmention send failed permanently for this source revision."""
-
-
-def temporary_http_status(status: int) -> bool:
- """
- Return whether an HTTP status indicates a temporary failure.
-
- :param status: HTTP response status code.
- :return: Whether the request should be retried.
- """
- return status in {408, 425, 429} or 500 <= status <= 599
-
-
-def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """
- Resolve and validate a discovered Webmention endpoint.
-
- :param response: Response from which the endpoint was discovered.
- :param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
- """
- try:
- endpoint = urljoin(str(response.url), href.strip())
- except ValueError as exc:
- raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
-
- if not is_http_url(endpoint):
- raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
-
- return endpoint
-
-
-def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """
- Parse a Link header value into its target and relations.
-
- :param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
- """
- value = value.strip()
-
- if not value.startswith("<"):
- return None
-
- href, separator, parameters = value[1:].partition(">")
-
- if not separator:
- return None
-
- relations: set[str] = set()
-
- for parameter in parameters.split(";"):
- name, separator, value = parameter.partition("=")
-
- if separator and name.strip().lower() == "rel":
- relations.update(
- relation.lower() for relation in value.strip().strip("\"'").split()
- )
-
- return href, relations
-
-
-def endpoint_from_headers(response: httpx.Response) -> str | None:
- """
- Find a Webmention endpoint in the response Link headers.
-
- :param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- for header in response.headers.get_list("Link"):
- for value in LINK_SPLIT.split(header):
- if (link := parse_link_value(value)) is None:
- continue
-
- href, relations = link
-
- if "webmention" in relations:
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """
- Find a Webmention endpoint in an HTML document.
-
- ``link`` and ``a`` elements are inspected in document order.
-
- :param response: HTTP response from which the document was retrieved.
- :param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- document = BeautifulSoup(body, "html5lib")
-
- for element in document.find_all(["link", "a"], href=True):
- relations = element.get("rel")
-
- if isinstance(relations, str):
- relations = relations.split()
-
- if not relations:
- continue
-
- if not any(
- isinstance(relation, str) and relation.lower() == "webmention"
- for relation in relations
- ):
- continue
-
- href = element.get("href")
-
- if not isinstance(href, str):
- continue
-
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def read_target_body(response: httpx.Response) -> bytes:
- """
- Read a target document up to the configured size limit.
-
- :param response: Streaming HTTP response to read.
- :return: Response body.
- :raises PermanentSenderError: If the document exceeds the configured maximum
- size.
- """
- max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_bytes:
- raise PermanentSenderError("Target document is too large")
- except ValueError:
- pass
-
- body = bytearray()
-
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_bytes:
- raise PermanentSenderError("Target document is too large")
-
- return bytes(body)
-
-
-def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """
- Discover the Webmention endpoint advertised by a target URL.
-
- Endpoint discovery first checks the response to a HEAD request and then falls
- back to a GET request, inspecting both HTTP Link headers and supported HTML
- documents.
-
- :param client: HTTP client to use for discovery requests.
- :param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
- :raises TemporarySenderError: If the target returns a temporary HTTP failure.
- :raises PermanentSenderError: If GET returns a permanent HTTP failure.
- """
- try:
- head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
- # HEAD is only an optimization. Servers and intermediaries may reject
- # or drop it even when fetching the document with GET would succeed.
- pass
- else:
- if endpoint := endpoint_from_headers(head_response):
- return endpoint
-
- with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
- if endpoint := endpoint_from_headers(response):
- return endpoint
-
- if temporary_http_status(response.status_code):
- raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
-
- if not response.is_success:
- raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
-
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
-
- if media_type not in {"text/html", "application/xhtml+xml"}:
- return None
-
- return endpoint_from_html(response, read_target_body(response))
-
-
-def post_webmention(
- client: httpx.Client, *, endpoint: str, source: str, target: str
-) -> tuple[int, str | None]:
- """
- Send a Webmention to a discovered endpoint.
-
- :param client: HTTP client to use for the request.
- :param endpoint: Webmention endpoint URL.
- :param source: Source URL of the Webmention.
- :param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
- """
- with client.stream(
- "POST", endpoint, data={"source": source, "target": target}
- ) as response:
- status_url = None
-
- if response.status_code == 201 and (
- location := response.headers.get("Location")
- ):
- try:
- candidate = urljoin(str(response.url), location)
- except ValueError:
- # A malformed Location must not fail an otherwise successful send.
- pass
- else:
- if is_http_url(candidate):
- status_url = candidate
-
- return (response.status_code, status_url)
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_http_url,
+ is_temporary_status,
+)
+from .discovery import (
+ PermanentSenderError,
+ TemporarySenderError,
+ discover_webmention_endpoint,
+ post_webmention,
+)
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
now = datetime.now(UTC)
deadline = webmention.next_retry_at
- # SQLite returns timezone-naive datetimes even for timezone-aware columns.
if deadline is not None:
aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
if aware_deadline > now:
@@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
endpoint: str | None = None
response_status: int | None = None
+ status_url: str | None = None
+ reason: str | None = None
+ terminal = True
+ temporary_failure = False
try:
if not is_http_url(target):
raise PermanentSenderError(f"Invalid target URL: {target!r}")
- with httpx.Client(
- headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")},
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client:
+ with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.UNSUPPORTED
- webmention.failure_reason = "No Webmention endpoint discovered"
- webmention.endpoint = None
- webmention.response_status = None
-
- db.session.commit()
- return
-
- (response_status, status_url) = post_webmention(
- client, endpoint=endpoint, source=source, target=target
- )
-
- if not (200 <= response_status <= 299):
- message = f"Webmention endpoint returned HTTP {response_status}"
-
- if temporary_http_status(response_status):
- raise TemporarySenderError(message)
-
- raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.UNSUPPORTED
+ reason = "No Webmention endpoint discovered"
+ else:
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.SENT
except (
PermanentSenderError,
@@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
ValueError,
UnicodeError,
) as exc:
- # Malformed URLs and local request/protocol errors cannot be repaired
- # by retrying this source revision. InvalidURL is not a RequestError;
- # protocol errors are, so handle them before the retryable branch.
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or type(exc).__name__
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
- return
-
+ # Invalid URLs and local protocol errors cannot improve with retries.
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or type(exc).__name__
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or "Webmention request failed"
- if attempt >= max_attempts:
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- current_app.logger.warning(
- "Webmention %s exhausted %d send attempts: %s",
- webmention_uuid,
- attempt,
- webmention.failure_reason,
- )
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
-
- # No Huey retries: only the scanner may requeue after next_retry_at.
- return
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or "Webmention request failed"
+ terminal = attempt >= max_attempts
+ temporary_failure = True
+ # A scanner may have reopened or superseded the revision during I/O.
+ # Refresh only once, and persist one outcome atomically with its metadata.
if not attempt_is_current(webmention, revision):
return
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.sent_revision = revision
-
- webmention.status = SentWebmentionStatus.SENT
- webmention.failure_reason = None
-
+ webmention.status = outcome
+ webmention.failure_reason = reason
webmention.endpoint = endpoint
webmention.response_status = response_status
- webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
- )
- webmention.last_sent_at = datetime.now(UTC)
+ if terminal:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+
+ if outcome == SentWebmentionStatus.SENT:
+ webmention.sent_revision = revision
+ webmention.status_url = (
+ status_url if status_url is not None and is_http_url(status_url) else None
+ )
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+
+ if temporary_failure and not terminal:
+ # Huey does not retry: scanner requeues after next_retry_at.
+ current_app.logger.info(
+ "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ )
+ elif temporary_failure and terminal:
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ reason,
+ )
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
new file mode 100644
index 0000000..cb15ab6
--- /dev/null
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -0,0 +1,390 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Extract canonical source URLs, content fingerprints, and outgoing links."""
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from itertools import chain
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urldefrag, urljoin, urlsplit
+
+import mf2py
+import xxhash
+from bs4 import BeautifulSoup, Tag
+from bs4.element import NavigableString, PageElement
+
+from ..net import is_http_url
+
+REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
+
+class SourceScanError(Exception):
+ """A source document cannot safely be processed."""
+
+
+@dataclass(frozen=True)
+class ScannedSource:
+ path: str
+ url: str
+ content_hash: str
+ targets: frozenset[str]
+
+
+def source_url_for_path(relative_path: Path, base_url: str) -> str:
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
+ directory = relative_path.parent.as_posix()
+
+ return urljoin(base_url, f"{quote(directory, safe='/')}/")
+
+
+def canonical_url(
+ document: BeautifulSoup, *, relative_path: Path, base_url: str | None
+) -> str | None:
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
+ if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
+ href = link.get("href")
+
+ if not isinstance(href, str) or not (href := href.strip()):
+ raise SourceScanError("Document contains an empty canonical URL")
+
+ if is_http_url(href):
+ return href
+
+ if base_url is None:
+ raise SourceScanError(
+ "Document contains a relative canonical URL, but "
+ "SOURCE_BASE_URL is not configured"
+ )
+
+ resolved = urljoin(source_url_for_path(relative_path, base_url), href)
+
+ if not is_http_url(resolved):
+ raise SourceScanError(
+ f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
+ )
+
+ return resolved
+
+
+def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
+ properties = entry.get("properties")
+
+ if not isinstance(properties, dict):
+ return
+
+ values = properties.get(property_name)
+
+ if not isinstance(values, list):
+ return
+
+ yield from (value for value in values if isinstance(value, str))
+
+
+def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
+ parsed = mf2py.parse(doc=str(element), url=base_url)
+
+ if not isinstance(parsed, dict):
+ raise SourceScanError("Microformats parser did not return a document object")
+
+ items = parsed.get("items")
+
+ if not isinstance(items, list) or len(items) != 1:
+ raise SourceScanError("Expected exactly one parsed h-entry")
+
+ entry = items[0]
+
+ if not isinstance(entry, dict):
+ raise SourceScanError("Parsed h-entry is not an object")
+
+ types = entry.get("type")
+
+ if not isinstance(types, list) or "h-entry" not in types:
+ raise SourceScanError("Parsed microformats item is not an h-entry")
+
+ return entry
+
+
+def primary_entry(
+ document: BeautifulSoup, source_url: str
+) -> tuple[Tag, dict[str, Any]]:
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
+ entries = document.find_all(class_="h-entry")
+
+ if len(entries) != 1:
+ raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
+
+ entry_element = entries[0]
+ mf2_entry = parse_entry(entry_element, source_url)
+
+ urls = tuple(property_urls(mf2_entry, "url"))
+
+ if source_url not in urls:
+ raise SourceScanError(
+ f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
+ )
+
+ return entry_element, mf2_entry
+
+
+def content_element(entry: Tag) -> Tag:
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
+ contents = entry.find_all(class_="e-content")
+
+ if len(contents) != 1:
+ raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
+
+ return contents[0]
+
+
+def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
+ value = value.strip()
+
+ if not value:
+ return None
+
+ target = urljoin(base_url, value)
+
+ if not is_http_url(target):
+ return None
+
+ if urldefrag(target)[0] == urldefrag(source_url)[0]:
+ return None
+
+ return target
+
+
+def iter_targets(
+ content: Tag,
+ mf2_entry: dict[str, Any],
+ *,
+ base_url: str,
+ source_url: str,
+ ignored_hostnames: Iterable[str] = (),
+) -> Iterator[str]:
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
+ hrefs = (
+ href
+ for element in content.find_all("a", href=True)
+ if isinstance(href := element.get("href"), str)
+ )
+
+ reactions = chain.from_iterable(
+ property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ )
+
+ for value in chain(hrefs, reactions):
+ target = normalize_target(value, base_url=base_url, source_url=source_url)
+ if target is not None:
+ hostname = (urlsplit(target).hostname or "").rstrip(".")
+ if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
+ yield target
+
+
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
+def scan_source_file(
+ path: Path,
+ *,
+ root: Path,
+ base_url: str | None,
+ ignored_hostnames: Iterable[str] = (),
+) -> ScannedSource:
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
+
+ relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
+
+ source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+
+ if source_url is None:
+ if base_url is None:
+ raise SourceScanError(
+ "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ )
+
+ source_url = source_url_for_path(relative_path, base_url)
+
+ entry_element, mf2_entry = primary_entry(document, source_url)
+
+ content = content_element(entry_element)
+
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+ return ScannedSource(
+ path=relative_path.as_posix(),
+ url=source_url,
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
+ )
diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html
index 1ff3690..3dbb896 100644
--- a/webmentions_ssg/templates/received.html
+++ b/webmentions_ssg/templates/received.html
@@ -6,6 +6,33 @@ SPDX-License-Identifier: BSD-3-Clause
{% extends "base.html" %}
{% from "bootstrap5/pagination.html" import render_pagination %}
+{% set received_status_presentation = {
+ "received": {
+ "row_class": "table-info",
+ "badge_class": "text-bg-info",
+ "label": "Pending"
+ },
+ "verifying": {
+ "row_class": "table-warning",
+ "badge_class": "text-bg-warning",
+ "label": "Verifying"
+ },
+ "verified": {
+ "row_class": "table-success",
+ "badge_class": "text-bg-success",
+ "label": "Verified"
+ },
+ "failed": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Declined"
+ },
+ "deleted": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Declined"
+ }
+} %}
{% block content %}
<div class="d-flex justify-content-between align-items-center mb-3">
@@ -27,27 +54,9 @@ SPDX-License-Identifier: BSD-3-Clause
</thead>
<tbody>
{% for webmention in webmentions %}
- {% if webmention.status == "received" %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% set display_status = "Pending" %}
- {% elif webmention.status == "verifying" %}
- {% set row_class = "table-warning" %}
- {% set badge_class = "text-bg-warning" %}
- {% set display_status = "Verifying" %}
- {% elif webmention.status == "verified" %}
- {% set row_class = "table-success" %}
- {% set badge_class = "text-bg-success" %}
- {% set display_status = "Verified" %}
- {% else %}
- {% set row_class = "table-danger" %}
- {% set badge_class = "text-bg-danger" %}
- {% set display_status = "Declined" %}
- {% endif %}
- <tr class="{{ row_class }}">
- <td>
- <span class="badge {{ badge_class }}">{{ display_status }}</span>
- </td>
+ {% set style = received_status_presentation.get(webmention.status, {"row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined"}) %}
+ <tr class="{{ style.row_class }}">
+ <td><span class="badge {{ style.badge_class }}">{{ style.label }}</span></td>
<td class="text-break">
<a href="{{ webmention.source }}">{{ webmention.source }}</a>
</td>
diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html
index 689c651..974c7b3 100644
--- a/webmentions_ssg/templates/sent_source.html
+++ b/webmentions_ssg/templates/sent_source.html
@@ -6,6 +6,23 @@ SPDX-License-Identifier: BSD-3-Clause
{% extends "base.html" %}
{% from "bootstrap5/pagination.html" import render_pagination %}
+{% set sent_status_presentation = {
+ "sent": {
+ "row_class": "table-success",
+ "badge_class": "text-bg-success",
+ "label": "Sent"
+ },
+ "unsupported": {
+ "row_class": "table-warning",
+ "badge_class": "text-bg-warning",
+ "label": "Unsupported"
+ },
+ "failed": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Failed"
+ }
+} %}
{% block content %}
<div class="mb-3">
@@ -85,32 +102,13 @@ SPDX-License-Identifier: BSD-3-Clause
</thead>
<tbody>
{% for webmention in webmentions %}
- {% if webmention.status is none %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% elif webmention.status.value == "sent" %}
- {% set row_class = "table-success" %}
- {% set badge_class = "text-bg-success" %}
- {% elif webmention.status.value == "unsupported" %}
- {% set row_class = "table-warning" %}
- {% set badge_class = "text-bg-warning" %}
- {% elif webmention.status.value == "failed" %}
- {% set row_class = "table-danger" %}
- {% set badge_class = "text-bg-danger" %}
- {% else %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% endif %}
- <tr class="{{ row_class }}">
+ {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %}
+ <tr class="{{ style.row_class }}">
<td class="text-break">
<a href="{{ webmention.target }}">{{ webmention.target }}</a>
</td>
<td>
- {% if webmention.status is none %}
- <span class="badge {{ badge_class }}">Pending</span>
- {% else %}
- <span class="badge {{ badge_class }}">{{ webmention.status.value.title() }}</span>
- {% endif %}
+ <span class="badge {{ style.badge_class }}">{{ style.label }}</span>
{% if webmention.failure_reason is not none %}
<div class="small mt-2">{{ webmention.failure_reason }}</div>
{% endif %}
diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py
deleted file mode 100644
index 256d037..0000000
--- a/webmentions_ssg/url_digest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Portable SHA-256 fingerprints for indexed Webmention URLs."""
-
-import hashlib
-import json
-
-
-def received_url_digest(source: str, target: str) -> str:
- """Fingerprint an unambiguous, ordered source/target pair."""
- encoded = json.dumps(
- [source, target], ensure_ascii=False, separators=(",", ":")
- ).encode("utf-8")
- return hashlib.sha256(encoded).hexdigest()
-
-
-def sent_target_digest(target: str) -> str:
- """Fingerprint an outgoing target (unique within each source ID)."""
- return hashlib.sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
deleted file mode 100644
index c4324e0..0000000
--- a/webmentions_ssg/url_security.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Validate URL schemes and publicly routable network destinations."""
-
-import ipaddress
-from urllib.parse import urlsplit
-
-import dns.exception
-import dns.resolver
-
-
-class AddressResolutionError(Exception):
- """A hostname could not be resolved (potentially transient)."""
-
-
-class UnsafeAddressError(ValueError):
- """A hostname or IP literal identifies a non-public network address."""
-
-
-_NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
-_IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
-
-
-def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
- """Reject non-public addresses, including IPv4 hidden in IPv6 forms.
-
- ``is_global`` alone does not reliably exclude IPv6 transition addresses
- across Python versions or networks with NAT64/6to4 routing.
- """
- if isinstance(address, ipaddress.IPv6Address):
- # The well-known NAT64 prefix can translate an otherwise global IPv6
- # address into a private IPv4 destination. IPv4-compatible addresses
- # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
- if address in _NAT64_PREFIX or address in _IPV4_COMPATIBLE_PREFIX:
- return False
-
- # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
- # destination through ipaddress; validate that destination as well.
- for embedded in (address.sixtofour, address.ipv4_mapped):
- if embedded is not None and not embedded.is_global:
- return False
-
- return address.is_global
-
-
-def public_addresses(
- hostname: str, *, lifetime: float | None = None
-) -> tuple[str, ...]:
- """Resolve *once* and return only checked, globally routable IP addresses.
-
- A DNS name with a mixture of public and private answers is rejected in its
- entirety. IP literals are handled without DNS, including IPv6 literals.
- The returned strings must be used for the actual TCP connection; checking
- them and then connecting to ``hostname`` reintroduces DNS rebinding.
- """
- try:
- literal = ipaddress.ip_address(hostname)
- except ValueError:
- literal = None
-
- if literal is not None:
- addresses = (literal,)
- else:
- try:
- answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
- addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
- except (dns.exception.DNSException, ValueError) as exc:
- raise AddressResolutionError(
- f"Could not resolve hostname {hostname!r}"
- ) from exc
-
- if not addresses:
- raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
-
- if any(not is_public_address(address) for address in addresses):
- raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
-
- return tuple(str(address) for address in addresses)
-
-
-def is_public_url(url: str) -> bool:
- """Return whether the URL resolves entirely to public IP addresses.
-
- This is useful for diagnostics, but **not** sufficient to authorize a
- request: network connections must use the exact validated IP address.
- """
- hostname = urlsplit(url).hostname
-
- if hostname is None:
- raise ValueError("No hostname was specified")
-
- try:
- public_addresses(hostname)
- except UnsafeAddressError:
- return False
- return True
-
-
-def is_http_url(url: str) -> bool:
- """
- Check whether a URL is an absolute HTTP or HTTPS URL.
-
- :param url: URL to validate.
- :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
- """
- try:
- parsed = urlsplit(url)
- except ValueError:
- return False
-
- return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
-
-
-def is_safe_next_url(next_url: str) -> bool:
- """Allow redirects only to paths on this site."""
- # Check the original string too: urlsplit("///evil.example").path is
- # "/evil.example", even though browsers can treat the original as external.
- if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"):
- return False
-
- try:
- target = urlsplit(next_url)
- except ValueError:
- return False
-
- return (
- not target.scheme
- and not target.netloc
- and target.path.startswith("/")
- and not target.path.startswith("//")
- )
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
deleted file mode 100644
index 5b563f0..0000000
--- a/webmentions_ssg/views.py
+++ /dev/null
@@ -1,454 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Provide routes for authentication and Webmention administration."""
-
-import hashlib
-import secrets
-import uuid
-from datetime import UTC, datetime, timedelta
-from typing import Any, cast
-
-import sqlalchemy as sa
-import sqlalchemy.exc as sa_exc
-from flask import (
- Blueprint,
- Response,
- abort,
- current_app,
- flash,
- redirect,
- render_template,
- request,
- url_for,
-)
-from flask.typing import ResponseReturnValue
-from flask_limiter.util import get_remote_address
-from flask_login import current_user, login_required, login_user, logout_user
-from sqlalchemy.engine import CursorResult
-from sqlalchemy.orm import selectinload
-
-from . import CSRF, LIMITER, forms
-from . import DATABASE as db
-from .models import ReceivedWebmention, SentWebmention, Source, User
-from .tasks.receiver import verify_webmention
-from .tasks.scanner import manual_scan_sources
-from .tasks.sender import send_webmention
-from .url_digest import received_url_digest
-from .url_security import is_safe_next_url
-
-root_page = Blueprint("root", __name__)
-
-DUMMY_USER = User(username=secrets.token_urlsafe())
-DUMMY_USER.password = secrets.token_urlsafe()
-
-
-def login_username_limit_key() -> str:
- """Group submitted usernames across IPs without storing raw account names."""
- username = request.form.get("username", "").strip().casefold()
- if not username:
- # A missing username must not put all clients into one global bucket.
- return f"missing-username:{get_remote_address()}"
- digest = hashlib.sha256(username.encode("utf-8")).hexdigest()
- return f"username:{digest}"
-
-
-@root_page.route("/")
-def index() -> ResponseReturnValue:
- """
- Redirect to the received Webmentions view.
-
- :return: Redirect response to the received Webmentions view.
- """
- return redirect(url_for("root.received"))
-
-
-@root_page.route("/login", methods=["GET", "POST"])
-@LIMITER.limit(
- lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"],
- key_func=login_username_limit_key,
- methods=["POST"],
- error_message="Too many login attempts. Please try again later.",
-)
-@LIMITER.limit(
- lambda: current_app.config["LOGIN_RATE_LIMIT"],
- methods=["POST"],
- error_message="Too many login attempts. Please try again later.",
-)
-def login() -> ResponseReturnValue:
- """
- Authenticate a user and start a login session.
-
- Authenticated users are redirected to the application index. After a
- successful login, the user is redirected to the requested local URL when
- provided.
-
- :return: Rendered login page or redirect response.
- """
- if current_user.is_authenticated:
- return redirect(url_for("root.index"))
-
- form = forms.LoginForm()
-
- if form.validate_on_submit():
- user = db.session.scalar(
- sa.select(User).where(User.username == form.username.data)
- )
-
- user_exists = user is not None
-
- if user is None:
- user = DUMMY_USER
-
- if (
- form.password.data is None
- or not user.check_password(form.password.data)
- or not user_exists
- ):
- flash("Invalid username or password", "danger")
- return redirect(url_for("root.login"))
- login_user(user)
-
- next_url = request.args.get("next")
- if not next_url or not is_safe_next_url(next_url):
- next_url = url_for("root.index")
- return redirect(next_url)
-
- return render_template("login.html", form=form)
-
-
-@root_page.route("/logout", methods=("POST",))
-@login_required
-def logout() -> ResponseReturnValue:
- """
- End the current user's login session.
-
- :return: Redirect response to the application index.
- """
- logout_user()
- return redirect(url_for("root.index"))
-
-
-@root_page.route("/received")
-@login_required
-def received() -> ResponseReturnValue:
- """
- Display received Webmentions.
-
- :return: Rendered page containing the paginated received Webmentions.
- """
- webmentions = db.paginate(
- sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()),
- per_page=25,
- )
-
- return render_template(
- "received.html",
- title="Received Webmentions",
- webmentions=webmentions,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/received/<uuid:identifier>/delete")
-@login_required
-def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Delete a received Webmention.
-
- :param identifier: Identifier of the Webmention to delete.
- :return: Redirect response to the received Webmentions view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- db.session.delete(webmention)
- db.session.commit()
-
- flash(f"Webmention {webmention.uuid} deleted.", "success")
-
- return redirect(
- url_for("root.received", page=request.args.get("page", 1, type=int))
- )
-
-
-@root_page.post("/received/<uuid:identifier>/reverify")
-@login_required
-def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Queue a received Webmention for reverification.
-
- The Webmention status is reset before a new verification task is queued.
-
- :param identifier: Identifier of the Webmention to reverify.
- :return: Redirect response to the received Webmentions view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- webmention.status = "received"
- webmention.failure_reason = None
- webmention.recovery_attempts = 0
-
- db.session.commit()
-
- verify_webmention(identifier)
-
- flash(f"Webmention {webmention.uuid} queued for reverification.", "success")
-
- return redirect(
- url_for("root.received", page=request.args.get("page", 1, type=int))
- )
-
-
-@root_page.route("/sent")
-@login_required
-def sent() -> ResponseReturnValue:
- """
- Display sources with sent Webmentions.
-
- :return: Rendered page containing the paginated source list.
- """
- sources = db.paginate(
- sa.select(Source)
- .options(selectinload(Source.sent_webmentions))
- .order_by(Source.last_seen_at.desc()),
- per_page=25,
- )
- return render_template(
- "sent.html",
- title="Sent Webmentions",
- sources=sources,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/sent/rescan")
-@login_required
-def rescan_sent_sources() -> ResponseReturnValue:
- """
- Queue a manual scan of sent Webmention sources.
-
- :return: Redirect response to the sent Webmentions view.
- """
- form = forms.AdminActionForm()
- if form.validate_on_submit():
- manual_scan_sources()
- flash("Source rescan queued.", "success")
- return redirect(url_for("root.sent"))
-
-
-@root_page.route("/sent/<uuid:source_identifier>")
-@login_required
-def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Display the sent Webmentions associated with a source.
-
- :param source_identifier: Identifier of the source to display.
- :return: Rendered source details page.
- """
- source = db.session.scalar(
- sa.select(Source).where(Source.uuid == source_identifier)
- )
-
- if source is None:
- return abort(404)
-
- webmentions = db.paginate(
- sa.select(SentWebmention)
- .where(SentWebmention.source_id == source.uuid)
- .order_by(SentWebmention.target, SentWebmention.uuid),
- per_page=25,
- )
-
- return render_template(
- "sent_source.html",
- title="Sent Webmentions",
- source=source,
- webmentions=webmentions,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend")
-@login_required
-def resend_sent_webmention(
- source_identifier: uuid.UUID, webmention_identifier: uuid.UUID
-) -> ResponseReturnValue:
- """
- Queue an outgoing Webmention to be sent again.
-
- The current desired source revision is reopened for processing so the
- sender task does not treat the Webmention as already completed. Historical
- successful-send fields are preserved, while result fields from the previous
- attempt are cleared.
-
- :param source_identifier: Identifier of the source owning the Webmention.
- :param webmention_identifier: Identifier of the sent Webmention to resend.
- :return: Redirect response to the source details view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(SentWebmention, webmention_identifier)
-
- if webmention is None or webmention.source_id != source_identifier:
- return abort(404)
-
- # Reopen the existing desired revision instead of inventing a new source
- # revision. send_webmention() otherwise exits when this revision has already
- # been processed.
- webmention.processed_revision = None
- webmention.attempt_count = 0
- webmention.next_retry_at = None
- webmention.status = None
- webmention.failure_reason = None
- webmention.endpoint = None
- webmention.response_status = None
- webmention.status_url = None
-
- db.session.commit()
-
- send_webmention(webmention_identifier)
-
- flash(f"Webmention {webmention.uuid} queued for resend.", "success")
-
- return redirect(url_for("root.sent_source", source_identifier=source_identifier))
-
-
-@root_page.post("/endpoint")
-@LIMITER.limit(
- lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"],
- key_func=lambda: "all-clients",
-)
-@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"])
-@CSRF.exempt
-def endpoint() -> ResponseReturnValue:
- """
- Receive and queue a Webmention for verification.
-
- Existing Webmentions with the same source and target are reset for
- reverification. Concurrent insertion of the same Webmention is handled by
- retrieving and updating the row created by the competing request.
-
- :return: HTTP 201 response with the Webmention status URL in the
- ``Location`` header, or validation errors with HTTP 400.
- """
- form = forms.EndpointForm(meta={"csrf": False})
-
- if not form.validate_on_submit():
- return form.errors, 400
-
- source = cast(str, form.source.data)
- target = cast(str, form.target.data)
-
- url_digest = received_url_digest(source, target)
- webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
- ).scalar_one_or_none()
-
- # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
- if webmention is not None and (webmention.source, webmention.target) != (
- source,
- target,
- ):
- return {"error": "Webmention URL fingerprint conflict"}, 409
-
- should_verify = False
-
- if webmention is None:
- identifier = uuid.uuid7()
-
- webmention = ReceivedWebmention(uuid=identifier, source=source, target=target)
-
- db.session.add(webmention)
-
- try:
- db.session.commit()
- should_verify = True
- except sa_exc.IntegrityError:
- db.session.rollback()
-
- # Another request may have inserted the same source/target pair
- # after our SELECT but before our COMMIT.
- webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(
- ReceivedWebmention.url_digest == url_digest
- )
- ).scalar_one_or_none()
- if webmention is None or (webmention.source, webmention.target) != (
- source,
- target,
- ):
- return {"error": "Webmention URL fingerprint conflict"}, 409
-
- else:
- now = datetime.now(UTC)
- cutoff = now - timedelta(
- seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
- )
-
- # Atomic conditional update: only one concurrent request can reopen an expired verification.
- result = cast(
- CursorResult[Any],
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention.uuid,
- ReceivedWebmention.updated_at <= cutoff,
- )
- .values(
- status="received",
- failure_reason=None,
- recovery_attempts=0,
- updated_at=now,
- )
- .execution_options(synchronize_session=False)
- ),
- )
- should_verify = result.rowcount == 1
- db.session.commit()
-
- identifier = webmention.uuid
-
- if should_verify:
- verify_webmention(identifier)
-
- status_url = url_for("root.status", identifier=str(identifier), _external=True)
-
- return Response(status=201, headers={"Location": status_url})
-
-
-@root_page.route("/status/<uuid:identifier>")
-def status(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Display the verification status of a received Webmention.
-
- :param identifier: Identifier of the Webmention to display.
- :return: Rendered Webmention status page.
- """
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- return render_template(
- "status.html", title="Webmention status", webmention=webmention
- )
diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py
new file mode 100644
index 0000000..91966d4
--- /dev/null
+++ b/webmentions_ssg/views/__init__.py
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Register all views under the historic root Blueprint."""
+
+from flask import Blueprint
+
+root_page = Blueprint("root", __name__)
+
+
+def register_routes() -> Blueprint:
+ """Import task-backed view handlers after Huey initialization and return the Blueprint."""
+ from . import admin, endpoint, session # noqa: F401
+
+ return root_page
diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py
new file mode 100644
index 0000000..46e3b18
--- /dev/null
+++ b/webmentions_ssg/views/admin.py
@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Authenticated incoming and outgoing Webmention management."""
+
+import uuid
+
+import sqlalchemy as sa
+from flask import abort, flash, redirect, render_template, request, url_for
+from flask.typing import ResponseReturnValue
+from flask_login import login_required
+from sqlalchemy.orm import selectinload
+
+from .. import forms
+from ..extensions import DATABASE as db
+from ..models import ReceivedWebmention, SentWebmention, Source
+from ..tasks.receiver import verify_webmention
+from ..tasks.scanner import manual_scan_sources
+from ..tasks.sender import send_webmention
+from . import root_page
+
+
+@root_page.route("/received")
+@login_required
+def received() -> ResponseReturnValue:
+ """
+ Display received Webmentions.
+
+ :return: Rendered page containing the paginated received Webmentions.
+ """
+ webmentions = db.paginate(
+ sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()),
+ per_page=25,
+ )
+
+ return render_template(
+ "received.html",
+ title="Received Webmentions",
+ webmentions=webmentions,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/received/<uuid:identifier>/delete")
+@login_required
+def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Delete a received Webmention.
+
+ :param identifier: Identifier of the Webmention to delete.
+ :return: Redirect response to the received Webmentions view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ db.session.delete(webmention)
+ db.session.commit()
+
+ flash(f"Webmention {webmention.uuid} deleted.", "success")
+
+ return redirect(
+ url_for("root.received", page=request.args.get("page", 1, type=int))
+ )
+
+
+@root_page.post("/received/<uuid:identifier>/reverify")
+@login_required
+def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Queue a received Webmention for reverification.
+
+ The Webmention status is reset before a new verification task is queued.
+
+ :param identifier: Identifier of the Webmention to reverify.
+ :return: Redirect response to the received Webmentions view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ webmention = webmention.reset_for_verification()
+
+ db.session.commit()
+
+ verify_webmention(identifier)
+
+ flash(f"Webmention {webmention.uuid} queued for reverification.", "success")
+
+ return redirect(
+ url_for("root.received", page=request.args.get("page", 1, type=int))
+ )
+
+
+@root_page.route("/sent")
+@login_required
+def sent() -> ResponseReturnValue:
+ """
+ Display sources with sent Webmentions.
+
+ :return: Rendered page containing the paginated source list.
+ """
+ sources = db.paginate(
+ sa.select(Source)
+ .options(selectinload(Source.sent_webmentions))
+ .order_by(Source.last_seen_at.desc()),
+ per_page=25,
+ )
+ return render_template(
+ "sent.html",
+ title="Sent Webmentions",
+ sources=sources,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/sent/rescan")
+@login_required
+def rescan_sent_sources() -> ResponseReturnValue:
+ """
+ Queue a manual scan of sent Webmention sources.
+
+ :return: Redirect response to the sent Webmentions view.
+ """
+ form = forms.AdminActionForm()
+ if form.validate_on_submit():
+ manual_scan_sources()
+ flash("Source rescan queued.", "success")
+ return redirect(url_for("root.sent"))
+
+
+@root_page.route("/sent/<uuid:source_identifier>")
+@login_required
+def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Display the sent Webmentions associated with a source.
+
+ :param source_identifier: Identifier of the source to display.
+ :return: Rendered source details page.
+ """
+ source = db.session.scalar(
+ sa.select(Source).where(Source.uuid == source_identifier)
+ )
+
+ if source is None:
+ return abort(404)
+
+ webmentions = db.paginate(
+ sa.select(SentWebmention)
+ .where(SentWebmention.source_id == source.uuid)
+ .order_by(SentWebmention.target, SentWebmention.uuid),
+ per_page=25,
+ )
+
+ return render_template(
+ "sent_source.html",
+ title="Sent Webmentions",
+ source=source,
+ webmentions=webmentions,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend")
+@login_required
+def resend_sent_webmention(
+ source_identifier: uuid.UUID, webmention_identifier: uuid.UUID
+) -> ResponseReturnValue:
+ """
+ Queue an outgoing Webmention to be sent again.
+
+ The current desired source revision is reopened for processing so the
+ sender task does not treat the Webmention as already completed. Historical
+ successful-send fields are preserved, while result fields from the previous
+ attempt are cleared.
+
+ :param source_identifier: Identifier of the source owning the Webmention.
+ :param webmention_identifier: Identifier of the sent Webmention to resend.
+ :return: Redirect response to the source details view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+
+ if webmention is None or webmention.source_id != source_identifier:
+ return abort(404)
+
+ # Reopen the existing desired revision instead of inventing a new source
+ # revision. send_webmention() otherwise exits when this revision has already
+ # been processed.
+ webmention = webmention.reopen(
+ webmention.desired_revision, webmention.active, resend=True
+ )
+
+ db.session.commit()
+
+ send_webmention(webmention_identifier)
+
+ flash(f"Webmention {webmention.uuid} queued for resend.", "success")
+
+ return redirect(url_for("root.sent_source", source_identifier=source_identifier))
diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py
new file mode 100644
index 0000000..1c2fe13
--- /dev/null
+++ b/webmentions_ssg/views/endpoint.py
@@ -0,0 +1,150 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Public Webmention endpoint and verification status view."""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+
+import sqlalchemy as sa
+import sqlalchemy.exc as sa_exc
+from flask import Response, abort, current_app, render_template, url_for
+from flask.typing import ResponseReturnValue
+from sqlalchemy.engine import CursorResult
+
+from .. import forms
+from ..extensions import CSRF, LIMITER
+from ..extensions import DATABASE as db
+from ..models import ReceivedWebmention, received_url_digest
+from ..tasks.receiver import verify_webmention
+from . import root_page
+
+
+def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None:
+ """Upsert a submission atomically and return (identifier, enqueue flag).
+
+ ``None`` means that the URL fingerprint matches a different source/target.
+ No background task is enqueued until the transaction is committed.
+ """
+ url_digest = received_url_digest(source, target)
+ webmention = db.session.execute(
+ sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
+ ).scalar_one_or_none()
+
+ # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
+ if webmention is not None and (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return None
+
+ should_verify = False
+
+ if webmention is None:
+ identifier = uuid.uuid7()
+ webmention = ReceivedWebmention(uuid=identifier, source=source, target=target)
+ db.session.add(webmention)
+
+ try:
+ db.session.commit()
+ should_verify = True
+ except sa_exc.IntegrityError:
+ db.session.rollback()
+
+ # Another request may have inserted the same source/target pair
+ # after our SELECT but before our COMMIT.
+ webmention = db.session.execute(
+ sa.select(ReceivedWebmention).where(
+ ReceivedWebmention.url_digest == url_digest
+ )
+ ).scalar_one_or_none()
+ if webmention is None or (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return None
+
+ else:
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(
+ seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
+ )
+
+ # Atomic conditional update: only one concurrent request can reopen an expired verification.
+ result = cast(
+ CursorResult[Any],
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention.uuid,
+ ReceivedWebmention.updated_at <= cutoff,
+ )
+ .values(ReceivedWebmention.verification_reset_values(at=now))
+ .execution_options(synchronize_session=False)
+ ),
+ )
+ should_verify = result.rowcount == 1
+ db.session.commit()
+
+ return webmention.uuid, should_verify
+
+
+@root_page.post("/endpoint")
+@LIMITER.limit(
+ lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"],
+ key_func=lambda: "all-clients",
+)
+@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"])
+@CSRF.exempt
+def endpoint() -> ResponseReturnValue:
+ """
+ Receive and queue a Webmention for verification.
+
+ Existing Webmentions with the same source and target are reset for
+ reverification. Concurrent insertion of the same Webmention is handled by
+ retrieving and updating the row created by the competing request.
+
+ :return: HTTP 201 response with the Webmention status URL in the
+ ``Location`` header, or validation errors with HTTP 400.
+ """
+ form = forms.EndpointForm(meta={"csrf": False})
+
+ if not form.validate_on_submit():
+ return form.errors, 400
+
+ source = cast(str, form.source.data)
+ target = cast(str, form.target.data)
+
+ result = submit_received(source, target)
+
+ if result is None:
+ return {"error": "Webmention URL fingerprint conflict"}, 409
+
+ identifier, should_verify = result
+
+ if should_verify:
+ verify_webmention(identifier)
+
+ status_url = url_for("root.status", identifier=str(identifier), _external=True)
+
+ return Response(status=201, headers={"Location": status_url})
+
+
+@root_page.route("/status/<uuid:identifier>")
+def status(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Display the verification status of a received Webmention.
+
+ :param identifier: Identifier of the Webmention to display.
+ :return: Rendered Webmention status page.
+ """
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ return render_template(
+ "status.html", title="Webmention status", webmention=webmention
+ )
diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py
new file mode 100644
index 0000000..7bd8a10
--- /dev/null
+++ b/webmentions_ssg/views/session.py
@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Authentication and session routes."""
+
+import sqlalchemy as sa
+from flask import current_app, flash, redirect, render_template, request, url_for
+from flask.typing import ResponseReturnValue
+from flask_login import current_user, login_required, login_user, logout_user
+
+from .. import forms
+from ..auth import DUMMY_USER, is_safe_next_url, login_username_limit_key
+from ..extensions import DATABASE as db
+from ..extensions import LIMITER
+from ..models import User
+from . import root_page
+
+
+@root_page.route("/")
+def index() -> ResponseReturnValue:
+ """
+ Redirect to the received Webmentions view.
+
+ :return: Redirect response to the received Webmentions view.
+ """
+ return redirect(url_for("root.received"))
+
+
+@root_page.route("/login", methods=["GET", "POST"])
+@LIMITER.limit(
+ lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"],
+ key_func=login_username_limit_key,
+ methods=["POST"],
+ error_message="Too many login attempts. Please try again later.",
+)
+@LIMITER.limit(
+ lambda: current_app.config["LOGIN_RATE_LIMIT"],
+ methods=["POST"],
+ error_message="Too many login attempts. Please try again later.",
+)
+def login() -> ResponseReturnValue:
+ """
+ Authenticate a user and start a login session.
+
+ Authenticated users are redirected to the application index. After a
+ successful login, the user is redirected to the requested local URL when
+ provided.
+
+ :return: Rendered login page or redirect response.
+ """
+ if current_user.is_authenticated:
+ return redirect(url_for("root.index"))
+
+ form = forms.LoginForm()
+
+ if form.validate_on_submit():
+ user = db.session.scalar(
+ sa.select(User).where(User.username == form.username.data)
+ )
+
+ user_exists = user is not None
+
+ if user is None:
+ user = DUMMY_USER
+
+ if (
+ form.password.data is None
+ or not user.check_password(form.password.data)
+ or not user_exists
+ ):
+ flash("Invalid username or password", "danger")
+ return redirect(url_for("root.login"))
+ login_user(user)
+
+ next_url = request.args.get("next")
+ if not next_url or not is_safe_next_url(next_url):
+ next_url = url_for("root.index")
+ return redirect(next_url)
+
+ return render_template("login.html", form=form)
+
+
+@root_page.route("/logout", methods=("POST",))
+@login_required
+def logout() -> ResponseReturnValue:
+ """
+ End the current user's login session.
+
+ :return: Redirect response to the application index.
+ """
+ logout_user()
+ return redirect(url_for("root.index"))