diff options
Diffstat (limited to '')
67 files changed, 6795 insertions, 6750 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py deleted file mode 100644 index 7d325aa..0000000 --- a/tests/cli/test_database.py +++ /dev/null @@ -1,267 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Tests for the simplified database CLI and Flask-Migrate integration.""" - -import logging -from collections.abc import Iterator -from pathlib import Path - -import pytest -import sqlalchemy as sa -from alembic.script import ScriptDirectory -from flask import Flask -from flask_migrate import upgrade as migrate_upgrade -from sqlalchemy import inspect - -from webmentions_ssg import DATABASE, MIGRATE, create_app -from webmentions_ssg.config import TestingConfig - - -@pytest.fixture -def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]: - monkeypatch.chdir(tmp_path) - - class DatabaseTestingConfig(TestingConfig): - LOG_FILE = str(tmp_path / "database-cli.log") - SQLALCHEMY_DATABASE_URI = f"sqlite:///{tmp_path / 'database.sqlite'}" - - app = create_app(DatabaseTestingConfig) - yield app - - with app.app_context(): - DATABASE.session.remove() - DATABASE.engine.dispose() - - -def _script_directory() -> ScriptDirectory: - """Open the migration scripts configured for this application.""" - return ScriptDirectory(str(MIGRATE.directory)) - - -def _head_revision(app: Flask) -> str: - with app.app_context(): - revision = _script_directory().get_current_head() - assert revision is not None, "Expected a migration head" - return revision - - -def _database_revision(app: Flask) -> str: - with app.app_context(), DATABASE.engine.connect() as connection: - return connection.execute( - sa.text("SELECT version_num FROM alembic_version") - ).scalar_one() - - -def test_database_commands_are_registered_without_hiding_flask_migrate( - database_app: Flask, -) -> None: - runner = database_app.test_cli_runner() - - result = runner.invoke(args=["db", "--help"]) - assert result.exit_code == 0, result.output - assert "create-db" in result.output - assert "upgrade" in result.output - - result = runner.invoke(args=["migrate", "--help"]) - assert result.exit_code == 0, result.output - assert "downgrade" in result.output - assert "revision" in result.output - - -def test_create_db_builds_schema_and_records_latest_migration( - database_app: Flask, -) -> None: - result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) - - assert result.exit_code == 0, result.output - assert "Database created" in result.output - assert _database_revision(database_app) == _head_revision(database_app) - - with database_app.app_context(): - inspector = inspect(DATABASE.engine) - table_names = set(inspector.get_table_names()) - columns = { - column["name"] for column in inspector.get_columns("sent_webmentions") - } - received_columns = { - column["name"] for column in inspector.get_columns("received_webmentions") - } - received_indexes = { - index["name"] for index in inspector.get_indexes("received_webmentions") - } - assert {"alembic_version", "users", "received_webmentions"} <= table_names - assert {"attempt_count", "next_retry_at"} <= columns - assert "recovery_attempts" in received_columns - assert "ix_received_webmentions_status_updated_at" in received_indexes - - -def test_create_db_refuses_existing_database_without_modifying_it( - database_app: Flask, -) -> None: - runner = database_app.test_cli_runner() - assert runner.invoke(args=["db", "create-db"]).exit_code == 0 - revision_before = _database_revision(database_app) - - result = runner.invoke(args=["db", "create-db"]) - - assert result.exit_code != 0 - assert "already contains tables" in result.output - assert _database_revision(database_app) == revision_before - - -def test_create_db_refuses_unversioned_existing_tables(database_app: Flask) -> None: - with database_app.app_context(), DATABASE.engine.begin() as connection: - connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)")) - - result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) - - assert result.exit_code != 0 - assert "already contains tables" in result.output - with database_app.app_context(): - assert "legacy_data" in inspect(DATABASE.engine).get_table_names() - assert "alembic_version" not in inspect(DATABASE.engine).get_table_names() - - -def test_upgrade_requires_an_initialized_database(database_app: Flask) -> None: - result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) - - assert result.exit_code != 0 - assert "flask db create-db" in result.output - - -def test_upgrade_refuses_unversioned_existing_database(database_app: Flask) -> None: - with database_app.app_context(), DATABASE.engine.begin() as connection: - connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)")) - - result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) - - assert result.exit_code != 0 - assert "No Flask-Migrate revision table" in result.output - with database_app.app_context(): - assert "legacy_data" in inspect(DATABASE.engine).get_table_names() - - -def test_upgrade_applies_pending_migrations_and_preserves_data( - database_app: Flask, -) -> None: - # Start with the first migration, as if this were an older deployment. - with database_app.app_context(): - base_revision = _script_directory().get_base() - assert base_revision is not None - assert base_revision != _head_revision(database_app) - migrate_upgrade(revision=base_revision) - - with DATABASE.engine.begin() as connection: - connection.execute( - sa.text( - "INSERT INTO users (username, password_hash) " - "VALUES (:username, :password_hash)" - ), - {"username": "existing-user", "password_hash": "existing-hash"}, - ) - - result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) - - assert result.exit_code == 0, result.output - assert "Database upgraded" in result.output - assert _database_revision(database_app) == _head_revision(database_app) - - with database_app.app_context(), DATABASE.engine.connect() as connection: - assert ( - connection.execute( - sa.text("SELECT password_hash FROM users WHERE username = :username"), - {"username": "existing-user"}, - ).scalar_one() - == "existing-hash" - ) - assert "sent_webmentions" in inspect(connection).get_table_names() - - -def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None: - runner = database_app.test_cli_runner() - assert runner.invoke(args=["db", "create-db"]).exit_code == 0 - - result = runner.invoke(args=["db", "upgrade"]) - - assert result.exit_code == 0, result.output - assert _database_revision(database_app) == _head_revision(database_app) - - -def test_create_db_does_not_report_success_when_migration_fails( - database_app: Flask, monkeypatch: pytest.MonkeyPatch -) -> None: - from webmentions_ssg.cli import database as database_cli - - def fail_upgrade() -> None: - raise RuntimeError("simulated migration failure") - - monkeypatch.setattr(database_cli, "migrate_upgrade", fail_upgrade) - - result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) - - assert result.exit_code != 0 - assert isinstance(result.exception, RuntimeError) - assert "Database created" not in result.output - - -def test_create_db_preserves_logging( - database_app: Flask, caplog: pytest.LogCaptureFixture -) -> None: - with caplog.at_level(logging.WARNING): - result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) - - assert result.exit_code == 0, result.output - - database_app.logger.warning("Logging still works after migration") - - assert "Logging still works after migration" in caplog.text - - -def test_upgrade_backfills_portable_digest_for_existing_received_rows( - database_app: Flask, -) -> None: - """Upgrade a pre-fingerprint SQLite database without losing historical rows.""" - from webmentions_ssg.url_digest import received_url_digest - - with database_app.app_context(): - migrate_upgrade(revision="c0272f25ce17") - with DATABASE.engine.begin() as connection: - connection.execute( - sa.text( - "INSERT INTO received_webmentions " - "(uuid, source, target, status, updated_at) " - "VALUES (:uuid, :source, :target, :status, :updated_at)" - ), - { - "uuid": "0" * 31 + "1", - "source": "https://source.example/old", - "target": "https://dennisfink.me/target", - "status": "verified", - "updated_at": "2026-01-01 00:00:00", - }, - ) - migrate_upgrade() - with DATABASE.engine.connect() as connection: - result = connection.execute( - sa.text( - "SELECT source, target, status, url_digest " - "FROM received_webmentions" - ) - ).one() - assert result.status == "verified" - recovery_attempts = connection.execute( - sa.text("SELECT recovery_attempts FROM received_webmentions") - ).scalar_one() - assert recovery_attempts == 0 - assert result.url_digest == received_url_digest( - result.source, result.target - ) - constraints = inspect(connection).get_unique_constraints( - "received_webmentions" - ) - assert any( - constraint["column_names"] == ["url_digest"] - for constraint in constraints - ) diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py deleted file mode 100644 index cbdab8c..0000000 --- a/tests/cli/test_init.py +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Test that all application CLI command groups are exposed.""" - -from flask import Flask - - -def test_cli_groups_are_registered(app: Flask) -> None: - commands = app.cli.commands - assert {"users", "mail"}.issubset(commands) diff --git a/tests/cli/test_mail.py b/tests/cli/test_mail.py deleted file mode 100644 index 0a2afce..0000000 --- a/tests/cli/test_mail.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -from unittest.mock import patch - -from flask import Flask - - -def test_mail_cli_sends_to_configured_notification_email(app: Flask) -> None: - app.config.update( - NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" - ) - - with patch("webmentions_ssg.MAIL.send") as send: - result = app.test_cli_runner().invoke(args=["mail", "test"]) - - assert result.exit_code == 0, result.output - send.assert_called_once() - message = send.call_args.args[0] - assert message.sender == "sender@example.com" - assert message.recipients == ["owner@example.com"] - assert "test email" in message.subject.lower() - assert "outgoing mail configuration" in message.body - assert "Test email sent to owner@example.com." in result.output - - -def test_mail_cli_overrides_configured_recipient(app: Flask) -> None: - app.config.update( - NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" - ) - - with patch("webmentions_ssg.MAIL.send") as send: - result = app.test_cli_runner().invoke( - args=["mail", "test", "--to", "other@example.com"] - ) - - assert result.exit_code == 0, result.output - send.assert_called_once() - assert send.call_args.args[0].recipients == ["other@example.com"] - - -def test_mail_cli_requires_recipient(app: Flask) -> None: - app.config.update(NOTIFICATION_EMAIL=None, MAIL_DEFAULT_SENDER="sender@example.com") - - with patch("webmentions_ssg.MAIL.send") as send: - result = app.test_cli_runner().invoke(args=["mail", "test"]) - - assert result.exit_code != 0 - assert "Set NOTIFICATION_EMAIL or pass --to" in result.output - send.assert_not_called() - - -def test_mail_cli_requires_default_sender(app: Flask) -> None: - app.config.update(NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER=None) - - with patch("webmentions_ssg.MAIL.send") as send: - result = app.test_cli_runner().invoke(args=["mail", "test"]) - - assert result.exit_code != 0 - assert "MAIL_DEFAULT_SENDER is not configured" in result.output - send.assert_not_called() - - -def test_mail_cli_reports_smtp_failure(app: Flask) -> None: - app.config.update( - NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" - ) - - with patch( - "webmentions_ssg.MAIL.send", side_effect=ConnectionError("SMTP unavailable") - ) as send: - result = app.test_cli_runner().invoke(args=["mail", "test"]) - - assert result.exit_code != 0 - assert "Could not send test email: SMTP unavailable" in result.output - send.assert_called_once() diff --git a/tests/cli/test_users.py b/tests/cli/test_users.py deleted file mode 100644 index 4764c85..0000000 --- a/tests/cli/test_users.py +++ /dev/null @@ -1,320 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -from unittest.mock import patch - -import pytest -import sqlalchemy as sa -from flask import Flask -from sqlalchemy.exc import IntegrityError - -from webmentions_ssg import DATABASE -from webmentions_ssg.models import User - - -def _add_user(app: Flask, username: str, password: str = "original") -> None: - """Persist a user using the same model as the CLI.""" - with app.app_context(): - DATABASE.session.add(User(username=username, password=password)) - DATABASE.session.commit() - - -def _get_user(app: Flask, username: str) -> User | None: - with app.app_context(): - return DATABASE.session.scalar(sa.select(User).where(User.username == username)) - - -def test_users_cli_help_lists_management_commands(app: Flask) -> None: - result = app.test_cli_runner().invoke(args=["users", "--help"]) - - assert result.exit_code == 0, result.output - for command in ("create", "list", "set-password", "delete"): - assert command in result.output - - -def test_users_cli_create_with_prompts_hashes_password(app: Flask) -> None: - result = app.test_cli_runner().invoke( - args=["users", "create"], - input="alice\nmy new secret passphrase\nmy new secret passphrase\n", - ) - - assert result.exit_code == 0, result.output - assert "Created user 'alice'." in result.output - assert "my new secret passphrase" not in result.output - user = _get_user(app, "alice") - assert user is not None - assert user.check_password("my new secret passphrase") - assert user.password_hash != "my new secret passphrase" - - -def test_users_cli_create_with_username_option(app: Flask) -> None: - result = app.test_cli_runner().invoke( - args=["users", "create", "--username", "alice"], - input="my new secret passphrase\nmy new secret passphrase\n", - ) - - assert result.exit_code == 0, result.output - assert _get_user(app, "alice") is not None - - -def test_users_cli_create_rejects_duplicate(app: Flask) -> None: - _add_user(app, "alice") - - result = app.test_cli_runner().invoke( - args=[ - "users", - "create", - "--username", - "alice", - "--password", - "a stronger duplicate password", - ] - ) - - assert result.exit_code != 0 - assert "User 'alice' already exists." in result.output - user = _get_user(app, "alice") - assert user is not None - assert user.check_password("original") - - -@pytest.mark.parametrize("username", ["", " "]) -def test_users_cli_create_rejects_invalid_username(app: Flask, username: str) -> None: - result = app.test_cli_runner().invoke( - args=[ - "users", - "create", - "--username", - username, - "--password", - "a strong password for testing", - ] - ) - - assert result.exit_code != 0 - assert "Username must not" in result.output - - -def test_users_cli_create_rejects_empty_password(app: Flask) -> None: - result = app.test_cli_runner().invoke( - args=["users", "create", "--username", "alice", "--password", ""] - ) - - assert result.exit_code != 0 - assert "Password must not be empty." in result.output - assert _get_user(app, "alice") is None - - -def test_users_cli_list_empty(app: Flask) -> None: - result = app.test_cli_runner().invoke(args=["users", "list"]) - - assert result.exit_code == 0, result.output - assert "No users found." in result.output - - -def test_users_cli_list_is_sorted_and_excludes_passwords(app: Flask) -> None: - _add_user(app, "zoe") - _add_user(app, "alice") - - result = app.test_cli_runner().invoke(args=["users", "list"]) - - assert result.exit_code == 0, result.output - lines = result.output.splitlines() - assert lines[0] == "ID\tUSERNAME" - assert lines[1].endswith("\talice") - assert lines[2].endswith("\tzoe") - assert "original" not in result.output - - -def test_users_cli_set_password_updates_hash(app: Flask) -> None: - _add_user(app, "alice") - old_hash = _get_user(app, "alice").password_hash # type: ignore[union-attr] - - result = app.test_cli_runner().invoke( - args=["users", "set-password", "alice"], - input="a long replacement passphrase\na long replacement passphrase\n", - ) - - assert result.exit_code == 0, result.output - assert "Updated password for user 'alice'." in result.output - assert "a long replacement passphrase" not in result.output - user = _get_user(app, "alice") - assert user is not None - assert user.password_hash != old_hash - assert not user.check_password("original") - assert user.check_password("a long replacement passphrase") - - -def test_users_cli_set_password_missing_user(app: Flask) -> None: - result = app.test_cli_runner().invoke( - args=[ - "users", - "set-password", - "missing", - "--password", - "a long replacement passphrase", - ] - ) - - assert result.exit_code != 0 - assert "User 'missing' does not exist." in result.output - - -def test_users_cli_set_password_rejects_empty_password(app: Flask) -> None: - _add_user(app, "alice") - - result = app.test_cli_runner().invoke( - args=["users", "set-password", "alice", "--password", ""] - ) - - assert result.exit_code != 0 - assert "Password must not be empty." in result.output - user = _get_user(app, "alice") - assert user is not None - assert user.check_password("original") - - -def test_users_cli_delete_with_confirmation(app: Flask) -> None: - _add_user(app, "alice") - - result = app.test_cli_runner().invoke( - args=["users", "delete", "alice"], input="y\n" - ) - - assert result.exit_code == 0, result.output - assert "Delete user 'alice'?" in result.output - assert "Deleted user 'alice'." in result.output - assert _get_user(app, "alice") is None - - -def test_users_cli_delete_confirmation_rejected(app: Flask) -> None: - _add_user(app, "alice") - - result = app.test_cli_runner().invoke( - args=["users", "delete", "alice"], input="n\n" - ) - - assert result.exit_code != 0 - assert "Aborted" in result.output - assert _get_user(app, "alice") is not None - - -def test_users_cli_delete_yes_only_deletes_target(app: Flask) -> None: - _add_user(app, "alice") - _add_user(app, "bob") - - result = app.test_cli_runner().invoke(args=["users", "delete", "alice", "--yes"]) - - assert result.exit_code == 0, result.output - assert "Deleted user 'alice'." in result.output - assert _get_user(app, "alice") is None - assert _get_user(app, "bob") is not None - - -def test_users_cli_delete_missing_user(app: Flask) -> None: - result = app.test_cli_runner().invoke(args=["users", "delete", "missing", "-y"]) - - assert result.exit_code != 0 - assert "User 'missing' does not exist." in result.output - - -def test_users_cli_delete_missing_blank_username(app: Flask) -> None: - result = app.test_cli_runner().invoke(args=["users", "delete", " ", "--yes"]) - - assert result.exit_code != 0 - assert "User ' ' does not exist." in result.output - - -def test_users_cli_can_manage_legacy_whitespace_username(app: Flask) -> None: - _add_user(app, " alice ") - - reset = app.test_cli_runner().invoke( - args=[ - "users", - "set-password", - " alice ", - "--password", - "a long replacement passphrase", - ] - ) - - assert reset.exit_code == 0, reset.output - user = _get_user(app, " alice ") - assert user is not None - assert user.check_password("a long replacement passphrase") - - deleted = app.test_cli_runner().invoke(args=["users", "delete", " alice ", "--yes"]) - assert deleted.exit_code == 0, deleted.output - assert _get_user(app, " alice ") is None - - -def test_users_cli_create_rolls_back_on_duplicate_race(app: Flask) -> None: - database_error = IntegrityError("INSERT", {}, Exception("duplicate")) - - with app.app_context(): - with ( - patch.object(DATABASE.session, "commit", side_effect=database_error), - patch.object(DATABASE.session, "rollback") as rollback, - ): - result = app.test_cli_runner().invoke( - args=[ - "users", - "create", - "--username", - "alice", - "--password", - "a long secret for testing", - ] - ) - rollback.assert_called_once() - - assert result.exit_code != 0 - assert "User 'alice' already exists." in result.output - - -@pytest.mark.parametrize( - ("password", "error"), - [ - ("short", "at least 8 characters"), - ("good passphrase with\tab", "control characters"), - ], -) -@pytest.mark.parametrize("operation", ["create", "set-password"]) -def test_users_cli_enforces_password_policy( - app: Flask, password: str, error: str, operation: str -) -> None: - _add_user(app, "existing") - if operation == "create": - command = ["users", "create", "--username", "new"] - else: - command = ["users", "set-password", "existing"] - - result = app.test_cli_runner().invoke(args=[*command, "--password", password]) - assert result.exit_code != 0 - assert error in result.output - assert _get_user(app, "new") is None - existing = _get_user(app, "existing") - assert existing is not None and existing.check_password("original") - - -def test_users_cli_accepts_long_unicode_passphrase(app: Flask) -> None: - passphrase = "🦊 several words of my passphrase 🪁" - result = app.test_cli_runner().invoke( - args=["users", "create", "--username", "alice", "--password", passphrase] - ) - assert result.exit_code == 0, result.output - user = _get_user(app, "alice") - assert user is not None and user.check_password(passphrase) - - -@pytest.mark.parametrize("passphrase", ["12345678", " " * 8, "🦊" + "a" * 1025]) -def test_users_cli_accepts_minimum_whitespace_and_unbounded_passwords( - app: Flask, passphrase: str -) -> None: - result = app.test_cli_runner().invoke( - args=["users", "create", "--username", "alice", "--password", passphrase] - ) - assert result.exit_code == 0, result.output - user = _get_user(app, "alice") - assert user is not None and user.check_password(passphrase) diff --git a/tests/forms/__init__.py b/tests/forms/__init__.py deleted file mode 100644 index 25d78d5..0000000 --- a/tests/forms/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py deleted file mode 100644 index ae4c8f2..0000000 --- a/tests/forms/test_validators.py +++ /dev/null @@ -1,166 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -from unittest.mock import Mock - -import pytest -from flask import Flask -from werkzeug.datastructures import MultiDict -from wtforms import Form, StringField, ValidationError - -from webmentions_ssg.forms.validators import ( - AllowedHostname, - NotEqualTo, - PublicIPLiteral, - PublicURL, - URLMaxBytes, -) -from webmentions_ssg.url_security import AddressResolutionError - - -class ComparisonForm(Form): - source = StringField("Source") - target = StringField("Target") - - -class URLForm(Form): - url = StringField("URL") - - -def test_not_equal_to_accepts_different_values() -> None: - form = ComparisonForm(MultiDict({"source": "source", "target": "target"})) - NotEqualTo("target")(form, form.source) - - -def test_not_equal_to_rejects_equal_values() -> None: - form = ComparisonForm(MultiDict({"source": "same", "target": "same"})) - with pytest.raises(ValidationError, match="Field must not be equal to target"): - NotEqualTo("target")(form, form.source) - - -def test_not_equal_to_uses_custom_message() -> None: - form = ComparisonForm(MultiDict({"source": "same", "target": "same"})) - with pytest.raises(ValidationError, match="Must differ from Target"): - NotEqualTo("target", "Must differ from %(other_label)s")(form, form.source) - - -def test_not_equal_to_rejects_unknown_field() -> None: - form = ComparisonForm(MultiDict({"source": "source", "target": "target"})) - with pytest.raises(ValidationError, match="Invalid field name 'missing'"): - NotEqualTo("missing")(form, form.source) - - -def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: - app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) - with app.app_context(): - AllowedHostname()(form, form.url) - - -def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: - app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedHostname()(form, form.url) - - -def test_allowed_hostname_uses_custom_message(app: Flask) -> None: - app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with ( - app.app_context(), - pytest.raises(ValidationError, match="Hostname is not allowed"), - ): - AllowedHostname("Hostname is not allowed")(form, form.url) - - -def test_public_url_accepts_public_url(monkeypatch: pytest.MonkeyPatch) -> None: - is_public_url = Mock(return_value=True) - monkeypatch.setattr("webmentions_ssg.forms.validators.is_public_url", is_public_url) - form = URLForm(MultiDict({"url": "https://example.com/post"})) - PublicURL()(form, form.url) - is_public_url.assert_called_once_with("https://example.com/post") - - -@pytest.mark.parametrize( - "outcome", - [ - pytest.param(False, id="non-public-address"), - pytest.param( - AddressResolutionError("Could not resolve hostname"), id="resolution-error" - ), - pytest.param(ValueError("No hostname was specified"), id="missing-hostname"), - ], -) -def test_public_url_rejects_invalid_url( - monkeypatch: pytest.MonkeyPatch, outcome: bool | Exception -) -> None: - is_public_url = Mock() - - if isinstance(outcome, Exception): - is_public_url.side_effect = outcome - else: - is_public_url.return_value = outcome - - monkeypatch.setattr("webmentions_ssg.forms.validators.is_public_url", is_public_url) - form = URLForm(MultiDict({"url": "https://example.com/post"})) - - with pytest.raises(ValidationError, match="URL must resolve to a public address"): - PublicURL()(form, form.url) - - -def test_public_url_uses_custom_message(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "webmentions_ssg.forms.validators.is_public_url", Mock(return_value=False) - ) - form = URLForm(MultiDict({"url": "https://example.com/post"})) - - with pytest.raises(ValidationError, match="Public URL required"): - PublicURL("Public URL required")(form, form.url) - - -@pytest.mark.parametrize( - "address", - ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], -) -def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: - form = URLForm(MultiDict({"url": f"https://[{address}]/"})) - with pytest.raises(ValidationError, match="URL must use a public address"): - PublicIPLiteral()(form, form.url) - - -def test_public_ip_literal_accepts_public_ipv6_address() -> None: - form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) - PublicIPLiteral()(form, form.url) - - -def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: - form = URLForm(MultiDict({"url": "http://[::1"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedHostname()(form, form.url) - - -def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: - validator = URLMaxBytes() - form = URLForm(MultiDict({"url": "é" * 3})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 6 - validator(form, form.url) - app.config["MAX_WEBMENTION_URL_BYTES"] = 5 - with pytest.raises(ValidationError, match="5 bytes"): - validator(form, form.url) - - -def test_url_max_bytes_custom_message(app: Flask) -> None: - form = URLForm(MultiDict({"url": "too-long"})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 2 - with pytest.raises(ValidationError, match="URL too long"): - URLMaxBytes(message="URL too long")(form, form.url) - - -def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: - form = URLForm(MultiDict({"url": "\ud800"})) - with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): - URLMaxBytes()(form, form.url) diff --git a/tests/tasks/scanner_helpers.py b/tests/tasks/scanner_helpers.py new file mode 100644 index 0000000..4100028 --- /dev/null +++ b/tests/tasks/scanner_helpers.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path + +from bs4 import BeautifulSoup +from flask import Flask + +BASE_URL = "https://dennisfink.me/blog/" +SOURCE_URL = f"{BASE_URL}example/" + + +def parse_html(html: str) -> BeautifulSoup: + return BeautifulSoup(html, "html5lib") + + +def write_post( + root: Path, + slug: str, + content: str, + *, + canonical: str | None = None, + entry_url: str | None = None, +) -> Path: + directory = root / slug + directory.mkdir(parents=True, exist_ok=True) + + source_url = f"{BASE_URL}{slug}/" + + if canonical is None: + canonical = source_url + + if entry_url is None: + entry_url = source_url + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{canonical}" + > + </head> + <body> + <article class="h-entry"> + <a + class="u-url" + href="{entry_url}" + > + Permalink + </a> + + <div class="e-content"> + {content} + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + return path + + +def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None: + app.config["SOURCE_DIRECTORY"] = str(root) + app.config["SOURCE_BASE_URL"] = base_url diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py new file mode 100644 index 0000000..2fc30b1 --- /dev/null +++ b/tests/tasks/test_discovery.py @@ -0,0 +1,951 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType +from unittest.mock import Mock + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg.tasks import discovery + +SOURCE_URL = "https://dennisfink.me/blog/example/" +TARGET_URL = "https://example.com/post" +ENDPOINT_URL = "https://example.com/webmention" +STATUS_URL = "https://example.com/webmention/status/123" + + +@pytest.fixture +def sender_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import discovery + + return discovery + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ( + '<https://example.com/webmention>; rel="webmention"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + "<https://example.com/webmention>; rel=webmention", + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/webmention>; rel="webmention alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + '<https://example.com/webmention>; REL="WebMention Alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + ( + '<https://example.com/webmention>; type="text/html"; ' + 'rel="webmention"; title="Endpoint"' + ), + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/stylesheet>; type="text/css"', + ("https://example.com/stylesheet", set()), + ), + ), +) +def test_parse_link_value( + sender_module: ModuleType, value: str, expected: tuple[str, set[str]] +) -> None: + assert sender_module.parse_link_value(value) == expected + + +@pytest.mark.parametrize( + "value", ("", "https://example.com/webmention", "<https://example.com/webmention") +) +def test_parse_link_value_rejects_malformed_value( + sender_module: ModuleType, value: str +) -> None: + assert sender_module.parse_link_value(value) is None + + +@pytest.mark.parametrize( + ("target", "headers", "html", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/1", + (("Link", "</test/1/webmention?head=true>; rel=webmention"),), + "", + "https://example.com/test/1/webmention?head=true", + id="1-http-link-unquoted-rel-relative-url", + ), + pytest.param( + "https://example.com/test/2", + ( + ( + "Link", + ( + "<https://example.com/test/2/webmention?head=true>;" + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/2/webmention?head=true", + id="2-http-link-unquoted-rel-absolute-url", + ), + pytest.param( + "https://example.com/test/3", + (), + """ + <link + rel="webmention" + href="/test/3/webmention" + > + """, + "https://example.com/test/3/webmention", + id="3-html-link-relative-url", + ), + pytest.param( + "https://example.com/test/4", + (), + """ + <link + rel="webmention" + href="https://example.com/test/4/webmention" + > + """, + "https://example.com/test/4/webmention", + id="4-html-link-absolute-url", + ), + pytest.param( + "https://example.com/test/5", + (), + """ + <a + rel="webmention" + href="/test/5/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/5/webmention", + id="5-html-a-relative-url", + ), + pytest.param( + "https://example.com/test/6", + (), + """ + <a + rel="webmention" + href="https://example.com/test/6/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/6/webmention", + id="6-html-a-absolute-url", + ), + pytest.param( + "https://example.com/test/7", + ( + ( + "LinK", + ( + "<https://example.com/test/7/webmention?head=true>; " + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/7/webmention?head=true", + id="7-http-link-strange-casing", + ), + pytest.param( + "https://example.com/test/8", + ( + ( + "Link", + ( + "<https://example.com/test/8/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/8/webmention?head=true", + id="8-http-link-quoted-rel", + ), + pytest.param( + "https://example.com/test/9", + (), + """ + <link + rel="webmention somethingelse" + href="https://example.com/test/9/webmention" + > + """, + "https://example.com/test/9/webmention", + id="9-multiple-html-rel-values", + ), + pytest.param( + "https://example.com/test/10", + ( + ( + "Link", + ( + "<https://example.com/test/10/webmention?head=true>; " + 'rel="webmention somethingelse"' + ), + ), + ), + "", + "https://example.com/test/10/webmention?head=true", + id="10-multiple-link-header-rel-values", + ), + pytest.param( + "https://example.com/test/11", + (("Link", '</test/11/webmention>; rel="webmention"'),), + """ + <link + rel="webmention" + href="/test/11/webmention/error" + > + + <a + rel="webmention" + href="/test/11/webmention/error" + > + Wrong endpoint + </a> + """, + "https://example.com/test/11/webmention", + id="11-http-link-precedence", + ), + pytest.param( + "https://example.com/test/12", + (), + """ + <link + rel="not-webmention" + href="/test/12/webmention/error" + > + + <a + rel="webmention" + href="/test/12/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/12/webmention", + id="12-exact-rel-match", + ), + pytest.param( + "https://example.com/test/13", + (), + """ + <!-- + <a + rel="webmention" + href="/test/13/webmention/error" + > + False endpoint + </a> + --> + + <a + rel="webmention" + href="/test/13/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/13/webmention", + id="13-endpoint-inside-html-comment", + ), + pytest.param( + "https://example.com/test/14", + (), + """ + <code> + <a + rel="webmention" + href="/test/14/webmention/error" + > + False endpoint + </a> + </code> + + <a + rel="webmention" + href="/test/14/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/14/webmention", + id="14-endpoint-in-escaped-html", + ), + pytest.param( + "https://example.com/test/15", + (), + """ + <link + rel="webmention" + href="" + > + """, + "https://example.com/test/15", + id="15-empty-href", + ), + pytest.param( + "https://example.com/test/16", + (), + """ + <a + rel="webmention" + href="/test/16/webmention" + > + First endpoint + </a> + + <link + rel="webmention" + href="/test/16/webmention/error" + > + """, + "https://example.com/test/16/webmention", + id="16-a-before-link", + ), + pytest.param( + "https://example.com/test/17", + (), + """ + <link + rel="webmention" + href="/test/17/webmention" + > + + <a + rel="webmention" + href="/test/17/webmention/error" + > + Second endpoint + </a> + """, + "https://example.com/test/17/webmention", + id="17-link-before-a", + ), + pytest.param( + "https://example.com/test/18", + ( + ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'), + ( + "Link", + ( + "<https://example.com/test/18/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/18/webmention?head=true", + id="18-multiple-http-link-headers", + ), + pytest.param( + "https://example.com/test/19", + ( + ( + "Link", + ( + "<https://example.com/test/19/webmention/error>; " + 'rel="other", ' + "<https://example.com/test/19/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/19/webmention?head=true", + id="19-single-header-multiple-values", + ), + pytest.param( + "https://example.com/test/20", + (), + """ + <link rel="webmention"> + + <a + rel="webmention" + href="/test/20/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/20/webmention", + id="20-link-without-href", + ), + pytest.param( + "https://example.com/test/21", + (), + """ + <link + rel="webmention" + href="/test/21/webmention?query=yes" + > + """, + "https://example.com/test/21/webmention?query=yes", + id="21-endpoint-query-string", + ), + pytest.param( + "https://example.com/test/22", + (), + """ + <link + rel="webmention" + href="22/webmention" + > + """, + "https://example.com/test/22/webmention", + id="22-endpoint-relative-to-path", + ), + ), +) +def test_discovery( + app: Flask, + sender_module: ModuleType, + target: str, + headers: tuple[tuple[str, str], ...], + html: str, + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers=headers) + + return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +@pytest.mark.parametrize( + ("target", "responses", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/23/page", + { + "/test/23/page": (302, {"Location": "page/redirect-key"}), + "/test/23/page/redirect-key": ( + 200, + {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")}, + ), + }, + ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), + id="relative-endpoint-after-redirect", + ), + ), +) +def test_discovery_redirect( + app: Flask, + sender_module: ModuleType, + target: str, + responses: dict[str, tuple[int, dict[str, str]]], + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + try: + status, headers = responses[request.url.path] + except KeyError: + raise AssertionError(f"Unexpected request: {request.url}") from None + + return httpx.Response(status, headers=headers) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD"] + + +def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=""" + <link + rel="webmention" + href="/webmention" + > + """, + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "failure", + [ + pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), + pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), + ], +) +def test_discovery_falls_back_to_get_after_head_request_error( + app: Flask, sender_module: ModuleType, failure: httpx.RequestError +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + + if request.method == "HEAD": + raise failure + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["HEAD", "GET"] + + +def test_discovery_falls_back_to_get_after_head_dns_failure( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def request_hook(request: httpx.Request) -> None: + if request.method == "HEAD": + raise sender_module.TemporarySenderError("HEAD DNS lookup failed") + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), + event_hooks={"request": [request_hook]}, + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["GET"] + + +def test_discovery_get_error_after_head_error_is_not_suppressed( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + raise httpx.ConnectError(f"{request.method} failed") + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(httpx.ConnectError, match="GET failed"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "invalid_href", + [ + pytest.param("javascript:alert(1)", id="unsupported-scheme"), + pytest.param("http://[invalid", id="malformed-url"), + ], +) +def test_discovery_skips_invalid_html_candidate( + app: Flask, sender_module: ModuleType, invalid_href: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=( + f'<link rel="webmention" href="{invalid_href}">' + '<a rel="webmention" href="/webmention">Valid endpoint</a>' + ), + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_skips_invalid_link_header_candidate( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers=[ + ("Link", "<mailto:invalid@example.com>; rel=webmention"), + ("Link", "<http://[invalid>; rel=webmention"), + ("Link", "</webmention>; rel=webmention"), + ], + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_returns_none_when_all_html_candidates_are_invalid( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"}) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text='<link rel="webmention" href="javascript:void(0)">', + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +def test_resolve_endpoint_rejects_invalid_url( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_http_url = Mock(return_value=False) + monkeypatch.setattr(discovery, "is_http_url", is_http_url) + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + with pytest.raises( + sender_module.PermanentSenderError, match="Invalid Webmention endpoint" + ): + sender_module.resolve_endpoint(response, "/webmention") + is_http_url.assert_called_once_with(ENDPOINT_URL) + + +def test_endpoint_from_headers_ignores_malformed_link( + sender_module: ModuleType, +) -> None: + response = httpx.Response( + 200, + headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")], + request=httpx.Request("GET", TARGET_URL), + ) + assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL + + +def test_endpoint_from_html_accepts_string_rel( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + element = Mock() + element.get.side_effect = lambda name: { + "rel": "webmention alternate", + "href": "/webmention", + }[name] + + document = Mock() + document.find_all.return_value = [element] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL + + +def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + missing_rel = Mock() + missing_rel.get.return_value = None + + invalid_href = Mock() + invalid_href.get.side_effect = lambda name: { + "rel": ["webmention"], + "href": ["invalid"], + }[name] + + document = Mock() + document.find_all.return_value = [missing_rel, invalid_href] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") is None + + +def test_read_target_body_rejects_declared_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_read_target_body_ignores_invalid_content_length( + app: Flask, sender_module: ModuleType +) -> None: + response = httpx.Response( + 200, headers={"Content-Length": "invalid"}, content=b"Document" + ) + + with app.app_context(): + body = sender_module.read_target_body(response) + + assert body == b"Document" + + +def test_read_target_body_rejects_streamed_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + response = httpx.Response(200, stream=BodyStream()) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_discovery_uses_link_header_from_get( + app: Flask, sender_module: ModuleType +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + pytest.param(503, "TemporarySenderError", id="temporary"), + pytest.param(404, "PermanentSenderError", id="permanent"), + ], +) +def test_discovery_maps_get_status_to_exception( + app: Flask, sender_module: ModuleType, status_code: int, exception_name: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(status_code) + + exception_type = getattr(sender_module, exception_name) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + +def test_discovery_returns_none_for_unsupported_media_type( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, headers={"Content-Type": "application/json"}, content=b"{}" + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +@pytest.mark.parametrize( + ("status_code", "location", "expected_status_url"), + [ + pytest.param( + 201, "/webmention/status/123", STATUS_URL, id="created-with-location" + ), + pytest.param(202, None, None, id="accepted"), + ], +) +def test_post_webmention( + sender_module: ModuleType, + status_code: int, + location: str | None, + expected_status_url: str | None, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + headers = {} + + if location is not None: + headers["Location"] = location + + return httpx.Response(status_code, headers=headers) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (status_code, expected_status_url) + assert len(requests) == 1 + + request = requests[0] + + assert request.method == "POST" + assert str(request.url) == ENDPOINT_URL + + form = httpx.QueryParams(request.content.decode()) + + assert form["source"] == SOURCE_URL + assert form["target"] == TARGET_URL + + +@pytest.mark.parametrize( + ("code", "location", "expected"), + [ + (201, "/webmention/status/123", "https://example.com/webmention/status/123"), + (201, "https://status.example.org/123", "https://status.example.org/123"), + (201, "status/123", "https://example.com/status/123"), + (201, "javascript:alert(1)", None), + (201, "data:text/html,evil", None), + (201, "ftp://example.com/status", None), + (201, "http://[invalid", None), + (202, "https://example.com/status", None), + ], +) +def test_post_webmention_validates_location( + sender_module: ModuleType, code: int, location: str, expected: str | None +) -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(code, headers={"Location": location}) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (code, expected) diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py index 108e8a6..c733dce 100644 --- a/tests/tasks/test_json_serializer.py +++ b/tests/tasks/test_json_serializer.py @@ -10,7 +10,7 @@ from datetime import UTC, datetime, timedelta import pytest from flask import Flask -from huey.utils import Error, SKIPPED +from huey.utils import SKIPPED, Error from webmentions_ssg.tasks.extension import Huey from webmentions_ssg.tasks.json_serializer import JSONSerializer diff --git a/tests/tasks/test_mentions.py b/tests/tasks/test_mentions.py new file mode 100644 index 0000000..a5da56c --- /dev/null +++ b/tests/tasks/test_mentions.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType +from unittest.mock import Mock, patch + +import pytest +from flask import Flask + +SOURCE_URL = "https://source.example/article" +TARGET_URL = "https://dennisfink.me/blog/example/" + + +@pytest.fixture +def receiver(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import mentions + + return mentions + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"), + pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"), + pytest.param( + f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href" + ), + pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"), + pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"), + pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"), + pytest.param( + f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src" + ), + pytest.param( + f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src" + ), + pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"), + pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"), + pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"), + pytest.param( + f'<video><track src="{TARGET_URL}"></video>', True, id="track-src" + ), + pytest.param( + f'<input type="image" src="{TARGET_URL}" alt="">', + True, + id="image-input-src", + ), + pytest.param( + f'<input type="IMAGE" src="{TARGET_URL}" alt="">', + True, + id="image-input-case-insensitive", + ), + pytest.param( + f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>', + True, + id="blockquote-cite", + ), + pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"), + pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"), + pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"), + pytest.param( + '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>', + True, + id="base-url", + ), + pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"), + pytest.param( + f'<blockquote src="{TARGET_URL}">Quote</blockquote>', + False, + id="blockquote-src-invalid", + ), + pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"), + pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"), + pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"), + pytest.param( + f'<input type="text" src="{TARGET_URL}">', + False, + id="text-input-src-invalid", + ), + pytest.param( + f'<picture><source src="{TARGET_URL}"></picture>', + False, + id="picture-source-src-invalid", + ), + pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"), + pytest.param( + f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url" + ), + pytest.param( + f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">', + False, + id="invalid-cite-does-not-override-valid-src", + ), + pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"), + pytest.param( + '<a href="https://example.com/">Other site</a>', False, id="missing-target" + ), + ], +) +def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None: + assert ( + receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected + ) + + +def test_html_mentions_relative_target(receiver: ModuleType) -> None: + assert receiver.html_mentions_target( + b'<a href="../target/">Reply</a>', + "https://source.example/posts/article/", + "https://source.example/posts/target/", + ) + + +@pytest.mark.parametrize( + ("body", "target_url", "expected"), + [ + (TARGET_URL, TARGET_URL, True), + (f"This post replies to {TARGET_URL}", TARGET_URL, True), + ( + f"https://example.com/first {TARGET_URL} https://example.com/last", + TARGET_URL, + True, + ), + pytest.param( + f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma" + ), + pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"), + pytest.param( + f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation" + ), + pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"), + pytest.param( + "https://example.com/wiki/Foo_(bar)", + "https://example.com/wiki/Foo_(bar)", + True, + id="balanced-parentheses-in-url", + ), + pytest.param( + "https://example.com/wiki/Foo_(bar)", + "https://example.com/wiki/Foo_(bar", + False, + id="balanced-parentheses-not-trimmed", + ), + (f"{TARGET_URL}more", TARGET_URL, False), + pytest.param( + f"{TARGET_URL},another-page", + TARGET_URL, + False, + id="comma-within-longer-url", + ), + ("https://dennisfink.me/blog/other/", TARGET_URL, False), + ("/blog/example/", TARGET_URL, False), + ("There are no links here.", TARGET_URL, False), + ], +) +def test_text_mentions_target( + receiver: ModuleType, body: str, target_url: str, expected: bool +) -> None: + assert receiver.text_mentions_target(body, target_url) is expected + + +@patch("webmentions_ssg.tasks.mentions.BeautifulSoup") +def test_html_mentions_target_ignores_non_string_attribute( + beautiful_soup: Mock, receiver: ModuleType +) -> None: + document = beautiful_soup.return_value + document.select_one.return_value = None + + element = Mock() + element.get.return_value = ["not", "a", "string"] + document.select.return_value = [element] + + assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL) diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 44d1cc2..5441eb7 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -15,7 +15,7 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import ReceivedWebmention -from webmentions_ssg.url_security import AddressResolutionError +from webmentions_ssg.net import AddressResolutionError SOURCE_URL = "https://source.example/article" TARGET_URL = "https://dennisfink.me/blog/example/" @@ -68,148 +68,6 @@ def set_stream_response(httpx_client: Mock, response: httpx.Response) -> Mock: return client -@pytest.mark.parametrize( - ("body", "expected"), - [ - pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"), - pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"), - pytest.param( - f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href" - ), - pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"), - pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"), - pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"), - pytest.param( - f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src" - ), - pytest.param( - f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src" - ), - pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"), - pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"), - pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"), - pytest.param( - f'<video><track src="{TARGET_URL}"></video>', True, id="track-src" - ), - pytest.param( - f'<input type="image" src="{TARGET_URL}" alt="">', - True, - id="image-input-src", - ), - pytest.param( - f'<input type="IMAGE" src="{TARGET_URL}" alt="">', - True, - id="image-input-case-insensitive", - ), - pytest.param( - f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>', - True, - id="blockquote-cite", - ), - pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"), - pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"), - pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"), - pytest.param( - '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>', - True, - id="base-url", - ), - pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"), - pytest.param( - f'<blockquote src="{TARGET_URL}">Quote</blockquote>', - False, - id="blockquote-src-invalid", - ), - pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"), - pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"), - pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"), - pytest.param( - f'<input type="text" src="{TARGET_URL}">', - False, - id="text-input-src-invalid", - ), - pytest.param( - f'<picture><source src="{TARGET_URL}"></picture>', - False, - id="picture-source-src-invalid", - ), - pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"), - pytest.param( - f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url" - ), - pytest.param( - f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">', - False, - id="invalid-cite-does-not-override-valid-src", - ), - pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"), - pytest.param( - '<a href="https://example.com/">Other site</a>', False, id="missing-target" - ), - ], -) -def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None: - assert ( - receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected - ) - - -def test_html_mentions_relative_target(receiver: ModuleType) -> None: - assert receiver.html_mentions_target( - b'<a href="../target/">Reply</a>', - "https://source.example/posts/article/", - "https://source.example/posts/target/", - ) - - -@pytest.mark.parametrize( - ("body", "target_url", "expected"), - [ - (TARGET_URL, TARGET_URL, True), - (f"This post replies to {TARGET_URL}", TARGET_URL, True), - ( - f"https://example.com/first {TARGET_URL} https://example.com/last", - TARGET_URL, - True, - ), - pytest.param( - f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma" - ), - pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"), - pytest.param( - f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation" - ), - pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"), - pytest.param( - "https://example.com/wiki/Foo_(bar)", - "https://example.com/wiki/Foo_(bar)", - True, - id="balanced-parentheses-in-url", - ), - pytest.param( - "https://example.com/wiki/Foo_(bar)", - "https://example.com/wiki/Foo_(bar", - False, - id="balanced-parentheses-not-trimmed", - ), - (f"{TARGET_URL}more", TARGET_URL, False), - pytest.param( - f"{TARGET_URL},another-page", - TARGET_URL, - False, - id="comma-within-longer-url", - ), - ("https://dennisfink.me/blog/other/", TARGET_URL, False), - ("/blog/example/", TARGET_URL, False), - ("There are no links here.", TARGET_URL, False), - ], -) -def test_text_mentions_target( - receiver: ModuleType, body: str, target_url: str, expected: bool -) -> None: - assert receiver.text_mentions_target(body, target_url) is expected - - @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_returns_response_and_body( httpx_client: Mock, app: Flask, receiver: ModuleType @@ -786,7 +644,7 @@ def test_verify_webmention_ignores_unknown_identifier( def test_fetch_source_rejects_unsafe_destination( httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: - from webmentions_ssg.url_security import UnsafeAddressError + from webmentions_ssg.net import UnsafeAddressError client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = UnsafeAddressError("Local address") @@ -810,20 +668,6 @@ def test_fetch_source_treats_dns_failure_as_temporary( receiver.fetch_source(SOURCE_URL) -@patch("webmentions_ssg.tasks.receiver.BeautifulSoup") -def test_html_mentions_target_ignores_non_string_attribute( - beautiful_soup: Mock, receiver: ModuleType -) -> None: - document = beautiful_soup.return_value - document.select_one.return_value = None - - element = Mock() - element.get.return_value = ["not", "a", "string"] - document.select.return_value = [element] - - assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL) - - @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_ignores_invalid_content_length( httpx_client: Mock, app: Flask, receiver: ModuleType diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 26e81c4..7eb970c 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -18,9 +18,9 @@ from pytest import MonkeyPatch from webmentions_ssg import DATABASE as db from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source +from webmentions_ssg.tasks import source_parser -BASE_URL = "https://dennisfink.me/blog/" -SOURCE_URL = f"{BASE_URL}example/" +from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post @pytest.fixture @@ -29,6 +29,7 @@ def scanner_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: from webmentions_ssg.tasks import scanner monkeypatch.setattr(scanner, "is_http_url", Mock(return_value=True)) + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) return scanner @@ -80,764 +81,6 @@ def test_manual_scan_propagates_unrelated_errors( scanner_module.manual_scan_sources.call_local() -def parse_html(html: str) -> BeautifulSoup: - return BeautifulSoup(html, "html5lib") - - -def write_post( - root: Path, - slug: str, - content: str, - *, - canonical: str | None = None, - entry_url: str | None = None, -) -> Path: - directory = root / slug - directory.mkdir(parents=True, exist_ok=True) - - source_url = f"{BASE_URL}{slug}/" - - if canonical is None: - canonical = source_url - - if entry_url is None: - entry_url = source_url - - path = directory / "index.html" - path.write_text( - f""" - <!doctype html> - <html> - <head> - <link - rel="canonical" - href="{canonical}" - > - </head> - <body> - <article class="h-entry"> - <a - class="u-url" - href="{entry_url}" - > - Permalink - </a> - - <div class="e-content"> - {content} - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - return path - - -def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None: - app.config["SOURCE_DIRECTORY"] = str(root) - app.config["SOURCE_BASE_URL"] = base_url - - -def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - <article class="h-entry"> - <a class="u-url" href="{SOURCE_URL}"> - Permalink - </a> - - <div class="e-content"> - Hello world - </div> - </article> - """ - ) - - element = document.find(class_="h-entry") - - assert element is not None - - entry = scanner_module.parse_entry(element, SOURCE_URL) - - assert "h-entry" in entry["type"] - assert entry["properties"]["url"] == [SOURCE_URL] - - -def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article> - <p>Hello world</p> - </article> - """ - ) - - element = document.find("article") - - assert element is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one parsed h-entry" - ): - scanner_module.parse_entry(element, SOURCE_URL) - - -def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - <article class="h-entry" id="source"> - <a class="u-url" href="{SOURCE_URL}"> - Permalink - </a> - - <p class="p-name"> - Example post - </p> - - <div class="e-content"> - Hello world - </div> - </article> - """ - ) - - element, entry = scanner_module.primary_entry(document, SOURCE_URL) - - assert element["id"] == "source" - assert entry["properties"]["name"] == ["Example post"] - assert entry["properties"]["url"] == [SOURCE_URL] - - -def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <main> - <p>No microformats here.</p> - </main> - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - <article class="h-entry"> - <a class="u-url" href="{SOURCE_URL}"> - First - </a> - </article> - - <article class="h-entry"> - <a - class="u-url" - href="https://example.com/other/" - > - Second - </a> - </article> - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - <article class="h-entry"> - <a class="u-url" href="{SOURCE_URL}"> - Permalink - </a> - - <div class="e-content"> - <article class="h-entry"> - <a - class="u-url" - href="https://example.com/nested/" - > - Nested - </a> - </article> - </div> - </article> - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article class="h-entry"> - <div class="e-content"> - Hello world - </div> - </article> - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="does not match"): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article class="h-entry"> - <a - class="u-url" - href="https://example.com/other/" - > - Other - </a> - - <div class="e-content"> - Hello world - </div> - </article> - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="does not match"): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_content_element_returns_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article class="h-entry"> - <div class="e-content" id="content"> - Hello world - </div> - </article> - """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - content = scanner_module.content_element(entry) - - assert content["id"] == "content" - - -def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article class="h-entry"> - <p>Hello world</p> - </article> - """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one e-content, found 0" - ): - scanner_module.content_element(entry) - - -def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <article class="h-entry"> - <div class="e-content"> - First - </div> - - <div class="e-content"> - Second - </div> - </article> - """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one e-content, found 2" - ): - scanner_module.content_element(entry) - - -def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - <html> - <head> - <link - rel="canonical" - href="{SOURCE_URL}" - > - </head> - </html> - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - assert result == SOURCE_URL - - -def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <html> - <head> - <title>Example</title> - </head> - </html> - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - assert result is None - - -def test_canonical_url_resolves_relative_url_with_base_url( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.side_effect = [False, True] - - document = parse_html( - """ - <html> - <head> - <link - rel="canonical" - href="./canonical/" - > - </head> - </html> - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - assert result == ("https://dennisfink.me/blog/example/canonical/") - - -def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None: - scanner_module.is_http_url.return_value = False - - document = parse_html( - """ - <html> - <head> - <link - rel="canonical" - href="./canonical/" - > - </head> - </html> - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="relative canonical"): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - -def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None: - document = parse_html( - """ - <html> - <head> - <link - rel="canonical" - href=" " - > - </head> - </html> - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - -def test_scan_source_uses_canonical_without_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - <a href="https://example.com/target"> - Target - </a> - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.path == "example/index.html" - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - assert len(scanned.content_hash) == 32 - - -def test_scan_source_ignores_markup_noise_in_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - "<p>Hello <strong>world</strong>!</p>" - '<a href="https://example.com/target">Target</a>', - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - write_post( - tmp_path, - "example", - '<p class="generated" data-build="123"> Hello world! </p>' - '<a class="external" href="https://example.com/target"> Target </a>', - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == second.targets - assert first.content_hash == second.content_hash - - -def test_scan_source_uses_stable_time_value_inside_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>', - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.content_hash == second.content_hash - - -def test_scan_source_hash_changes_when_image_changes( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">') - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.content_hash != second.content_hash - - -def test_scan_source_hash_includes_reaction_targets_outside_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", "<p>Same content.</p>") - html = path.read_text(encoding="utf-8") - path.write_text( - html.replace( - '<div class="e-content">', - '<a class="u-like-of" href="https://example.com/first">Like</a>' - '<div class="e-content">', - ), - encoding="utf-8", - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace( - "https://example.com/first", "https://example.com/second" - ), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == frozenset({"https://example.com/first"}) - assert second.targets == frozenset({"https://example.com/second"}) - assert first.content_hash != second.content_hash - - -def test_scan_source_hash_ignores_reaction_target_order( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", "<p>Same content.</p>") - html = path.read_text(encoding="utf-8") - first_link = '<a class="u-like-of" href="https://example.com/a">A</a>' - second_link = '<a class="u-like-of" href="https://example.com/b">B</a>' - path.write_text( - html.replace( - '<div class="e-content">', - first_link + second_link + '<div class="e-content">', - ), - encoding="utf-8", - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace( - first_link + second_link, second_link + first_link - ), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == second.targets - assert first.content_hash == second.content_hash - - -def test_scan_source_falls_back_to_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - """ - <!doctype html> - <html> - <body> - <article class="h-entry"> - <a class="u-url" href="./"> - Permalink - </a> - - <div class="e-content"> - <a href="https://example.com/target"> - Target - </a> - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL) - - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - - -def test_scan_source_requires_canonical_or_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - """ - <!doctype html> - <html> - <body> - <article class="h-entry"> - <a class="u-url" href="./"> - Permalink - </a> - - <div class="e-content"> - <p>Hello world.</p> - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"): - scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - -def test_scan_source_extracts_only_content_links( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - f""" - <!doctype html> - <html> - <head> - <link - rel="canonical" - href="{SOURCE_URL}" - > - </head> - <body> - <article class="h-entry"> - <a class="u-url" href="{SOURCE_URL}"> - Permalink - </a> - - <a href="https://example.com/metadata"> - Metadata link - </a> - - <div class="e-content"> - <a href="https://example.com/content"> - Content link - </a> - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://example.com/content"}) - - -def test_scan_source_extracts_relative_content_link( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - <a href="../other/"> - Other post - </a> - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) - - -@pytest.mark.parametrize( - "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") -) -def test_scan_source_extracts_reaction_properties( - scanner_module: ModuleType, tmp_path: Path, property_name: str -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - target = f"https://example.com/{property_name}" - - path = directory / "index.html" - path.write_text( - f""" - <!doctype html> - <html> - <head> - <link - rel="canonical" - href="{SOURCE_URL}" - > - </head> - <body> - <article class="h-entry"> - <a class="u-url" href="{SOURCE_URL}"> - Permalink - </a> - - <a - class="u-{property_name}" - href="{target}" - > - Reaction target - </a> - - <div class="e-content"> - <p>Post content.</p> - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({target}) - - -@pytest.mark.parametrize("tag", ("link", "area")) -def test_scan_source_ignores_non_anchor_href_elements( - scanner_module: ModuleType, tmp_path: Path, tag: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - <{tag} href="https://example.com/target"> - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset() - - -def test_scan_source_ignores_self_fragment( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - <a href="#section"> - Section - </a> - - <a href="https://example.com/target"> - Target - </a> - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://example.com/target"}) - - def test_scan_creates_source_and_webmentions( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1885,23 +1128,23 @@ def test_unexpected_parser_error_does_not_abort_other_sources( write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>') if parser == "bs4": - original_soup = scanner_module.BeautifulSoup + original_soup = source_parser.BeautifulSoup def failing_soup(markup: bytes, features: str) -> BeautifulSoup: if b"/bad/" in markup: raise failure return original_soup(markup, features) - monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup) + monkeypatch.setattr(source_parser, "BeautifulSoup", failing_soup) else: - original_parse = scanner_module.mf2py.parse + original_parse = source_parser.mf2py.parse def failing_parse(*, doc: str, url: str) -> dict[str, Any]: if "/bad/" in doc: raise failure return original_parse(doc=doc, url=url) - monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse) + monkeypatch.setattr(source_parser.mf2py, "parse", failing_parse) queued: list[UUID] = [] monkeypatch.setattr(scanner_module, "send_webmention", queued.append) @@ -1979,46 +1222,6 @@ def test_unexpected_persistence_error_is_not_suppressed( run_scan(app, scanner_module) -def test_scan_source_rejects_symlink_outside_root_before_read( - scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch -) -> None: - root = tmp_path / "site" - root.mkdir() - external = write_post(tmp_path / "outside", "private", "Sensitive content") - linked = root / "escaped" / "index.html" - linked.parent.mkdir() - linked.symlink_to(external) - - read_bytes = Mock(side_effect=AssertionError("External file must not be read")) - monkeypatch.setattr(Path, "read_bytes", read_bytes) - - with pytest.raises( - scanner_module.SourceScanError, match="escapes source directory" - ): - scanner_module.scan_source_file(linked, root=root, base_url=None) - - read_bytes.assert_not_called() - - -def test_scan_source_accepts_symlink_within_root( - scanner_module: ModuleType, tmp_path: Path -) -> None: - root = tmp_path / "site" - root.mkdir() - linked = write_post( - root, "example", '<a href="https://example.com/target">Target</a>' - ) - shared = root / "shared.html" - linked.rename(shared) - linked.symlink_to(shared) - - scanned = scanner_module.scan_source_file(linked, root=root, base_url=None) - - assert scanned.path == "example/index.html" - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - - def test_scan_sources_skips_symlink_outside_root( app: Flask, scanner_module: ModuleType, @@ -2043,16 +1246,6 @@ def test_scan_sources_skips_symlink_outside_root( assert "escapes source directory" in caplog.text -def test_scan_sources_rejects_malformed_runtime_ignored_hostnames( - app: Flask, scanner_module: ModuleType, tmp_path: Path -) -> None: - configure_scanner(app, tmp_path) - app.config["IGNORED_HOSTNAMES"] = "example.com" - - with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"): - run_scan(app, scanner_module) - - def test_scan_sources_accepts_valid_base_url( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -2104,220 +1297,6 @@ def test_scan_sources_rejects_invalid_base_url( run_scan(app, scanner_module) -@pytest.mark.parametrize( - "href", - ( - "https://dennisfink.me/blog/other/", - "https://www.dennisfink.me/blog/other/", - "https://webmentions.dennisfink.me/endpoint", - "https://foo.bar.dennisfink.me/example", - ), -) -def test_scan_source_ignores_matching_hostname( - scanner_module: ModuleType, tmp_path: Path, href: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - <a href="{href}"> - Ignored - </a> - - <a href="https://example.com/target"> - External - </a> - """, - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset({"https://example.com/target"}) - - -@pytest.mark.parametrize( - "href", - ( - "https://notdennisfink.me/example", - "https://dennisfink.me.example.com/example", - "https://example.com/example", - ), -) -def test_scan_source_keeps_nonmatching_hostname( - scanner_module: ModuleType, tmp_path: Path, href: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - <a href="{href}"> - Target - </a> - """, - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset({href}) - - -def test_scan_source_ignores_reaction_to_matching_hostname( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - f""" - <!doctype html> - <html> - <head> - <link - rel="canonical" - href="{SOURCE_URL}" - > - </head> - <body> - <article class="h-entry"> - <a - class="u-url" - href="{SOURCE_URL}" - > - Permalink - </a> - - <a - class="u-in-reply-to" - href="https://www.dennisfink.me/blog/other/" - > - Reply target - </a> - - <div class="e-content"> - <p>Reply content.</p> - </div> - </article> - </body> - </html> - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset() - - -def test_canonical_url_rejects_non_http_resolved_url( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.return_value = False - - document = parse_html( - """ - <html> - <head> - <link - rel="canonical" - href="mailto:example@example.com" - > - </head> - </html> - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, - match="Canonical URL is not a valid HTTP or HTTPS URL", - ): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - -def test_property_urls_ignores_non_mapping_properties( - scanner_module: ModuleType, -) -> None: - entry = {"properties": "invalid"} - - assert tuple(scanner_module.property_urls(entry, "url")) == () - - -@pytest.mark.parametrize( - ("parsed", "message"), - [ - pytest.param( - [], - "Microformats parser did not return a document object", - id="non-object-document", - ), - pytest.param( - {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" - ), - pytest.param( - {"items": [{"type": ["h-card"]}]}, - "Parsed microformats item is not an h-entry", - id="wrong-item-type", - ), - ], -) -def test_parse_entry_rejects_invalid_parser_output( - scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str -) -> None: - document = parse_html( - """ - <article class="h-entry"> - <p>Hello world</p> - </article> - """ - ) - - element = document.find(class_="h-entry") - - assert element is not None - - monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed) - - with pytest.raises(scanner_module.SourceScanError, match=message): - scanner_module.parse_entry(element, SOURCE_URL) - - -def test_normalize_target_ignores_empty_target(scanner_module: ModuleType) -> None: - assert ( - scanner_module.normalize_target(" ", base_url=SOURCE_URL, source_url=SOURCE_URL) - is None - ) - - -def test_normalize_target_ignores_url_rejected_by_validator( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.return_value = False - - assert ( - scanner_module.normalize_target( - "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL - ) - is None - ) - - def test_changed_source_url_is_rejected( app: Flask, scanner_module: ModuleType, diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 7ebe083..c2f6e02 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -14,7 +14,6 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source -from webmentions_ssg.url_security import AddressResolutionError SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" @@ -101,25 +100,6 @@ def sender_module(app: Flask) -> ModuleType: @pytest.mark.parametrize( - ("status", "expected"), - [ - pytest.param(200, False, id="success"), - pytest.param(400, False, id="client-error"), - pytest.param(408, True, id="request-timeout"), - pytest.param(425, True, id="too-early"), - pytest.param(429, True, id="too-many-requests"), - pytest.param(500, True, id="server-error-lower-bound"), - pytest.param(599, True, id="server-error-upper-bound"), - pytest.param(600, False, id="outside-server-error-range"), - ], -) -def test_temporary_http_status( - sender_module: ModuleType, status: int, expected: bool -) -> None: - assert sender_module.temporary_http_status(status) is expected - - -@pytest.mark.parametrize( ("desired_revision", "processed_revision", "revision", "expected"), [ pytest.param(2, None, 2, True, id="unprocessed-current"), @@ -147,671 +127,6 @@ def test_attempt_is_current( assert sender_module.attempt_is_current(webmention, revision) is expected -@pytest.mark.parametrize( - ("value", "expected"), - ( - ( - '<https://example.com/webmention>; rel="webmention"', - ("https://example.com/webmention", {"webmention"}), - ), - ( - "<https://example.com/webmention>; rel=webmention", - ("https://example.com/webmention", {"webmention"}), - ), - ( - '<https://example.com/webmention>; rel="webmention alternate"', - ("https://example.com/webmention", {"webmention", "alternate"}), - ), - ( - '<https://example.com/webmention>; REL="WebMention Alternate"', - ("https://example.com/webmention", {"webmention", "alternate"}), - ), - ( - ( - '<https://example.com/webmention>; type="text/html"; ' - 'rel="webmention"; title="Endpoint"' - ), - ("https://example.com/webmention", {"webmention"}), - ), - ( - '<https://example.com/stylesheet>; type="text/css"', - ("https://example.com/stylesheet", set()), - ), - ), -) -def test_parse_link_value( - sender_module: ModuleType, value: str, expected: tuple[str, set[str]] -) -> None: - assert sender_module.parse_link_value(value) == expected - - -@pytest.mark.parametrize( - "value", ("", "https://example.com/webmention", "<https://example.com/webmention") -) -def test_parse_link_value_rejects_malformed_value( - sender_module: ModuleType, value: str -) -> None: - assert sender_module.parse_link_value(value) is None - - -@pytest.mark.parametrize( - ("target", "headers", "html", "expected_endpoint"), - ( - pytest.param( - "https://example.com/test/1", - (("Link", "</test/1/webmention?head=true>; rel=webmention"),), - "", - "https://example.com/test/1/webmention?head=true", - id="1-http-link-unquoted-rel-relative-url", - ), - pytest.param( - "https://example.com/test/2", - ( - ( - "Link", - ( - "<https://example.com/test/2/webmention?head=true>;" - "rel=webmention" - ), - ), - ), - "", - "https://example.com/test/2/webmention?head=true", - id="2-http-link-unquoted-rel-absolute-url", - ), - pytest.param( - "https://example.com/test/3", - (), - """ - <link - rel="webmention" - href="/test/3/webmention" - > - """, - "https://example.com/test/3/webmention", - id="3-html-link-relative-url", - ), - pytest.param( - "https://example.com/test/4", - (), - """ - <link - rel="webmention" - href="https://example.com/test/4/webmention" - > - """, - "https://example.com/test/4/webmention", - id="4-html-link-absolute-url", - ), - pytest.param( - "https://example.com/test/5", - (), - """ - <a - rel="webmention" - href="/test/5/webmention" - > - Endpoint - </a> - """, - "https://example.com/test/5/webmention", - id="5-html-a-relative-url", - ), - pytest.param( - "https://example.com/test/6", - (), - """ - <a - rel="webmention" - href="https://example.com/test/6/webmention" - > - Endpoint - </a> - """, - "https://example.com/test/6/webmention", - id="6-html-a-absolute-url", - ), - pytest.param( - "https://example.com/test/7", - ( - ( - "LinK", - ( - "<https://example.com/test/7/webmention?head=true>; " - "rel=webmention" - ), - ), - ), - "", - "https://example.com/test/7/webmention?head=true", - id="7-http-link-strange-casing", - ), - pytest.param( - "https://example.com/test/8", - ( - ( - "Link", - ( - "<https://example.com/test/8/webmention?head=true>; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/8/webmention?head=true", - id="8-http-link-quoted-rel", - ), - pytest.param( - "https://example.com/test/9", - (), - """ - <link - rel="webmention somethingelse" - href="https://example.com/test/9/webmention" - > - """, - "https://example.com/test/9/webmention", - id="9-multiple-html-rel-values", - ), - pytest.param( - "https://example.com/test/10", - ( - ( - "Link", - ( - "<https://example.com/test/10/webmention?head=true>; " - 'rel="webmention somethingelse"' - ), - ), - ), - "", - "https://example.com/test/10/webmention?head=true", - id="10-multiple-link-header-rel-values", - ), - pytest.param( - "https://example.com/test/11", - (("Link", '</test/11/webmention>; rel="webmention"'),), - """ - <link - rel="webmention" - href="/test/11/webmention/error" - > - - <a - rel="webmention" - href="/test/11/webmention/error" - > - Wrong endpoint - </a> - """, - "https://example.com/test/11/webmention", - id="11-http-link-precedence", - ), - pytest.param( - "https://example.com/test/12", - (), - """ - <link - rel="not-webmention" - href="/test/12/webmention/error" - > - - <a - rel="webmention" - href="/test/12/webmention" - > - Correct endpoint - </a> - """, - "https://example.com/test/12/webmention", - id="12-exact-rel-match", - ), - pytest.param( - "https://example.com/test/13", - (), - """ - <!-- - <a - rel="webmention" - href="/test/13/webmention/error" - > - False endpoint - </a> - --> - - <a - rel="webmention" - href="/test/13/webmention" - > - Correct endpoint - </a> - """, - "https://example.com/test/13/webmention", - id="13-endpoint-inside-html-comment", - ), - pytest.param( - "https://example.com/test/14", - (), - """ - <code> - <a - rel="webmention" - href="/test/14/webmention/error" - > - False endpoint - </a> - </code> - - <a - rel="webmention" - href="/test/14/webmention" - > - Correct endpoint - </a> - """, - "https://example.com/test/14/webmention", - id="14-endpoint-in-escaped-html", - ), - pytest.param( - "https://example.com/test/15", - (), - """ - <link - rel="webmention" - href="" - > - """, - "https://example.com/test/15", - id="15-empty-href", - ), - pytest.param( - "https://example.com/test/16", - (), - """ - <a - rel="webmention" - href="/test/16/webmention" - > - First endpoint - </a> - - <link - rel="webmention" - href="/test/16/webmention/error" - > - """, - "https://example.com/test/16/webmention", - id="16-a-before-link", - ), - pytest.param( - "https://example.com/test/17", - (), - """ - <link - rel="webmention" - href="/test/17/webmention" - > - - <a - rel="webmention" - href="/test/17/webmention/error" - > - Second endpoint - </a> - """, - "https://example.com/test/17/webmention", - id="17-link-before-a", - ), - pytest.param( - "https://example.com/test/18", - ( - ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'), - ( - "Link", - ( - "<https://example.com/test/18/webmention?head=true>; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/18/webmention?head=true", - id="18-multiple-http-link-headers", - ), - pytest.param( - "https://example.com/test/19", - ( - ( - "Link", - ( - "<https://example.com/test/19/webmention/error>; " - 'rel="other", ' - "<https://example.com/test/19/webmention?head=true>; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/19/webmention?head=true", - id="19-single-header-multiple-values", - ), - pytest.param( - "https://example.com/test/20", - (), - """ - <link rel="webmention"> - - <a - rel="webmention" - href="/test/20/webmention" - > - Correct endpoint - </a> - """, - "https://example.com/test/20/webmention", - id="20-link-without-href", - ), - pytest.param( - "https://example.com/test/21", - (), - """ - <link - rel="webmention" - href="/test/21/webmention?query=yes" - > - """, - "https://example.com/test/21/webmention?query=yes", - id="21-endpoint-query-string", - ), - pytest.param( - "https://example.com/test/22", - (), - """ - <link - rel="webmention" - href="22/webmention" - > - """, - "https://example.com/test/22/webmention", - id="22-endpoint-relative-to-path", - ), - ), -) -def test_discovery( - app: Flask, - sender_module: ModuleType, - target: str, - headers: tuple[tuple[str, str], ...], - html: str, - expected_endpoint: str, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200, headers=headers) - - return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, target) - - assert endpoint == expected_endpoint - - -@pytest.mark.parametrize( - ("target", "responses", "expected_endpoint"), - ( - pytest.param( - "https://example.com/test/23/page", - { - "/test/23/page": (302, {"Location": "page/redirect-key"}), - "/test/23/page/redirect-key": ( - 200, - {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")}, - ), - }, - ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), - id="relative-endpoint-after-redirect", - ), - ), -) -def test_discovery_redirect( - app: Flask, - sender_module: ModuleType, - target: str, - responses: dict[str, tuple[int, dict[str, str]]], - expected_endpoint: str, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - try: - status, headers = responses[request.url.path] - except KeyError: - raise AssertionError(f"Unexpected request: {request.url}") from None - - return httpx.Response(status, headers=headers) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, target) - - assert endpoint == expected_endpoint - - -def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint( - client, "https://example.com/post" - ) - - assert endpoint == "https://example.com/webmention" - assert [request.method for request in requests] == ["HEAD"] - - -def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text=""" - <link - rel="webmention" - href="/webmention" - > - """, - ) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint( - client, "https://example.com/post" - ) - - assert endpoint == "https://example.com/webmention" - assert [request.method for request in requests] == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - "failure", - [ - pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), - pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), - ], -) -def test_discovery_falls_back_to_get_after_head_request_error( - app: Flask, sender_module: ModuleType, failure: httpx.RequestError -) -> None: - methods: list[str] = [] - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - - if request.method == "HEAD": - raise failure - - return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert methods == ["HEAD", "GET"] - - -def test_discovery_falls_back_to_get_after_head_dns_failure( - app: Flask, sender_module: ModuleType -) -> None: - methods: list[str] = [] - - def request_hook(request: httpx.Request) -> None: - if request.method == "HEAD": - raise sender_module.TemporarySenderError("HEAD DNS lookup failed") - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), - event_hooks={"request": [request_hook]}, - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert methods == ["GET"] - - -def test_discovery_get_error_after_head_error_is_not_suppressed( - app: Flask, sender_module: ModuleType -) -> None: - methods: list[str] = [] - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - raise httpx.ConnectError(f"{request.method} failed") - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - pytest.raises(httpx.ConnectError, match="GET failed"), - ): - sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert methods == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - "invalid_href", - [ - pytest.param("javascript:alert(1)", id="unsupported-scheme"), - pytest.param("http://[invalid", id="malformed-url"), - ], -) -def test_discovery_skips_invalid_html_candidate( - app: Flask, sender_module: ModuleType, invalid_href: str -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text=( - f'<link rel="webmention" href="{invalid_href}">' - '<a rel="webmention" href="/webmention">Valid endpoint</a>' - ), - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - - -def test_discovery_skips_invalid_link_header_candidate( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - headers=[ - ("Link", "<mailto:invalid@example.com>; rel=webmention"), - ("Link", "<http://[invalid>; rel=webmention"), - ("Link", "</webmention>; rel=webmention"), - ], - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - - -def test_discovery_returns_none_when_all_html_candidates_are_invalid( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"}) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text='<link rel="webmention" href="javascript:void(0)">', - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint is None - - def test_send_webmention_persists_success( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1315,267 +630,6 @@ def test_send_webmention_ignores_unknown_identifier( assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text -def test_resolve_endpoint_rejects_invalid_url( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - is_http_url = Mock(return_value=False) - monkeypatch.setattr(sender_module, "is_http_url", is_http_url) - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - with pytest.raises( - sender_module.PermanentSenderError, match="Invalid Webmention endpoint" - ): - sender_module.resolve_endpoint(response, "/webmention") - is_http_url.assert_called_once_with(ENDPOINT_URL) - - -def test_endpoint_from_headers_ignores_malformed_link( - sender_module: ModuleType, -) -> None: - response = httpx.Response( - 200, - headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")], - request=httpx.Request("GET", TARGET_URL), - ) - assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL - - -def test_endpoint_from_html_accepts_string_rel( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - element = Mock() - element.get.side_effect = lambda name: { - "rel": "webmention alternate", - "href": "/webmention", - }[name] - - document = Mock() - document.find_all.return_value = [element] - - beautiful_soup = Mock(return_value=document) - monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) - - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - - assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL - - -def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - missing_rel = Mock() - missing_rel.get.return_value = None - - invalid_href = Mock() - invalid_href.get.side_effect = lambda name: { - "rel": ["webmention"], - "href": ["invalid"], - }[name] - - document = Mock() - document.find_all.return_value = [missing_rel, invalid_href] - - beautiful_soup = Mock(return_value=document) - monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) - - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - - assert sender_module.endpoint_from_html(response, b"") is None - - -def test_read_target_body_rejects_declared_oversized_body( - app: Flask, sender_module: ModuleType -) -> None: - app.config["MAX_TARGET_BYTES"] = 10 - - response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) - - with ( - app.app_context(), - pytest.raises( - sender_module.PermanentSenderError, match="Target document is too large" - ), - ): - sender_module.read_target_body(response) - - -def test_read_target_body_ignores_invalid_content_length( - app: Flask, sender_module: ModuleType -) -> None: - response = httpx.Response( - 200, headers={"Content-Length": "invalid"}, content=b"Document" - ) - - with app.app_context(): - body = sender_module.read_target_body(response) - - assert body == b"Document" - - -def test_read_target_body_rejects_streamed_oversized_body( - app: Flask, sender_module: ModuleType -) -> None: - app.config["MAX_TARGET_BYTES"] = 10 - - class BodyStream(httpx.SyncByteStream): - def __iter__(self): - yield b"x" * 6 - yield b"x" * 6 - - response = httpx.Response(200, stream=BodyStream()) - - with ( - app.app_context(), - pytest.raises( - sender_module.PermanentSenderError, match="Target document is too large" - ), - ): - sender_module.read_target_body(response) - - -def test_discovery_uses_link_header_from_get( - app: Flask, sender_module: ModuleType -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert [request.method for request in requests] == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - ("status_code", "exception_name"), - [ - pytest.param(503, "TemporarySenderError", id="temporary"), - pytest.param(404, "PermanentSenderError", id="permanent"), - ], -) -def test_discovery_maps_get_status_to_exception( - app: Flask, sender_module: ModuleType, status_code: int, exception_name: str -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response(status_code) - - exception_type = getattr(sender_module, exception_name) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), - ): - sender_module.discover_webmention_endpoint(client, TARGET_URL) - - -def test_discovery_returns_none_for_unsupported_media_type( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, headers={"Content-Type": "application/json"}, content=b"{}" - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint is None - - -@pytest.mark.parametrize( - ("status_code", "location", "expected_status_url"), - [ - pytest.param( - 201, "/webmention/status/123", STATUS_URL, id="created-with-location" - ), - pytest.param(202, None, None, id="accepted"), - ], -) -def test_post_webmention( - sender_module: ModuleType, - status_code: int, - location: str | None, - expected_status_url: str | None, -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - headers = {} - - if location is not None: - headers["Location"] = location - - return httpx.Response(status_code, headers=headers) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - result = sender_module.post_webmention( - client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL - ) - - assert result == (status_code, expected_status_url) - assert len(requests) == 1 - - request = requests[0] - - assert request.method == "POST" - assert str(request.url) == ENDPOINT_URL - - form = httpx.QueryParams(request.content.decode()) - - assert form["source"] == SOURCE_URL - assert form["target"] == TARGET_URL - - -@pytest.mark.parametrize( - ("code", "location", "expected"), - [ - (201, "/webmention/status/123", "https://example.com/webmention/status/123"), - (201, "https://status.example.org/123", "https://status.example.org/123"), - (201, "status/123", "https://example.com/status/123"), - (201, "javascript:alert(1)", None), - (201, "data:text/html,evil", None), - (201, "ftp://example.com/status", None), - (201, "http://[invalid", None), - (202, "https://example.com/status", None), - ], -) -def test_post_webmention_validates_location( - sender_module: ModuleType, code: int, location: str, expected: str | None -) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(code, headers={"Location": location}) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - result = sender_module.post_webmention( - client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL - ) - - assert result == (code, expected) - - @pytest.mark.parametrize( ("status_url", "expected"), [ diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py new file mode 100644 index 0000000..3c3a93e --- /dev/null +++ b/tests/tasks/test_source_parser.py @@ -0,0 +1,1008 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from pytest import MonkeyPatch + +from webmentions_ssg.tasks import source_parser + +from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post + + +@pytest.fixture +def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + _ = app + from webmentions_ssg.tasks import source_parser + + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) + return source_parser + + +def test_parse_entry_returns_microformats_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + entry = source_parser_module.parse_entry(element, SOURCE_URL) + + assert "h-entry" in entry["type"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article> + <p>Hello world</p> + </article> + """ + ) + + element = document.find("article") + + assert element is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one parsed h-entry", + ): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry" id="source"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <p class="p-name"> + Example post + </p> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element, entry = source_parser_module.primary_entry(document, SOURCE_URL) + + assert element["id"] == "source" + assert entry["properties"]["name"] == ["Example post"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_primary_entry_rejects_missing_h_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <main> + <p>No microformats here.</p> + </main> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 0", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_multiple_h_entries( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + First + </a> + </article> + + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Second + </a> + </article> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/nested/" + > + Nested + </a> + </article> + </div> + </article> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nonmatching_u_url( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Other + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content" id="content"> + Hello world + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + content = source_parser_module.content_element(entry) + + assert content["id"] == "content" + + +def test_content_element_rejects_missing_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 0", + ): + source_parser_module.content_element(entry) + + +def test_content_element_rejects_multiple_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + First + </div> + + <div class="e-content"> + Second + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 2", + ): + source_parser_module.content_element(entry) + + +def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result == SOURCE_URL + + +def test_canonical_url_returns_none_when_missing( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <html> + <head> + <title>Example</title> + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result is None + + +def test_canonical_url_resolves_relative_url_with_base_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.side_effect = [False, True] + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + assert result == ("https://dennisfink.me/blog/example/canonical/") + + +def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="relative canonical" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + +def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href=" " + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="empty canonical URL" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_scan_source_uses_canonical_without_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + assert len(scanned.content_hash) == 32 + + +def test_scan_source_ignores_markup_noise_in_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "<p>Hello <strong>world</strong>!</p>" + '<a href="https://example.com/target">Target</a>', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '<p class="generated" data-build="123"> Hello world! </p>' + '<a class="external" href="https://example.com/target"> Target </a>', + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">') + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '<div class="e-content">', + '<a class="u-like-of" href="https://example.com/first">Like</a>' + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + first_link = '<a class="u-like-of" href="https://example.com/a">A</a>' + second_link = '<a class="u-like-of" href="https://example.com/b">B</a>' + path.write_text( + html.replace( + '<div class="e-content">', + first_link + second_link + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_falls_back_to_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <a href="https://example.com/target"> + Target + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, root=tmp_path, base_url=BASE_URL + ) + + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_requires_canonical_or_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <p>Hello world.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"): + source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + +def test_scan_source_extracts_only_content_links( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a href="https://example.com/metadata"> + Metadata link + </a> + + <div class="e-content"> + <a href="https://example.com/content"> + Content link + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/content"}) + + +def test_scan_source_extracts_relative_content_link( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="../other/"> + Other post + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) + + +@pytest.mark.parametrize( + "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") +) +def test_scan_source_extracts_reaction_properties( + source_parser_module: ModuleType, tmp_path: Path, property_name: str +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + target = f"https://example.com/{property_name}" + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a + class="u-{property_name}" + href="{target}" + > + Reaction target + </a> + + <div class="e-content"> + <p>Post content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({target}) + + +@pytest.mark.parametrize("tag", ("link", "area")) +def test_scan_source_ignores_non_anchor_href_elements( + source_parser_module: ModuleType, tmp_path: Path, tag: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <{tag} href="https://example.com/target"> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset() + + +def test_scan_source_ignores_self_fragment( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="#section"> + Section + </a> + + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_rejects_symlink_outside_root_before_read( + source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + root = tmp_path / "site" + root.mkdir() + external = write_post(tmp_path / "outside", "private", "Sensitive content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + + read_bytes = Mock(side_effect=AssertionError("External file must not be read")) + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + with pytest.raises( + source_parser_module.SourceScanError, match="escapes source directory" + ): + source_parser_module.scan_source_file(linked, root=root, base_url=None) + + read_bytes.assert_not_called() + + +def test_scan_source_accepts_symlink_within_root( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + root = tmp_path / "site" + root.mkdir() + linked = write_post( + root, "example", '<a href="https://example.com/target">Target</a>' + ) + shared = root / "shared.html" + linked.rename(shared) + linked.symlink_to(shared) + + scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://dennisfink.me/blog/other/", + "https://www.dennisfink.me/blog/other/", + "https://webmentions.dennisfink.me/endpoint", + "https://foo.bar.dennisfink.me/example", + ), +) +def test_scan_source_ignores_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Ignored + </a> + + <a href="https://example.com/target"> + External + </a> + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://notdennisfink.me/example", + "https://dennisfink.me.example.com/example", + "https://example.com/example", + ), +) +def test_scan_source_keeps_nonmatching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({href}) + + +def test_scan_source_ignores_reaction_to_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a + class="u-url" + href="{SOURCE_URL}" + > + Permalink + </a> + + <a + class="u-in-reply-to" + href="https://www.dennisfink.me/blog/other/" + > + Reply target + </a> + + <div class="e-content"> + <p>Reply content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset() + + +def test_canonical_url_rejects_non_http_resolved_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="mailto:example@example.com" + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Canonical URL is not a valid HTTP or HTTPS URL", + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_property_urls_ignores_non_mapping_properties( + source_parser_module: ModuleType, +) -> None: + entry = {"properties": "invalid"} + + assert tuple(source_parser_module.property_urls(entry, "url")) == () + + +@pytest.mark.parametrize( + ("parsed", "message"), + [ + pytest.param( + [], + "Microformats parser did not return a document object", + id="non-object-document", + ), + pytest.param( + {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" + ), + pytest.param( + {"items": [{"type": ["h-card"]}]}, + "Parsed microformats item is not an h-entry", + id="wrong-item-type", + ), + ], +) +def test_parse_entry_rejects_invalid_parser_output( + source_parser_module: ModuleType, + monkeypatch: MonkeyPatch, + parsed: object, + message: str, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed) + + with pytest.raises(source_parser_module.SourceScanError, match=message): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_normalize_target_ignores_empty_target( + source_parser_module: ModuleType, +) -> None: + assert ( + source_parser_module.normalize_target( + " ", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) + + +def test_normalize_target_ignores_url_rejected_by_validator( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + assert ( + source_parser_module.normalize_target( + "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) diff --git a/tests/test_auth.py b/tests/test_auth.py index a6117b2..3b87923 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -4,10 +4,11 @@ """Test the session user loader.""" +import pytest from flask import Flask from webmentions_ssg import DATABASE as db -from webmentions_ssg.auth import AUTH, load_user +from webmentions_ssg.auth import AUTH, is_safe_next_url, load_user from webmentions_ssg.models import User @@ -89,3 +90,33 @@ def test_password_change_invalidates_existing_session(app: Flask) -> None: ) assert response.status_code == 302 assert client.get("/received").status_code == 200 + + +@pytest.mark.parametrize( + ("next_url", "expected"), + [ + pytest.param("/", True, id="root"), + pytest.param("/received", True, id="local-path"), + pytest.param("/received?page=2", True, id="query-string"), + pytest.param("/received#details", True, id="fragment"), + pytest.param( + "/sent?next=https://evil.example", True, id="external-url-in-query" + ), + pytest.param("/foo//bar", True, id="double-slash-within-path"), + pytest.param("", False, id="empty"), + pytest.param("received", False, id="relative-path"), + pytest.param(" /received", False, id="leading-space"), + pytest.param("https://evil.example/phishing", False, id="absolute-https"), + pytest.param("http://evil.example", False, id="absolute-http"), + pytest.param("javascript:alert(1)", False, id="non-http-scheme"), + pytest.param("//evil.example/phishing", False, id="scheme-relative"), + pytest.param("///evil.example/phishing", False, id="triple-slash"), + pytest.param("////evil.example", False, id="four-slashes"), + pytest.param("//", False, id="two-slashes"), + pytest.param(r"/\evil.example", False, id="slash-backslash-host"), + pytest.param(r"/received\evil.example", False, id="backslash-in-path"), + pytest.param(r"\/evil.example", False, id="backslash-slash-host"), + ], +) +def test_is_safe_next_url(next_url: str, expected: bool) -> None: + assert is_safe_next_url(next_url) is expected diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..f6b7051 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,650 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import logging +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch + +import pytest +import sqlalchemy as sa +from alembic.script import ScriptDirectory +from flask import Flask +from flask_migrate import upgrade as migrate_upgrade +from sqlalchemy import inspect +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import DATABASE, MIGRATE, create_app +from webmentions_ssg.config import TestingConfig +from webmentions_ssg.models import User + + +def test_cli_groups_are_registered(app: Flask) -> None: + commands = app.cli.commands + assert {"users", "mail"}.issubset(commands) + + +@pytest.fixture +def database_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]: + monkeypatch.chdir(tmp_path) + + class DatabaseTestingConfig(TestingConfig): + LOG_FILE = str(tmp_path / "database-cli.log") + SQLALCHEMY_DATABASE_URI = f"sqlite:///{tmp_path / 'database.sqlite'}" + + app = create_app(DatabaseTestingConfig) + yield app + + with app.app_context(): + DATABASE.session.remove() + DATABASE.engine.dispose() + + +def _script_directory() -> ScriptDirectory: + """Open the migration scripts configured for this application.""" + return ScriptDirectory(str(MIGRATE.directory)) + + +def _head_revision(app: Flask) -> str: + with app.app_context(): + revision = _script_directory().get_current_head() + assert revision is not None, "Expected a migration head" + return revision + + +def _database_revision(app: Flask) -> str: + with app.app_context(), DATABASE.engine.connect() as connection: + return connection.execute( + sa.text("SELECT version_num FROM alembic_version") + ).scalar_one() + + +def test_database_commands_are_registered_without_hiding_flask_migrate( + database_app: Flask, +) -> None: + runner = database_app.test_cli_runner() + + result = runner.invoke(args=["db", "--help"]) + assert result.exit_code == 0, result.output + assert "create-db" in result.output + assert "upgrade" in result.output + + result = runner.invoke(args=["migrate", "--help"]) + assert result.exit_code == 0, result.output + assert "downgrade" in result.output + assert "revision" in result.output + + +def test_create_db_builds_schema_and_records_latest_migration( + database_app: Flask, +) -> None: + result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) + + assert result.exit_code == 0, result.output + assert "Database created" in result.output + assert _database_revision(database_app) == _head_revision(database_app) + + with database_app.app_context(): + inspector = inspect(DATABASE.engine) + table_names = set(inspector.get_table_names()) + columns = { + column["name"] for column in inspector.get_columns("sent_webmentions") + } + received_columns = { + column["name"] for column in inspector.get_columns("received_webmentions") + } + received_indexes = { + index["name"] for index in inspector.get_indexes("received_webmentions") + } + assert {"alembic_version", "users", "received_webmentions"} <= table_names + assert {"attempt_count", "next_retry_at"} <= columns + assert "recovery_attempts" in received_columns + assert "ix_received_webmentions_status_updated_at" in received_indexes + + +def test_create_db_refuses_existing_database_without_modifying_it( + database_app: Flask, +) -> None: + runner = database_app.test_cli_runner() + assert runner.invoke(args=["db", "create-db"]).exit_code == 0 + revision_before = _database_revision(database_app) + + result = runner.invoke(args=["db", "create-db"]) + + assert result.exit_code != 0 + assert "already contains tables" in result.output + assert _database_revision(database_app) == revision_before + + +def test_create_db_refuses_unversioned_existing_tables(database_app: Flask) -> None: + with database_app.app_context(), DATABASE.engine.begin() as connection: + connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)")) + + result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) + + assert result.exit_code != 0 + assert "already contains tables" in result.output + with database_app.app_context(): + assert "legacy_data" in inspect(DATABASE.engine).get_table_names() + assert "alembic_version" not in inspect(DATABASE.engine).get_table_names() + + +def test_upgrade_requires_an_initialized_database(database_app: Flask) -> None: + result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) + + assert result.exit_code != 0 + assert "flask db create-db" in result.output + + +def test_upgrade_refuses_unversioned_existing_database(database_app: Flask) -> None: + with database_app.app_context(), DATABASE.engine.begin() as connection: + connection.execute(sa.text("CREATE TABLE legacy_data (id INTEGER PRIMARY KEY)")) + + result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) + + assert result.exit_code != 0 + assert "No Flask-Migrate revision table" in result.output + with database_app.app_context(): + assert "legacy_data" in inspect(DATABASE.engine).get_table_names() + + +def test_upgrade_applies_pending_migrations_and_preserves_data( + database_app: Flask, +) -> None: + # Start with the first migration, as if this were an older deployment. + with database_app.app_context(): + base_revision = _script_directory().get_base() + assert base_revision is not None + assert base_revision != _head_revision(database_app) + migrate_upgrade(revision=base_revision) + + with DATABASE.engine.begin() as connection: + connection.execute( + sa.text( + "INSERT INTO users (username, password_hash) " + "VALUES (:username, :password_hash)" + ), + {"username": "existing-user", "password_hash": "existing-hash"}, + ) + + result = database_app.test_cli_runner().invoke(args=["db", "upgrade"]) + + assert result.exit_code == 0, result.output + assert "Database upgraded" in result.output + assert _database_revision(database_app) == _head_revision(database_app) + + with database_app.app_context(), DATABASE.engine.connect() as connection: + assert ( + connection.execute( + sa.text("SELECT password_hash FROM users WHERE username = :username"), + {"username": "existing-user"}, + ).scalar_one() + == "existing-hash" + ) + assert "sent_webmentions" in inspect(connection).get_table_names() + + +def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None: + runner = database_app.test_cli_runner() + assert runner.invoke(args=["db", "create-db"]).exit_code == 0 + + result = runner.invoke(args=["db", "upgrade"]) + + assert result.exit_code == 0, result.output + assert _database_revision(database_app) == _head_revision(database_app) + + +def test_create_db_does_not_report_success_when_migration_fails( + database_app: Flask, monkeypatch: pytest.MonkeyPatch +) -> None: + import webmentions_ssg.cli as database_cli + + def fail_upgrade() -> None: + raise RuntimeError("simulated migration failure") + + monkeypatch.setattr(database_cli, "migrate_upgrade", fail_upgrade) + + result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) + + assert result.exit_code != 0 + assert isinstance(result.exception, RuntimeError) + assert "Database created" not in result.output + + +def test_create_db_preserves_logging( + database_app: Flask, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING): + result = database_app.test_cli_runner().invoke(args=["db", "create-db"]) + + assert result.exit_code == 0, result.output + + database_app.logger.warning("Logging still works after migration") + + assert "Logging still works after migration" in caplog.text + + +def test_upgrade_backfills_portable_digest_for_existing_received_rows( + database_app: Flask, +) -> None: + """Upgrade a pre-fingerprint SQLite database without losing historical rows.""" + from webmentions_ssg.models import received_url_digest + + with database_app.app_context(): + migrate_upgrade(revision="c0272f25ce17") + with DATABASE.engine.begin() as connection: + connection.execute( + sa.text( + "INSERT INTO received_webmentions " + "(uuid, source, target, status, updated_at) " + "VALUES (:uuid, :source, :target, :status, :updated_at)" + ), + { + "uuid": "0" * 31 + "1", + "source": "https://source.example/old", + "target": "https://dennisfink.me/target", + "status": "verified", + "updated_at": "2026-01-01 00:00:00", + }, + ) + migrate_upgrade() + with DATABASE.engine.connect() as connection: + result = connection.execute( + sa.text( + "SELECT source, target, status, url_digest " + "FROM received_webmentions" + ) + ).one() + assert result.status == "verified" + recovery_attempts = connection.execute( + sa.text("SELECT recovery_attempts FROM received_webmentions") + ).scalar_one() + assert recovery_attempts == 0 + assert result.url_digest == received_url_digest( + result.source, result.target + ) + constraints = inspect(connection).get_unique_constraints( + "received_webmentions" + ) + assert any( + constraint["column_names"] == ["url_digest"] + for constraint in constraints + ) + + +def test_mail_cli_sends_to_configured_notification_email(app: Flask) -> None: + app.config.update( + NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" + ) + + with patch("webmentions_ssg.MAIL.send") as send: + result = app.test_cli_runner().invoke(args=["mail", "test"]) + + assert result.exit_code == 0, result.output + send.assert_called_once() + message = send.call_args.args[0] + assert message.sender == "sender@example.com" + assert message.recipients == ["owner@example.com"] + assert "test email" in message.subject.lower() + assert "outgoing mail configuration" in message.body + assert "Test email sent to owner@example.com." in result.output + + +def test_mail_cli_overrides_configured_recipient(app: Flask) -> None: + app.config.update( + NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" + ) + + with patch("webmentions_ssg.MAIL.send") as send: + result = app.test_cli_runner().invoke( + args=["mail", "test", "--to", "other@example.com"] + ) + + assert result.exit_code == 0, result.output + send.assert_called_once() + assert send.call_args.args[0].recipients == ["other@example.com"] + + +def test_mail_cli_requires_recipient(app: Flask) -> None: + app.config.update(NOTIFICATION_EMAIL=None, MAIL_DEFAULT_SENDER="sender@example.com") + + with patch("webmentions_ssg.MAIL.send") as send: + result = app.test_cli_runner().invoke(args=["mail", "test"]) + + assert result.exit_code != 0 + assert "Set NOTIFICATION_EMAIL or pass --to" in result.output + send.assert_not_called() + + +def test_mail_cli_requires_default_sender(app: Flask) -> None: + app.config.update(NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER=None) + + with patch("webmentions_ssg.MAIL.send") as send: + result = app.test_cli_runner().invoke(args=["mail", "test"]) + + assert result.exit_code != 0 + assert "MAIL_DEFAULT_SENDER is not configured" in result.output + send.assert_not_called() + + +def test_mail_cli_reports_smtp_failure(app: Flask) -> None: + app.config.update( + NOTIFICATION_EMAIL="owner@example.com", MAIL_DEFAULT_SENDER="sender@example.com" + ) + + with patch( + "webmentions_ssg.MAIL.send", side_effect=ConnectionError("SMTP unavailable") + ) as send: + result = app.test_cli_runner().invoke(args=["mail", "test"]) + + assert result.exit_code != 0 + assert "Could not send test email: SMTP unavailable" in result.output + send.assert_called_once() + + +def _add_user(app: Flask, username: str, password: str = "original") -> None: + """Persist a user using the same model as the CLI.""" + with app.app_context(): + DATABASE.session.add(User(username=username, password=password)) + DATABASE.session.commit() + + +def _get_user(app: Flask, username: str) -> User | None: + with app.app_context(): + return DATABASE.session.scalar(sa.select(User).where(User.username == username)) + + +def test_users_cli_help_lists_management_commands(app: Flask) -> None: + result = app.test_cli_runner().invoke(args=["users", "--help"]) + + assert result.exit_code == 0, result.output + for command in ("create", "list", "set-password", "delete"): + assert command in result.output + + +def test_users_cli_create_with_prompts_hashes_password(app: Flask) -> None: + result = app.test_cli_runner().invoke( + args=["users", "create"], + input="alice\nmy new secret passphrase\nmy new secret passphrase\n", + ) + + assert result.exit_code == 0, result.output + assert "Created user 'alice'." in result.output + assert "my new secret passphrase" not in result.output + user = _get_user(app, "alice") + assert user is not None + assert user.check_password("my new secret passphrase") + assert user.password_hash != "my new secret passphrase" + + +def test_users_cli_create_with_username_option(app: Flask) -> None: + result = app.test_cli_runner().invoke( + args=["users", "create", "--username", "alice"], + input="my new secret passphrase\nmy new secret passphrase\n", + ) + + assert result.exit_code == 0, result.output + assert _get_user(app, "alice") is not None + + +def test_users_cli_create_rejects_duplicate(app: Flask) -> None: + _add_user(app, "alice") + + result = app.test_cli_runner().invoke( + args=[ + "users", + "create", + "--username", + "alice", + "--password", + "a stronger duplicate password", + ] + ) + + assert result.exit_code != 0 + assert "User 'alice' already exists." in result.output + user = _get_user(app, "alice") + assert user is not None + assert user.check_password("original") + + +@pytest.mark.parametrize("username", ["", " "]) +def test_users_cli_create_rejects_invalid_username(app: Flask, username: str) -> None: + result = app.test_cli_runner().invoke( + args=[ + "users", + "create", + "--username", + username, + "--password", + "a strong password for testing", + ] + ) + + assert result.exit_code != 0 + assert "Username must not" in result.output + + +def test_users_cli_create_rejects_empty_password(app: Flask) -> None: + result = app.test_cli_runner().invoke( + args=["users", "create", "--username", "alice", "--password", ""] + ) + + assert result.exit_code != 0 + assert "Password must not be empty." in result.output + assert _get_user(app, "alice") is None + + +def test_users_cli_list_empty(app: Flask) -> None: + result = app.test_cli_runner().invoke(args=["users", "list"]) + + assert result.exit_code == 0, result.output + assert "No users found." in result.output + + +def test_users_cli_list_is_sorted_and_excludes_passwords(app: Flask) -> None: + _add_user(app, "zoe") + _add_user(app, "alice") + + result = app.test_cli_runner().invoke(args=["users", "list"]) + + assert result.exit_code == 0, result.output + lines = result.output.splitlines() + assert lines[0] == "ID\tUSERNAME" + assert lines[1].endswith("\talice") + assert lines[2].endswith("\tzoe") + assert "original" not in result.output + + +def test_users_cli_set_password_updates_hash(app: Flask) -> None: + _add_user(app, "alice") + old_hash = _get_user(app, "alice").password_hash # type: ignore[union-attr] + + result = app.test_cli_runner().invoke( + args=["users", "set-password", "alice"], + input="a long replacement passphrase\na long replacement passphrase\n", + ) + + assert result.exit_code == 0, result.output + assert "Updated password for user 'alice'." in result.output + assert "a long replacement passphrase" not in result.output + user = _get_user(app, "alice") + assert user is not None + assert user.password_hash != old_hash + assert not user.check_password("original") + assert user.check_password("a long replacement passphrase") + + +def test_users_cli_set_password_missing_user(app: Flask) -> None: + result = app.test_cli_runner().invoke( + args=[ + "users", + "set-password", + "missing", + "--password", + "a long replacement passphrase", + ] + ) + + assert result.exit_code != 0 + assert "User 'missing' does not exist." in result.output + + +def test_users_cli_set_password_rejects_empty_password(app: Flask) -> None: + _add_user(app, "alice") + + result = app.test_cli_runner().invoke( + args=["users", "set-password", "alice", "--password", ""] + ) + + assert result.exit_code != 0 + assert "Password must not be empty." in result.output + user = _get_user(app, "alice") + assert user is not None + assert user.check_password("original") + + +def test_users_cli_delete_with_confirmation(app: Flask) -> None: + _add_user(app, "alice") + + result = app.test_cli_runner().invoke( + args=["users", "delete", "alice"], input="y\n" + ) + + assert result.exit_code == 0, result.output + assert "Delete user 'alice'?" in result.output + assert "Deleted user 'alice'." in result.output + assert _get_user(app, "alice") is None + + +def test_users_cli_delete_confirmation_rejected(app: Flask) -> None: + _add_user(app, "alice") + + result = app.test_cli_runner().invoke( + args=["users", "delete", "alice"], input="n\n" + ) + + assert result.exit_code != 0 + assert "Aborted" in result.output + assert _get_user(app, "alice") is not None + + +def test_users_cli_delete_yes_only_deletes_target(app: Flask) -> None: + _add_user(app, "alice") + _add_user(app, "bob") + + result = app.test_cli_runner().invoke(args=["users", "delete", "alice", "--yes"]) + + assert result.exit_code == 0, result.output + assert "Deleted user 'alice'." in result.output + assert _get_user(app, "alice") is None + assert _get_user(app, "bob") is not None + + +def test_users_cli_delete_missing_user(app: Flask) -> None: + result = app.test_cli_runner().invoke(args=["users", "delete", "missing", "-y"]) + + assert result.exit_code != 0 + assert "User 'missing' does not exist." in result.output + + +def test_users_cli_delete_missing_blank_username(app: Flask) -> None: + result = app.test_cli_runner().invoke(args=["users", "delete", " ", "--yes"]) + + assert result.exit_code != 0 + assert "User ' ' does not exist." in result.output + + +def test_users_cli_can_manage_legacy_whitespace_username(app: Flask) -> None: + _add_user(app, " alice ") + + reset = app.test_cli_runner().invoke( + args=[ + "users", + "set-password", + " alice ", + "--password", + "a long replacement passphrase", + ] + ) + + assert reset.exit_code == 0, reset.output + user = _get_user(app, " alice ") + assert user is not None + assert user.check_password("a long replacement passphrase") + + deleted = app.test_cli_runner().invoke(args=["users", "delete", " alice ", "--yes"]) + assert deleted.exit_code == 0, deleted.output + assert _get_user(app, " alice ") is None + + +def test_users_cli_create_rolls_back_on_duplicate_race(app: Flask) -> None: + database_error = IntegrityError("INSERT", {}, Exception("duplicate")) + + with app.app_context(): + with ( + patch.object(DATABASE.session, "commit", side_effect=database_error), + patch.object(DATABASE.session, "rollback") as rollback, + ): + result = app.test_cli_runner().invoke( + args=[ + "users", + "create", + "--username", + "alice", + "--password", + "a long secret for testing", + ] + ) + rollback.assert_called_once() + + assert result.exit_code != 0 + assert "User 'alice' already exists." in result.output + + +@pytest.mark.parametrize( + ("password", "error"), + [ + ("short", "at least 8 characters"), + ("good passphrase with\tab", "control characters"), + ], +) +@pytest.mark.parametrize("operation", ["create", "set-password"]) +def test_users_cli_enforces_password_policy( + app: Flask, password: str, error: str, operation: str +) -> None: + _add_user(app, "existing") + if operation == "create": + command = ["users", "create", "--username", "new"] + else: + command = ["users", "set-password", "existing"] + + result = app.test_cli_runner().invoke(args=[*command, "--password", password]) + assert result.exit_code != 0 + assert error in result.output + assert _get_user(app, "new") is None + existing = _get_user(app, "existing") + assert existing is not None and existing.check_password("original") + + +def test_users_cli_accepts_long_unicode_passphrase(app: Flask) -> None: + passphrase = "🦊 several words of my passphrase 🪁" + result = app.test_cli_runner().invoke( + args=["users", "create", "--username", "alice", "--password", passphrase] + ) + assert result.exit_code == 0, result.output + user = _get_user(app, "alice") + assert user is not None and user.check_password(passphrase) + + +@pytest.mark.parametrize("passphrase", ["12345678", " " * 8, "🦊" + "a" * 1025]) +def test_users_cli_accepts_minimum_whitespace_and_unbounded_passwords( + app: Flask, passphrase: str +) -> None: + result = app.test_cli_runner().invoke( + args=["users", "create", "--username", "alice", "--password", passphrase] + ) + assert result.exit_code == 0, result.output + user = _get_user(app, "alice") + assert user is not None and user.check_password(passphrase) diff --git a/tests/test_config.py b/tests/test_config.py index cc58f73..7bec8b6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,10 +6,9 @@ from pathlib import Path +import pytest from flask import request from flask_limiter.util import get_remote_address - -import pytest from wtforms import Field, ValidationError from wtforms.form import BaseForm @@ -20,7 +19,7 @@ from webmentions_ssg.config import ( ProductionConfig, TestingConfig, ) -from webmentions_ssg.forms.validators import AllowedHostname +from webmentions_ssg.forms import AllowedHostname @pytest.mark.parametrize( @@ -100,6 +99,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None: for config in (DefaultConfig, DevelopmentConfig, ProductionConfig, TestingConfig): assert config.MAX_REDIRECTS == 20 assert config.MAX_SOURCE_BYTES == 1_000_000 + assert config.MAX_TARGET_BYTES == 1_000_000 assert config.REQUEST_TIMEOUT == 5.0 assert config.REQUEST_TOTAL_TIMEOUT == 30.0 @@ -160,7 +160,7 @@ def test_ignored_hostnames_are_normalized_at_startup(tmp_path: Path) -> None: IGNORED_HOSTNAMES = [" Example.COM. ", "*.EXAMPLE.com.", "example.com"] app = create_app(MixedCaseConfig) - assert app.config["IGNORED_HOSTNAMES"] == ("*.example.com", "example.com") + assert app.config["IGNORED_HOSTNAMES"] == {"*.example.com", "example.com"} @pytest.mark.parametrize( @@ -360,7 +360,7 @@ def test_resource_limits_have_safe_defaults() -> None: @pytest.mark.parametrize( "key", ["MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS"] ) -@pytest.mark.parametrize("value", [None, "100", 0, -1, True, 1.5]) +@pytest.mark.parametrize("value", [None, "100", 0, -1, 1.5]) def test_invalid_resource_limits_fail_at_startup( tmp_path: Path, key: str, value: object ) -> None: @@ -425,7 +425,7 @@ def test_proxy_fix_trusts_only_configured_proxy_hops(tmp_path: Path) -> None: assert response.json == {"ip": "203.0.113.4", "scheme": "http", "host": "localhost"} -@pytest.mark.parametrize("hops", [-1, True, False, None, "1", 1.5]) +@pytest.mark.parametrize("hops", [-1, None, "1", 1.5]) def test_invalid_proxy_fix_hop_counts_fail_at_startup( tmp_path: Path, hops: object ) -> None: diff --git a/tests/forms/test_init.py b/tests/test_forms.py index 42e6a2c..03789da 100644 --- a/tests/forms/test_init.py +++ b/tests/test_forms.py @@ -5,13 +5,31 @@ import pytest from flask import Flask from werkzeug.datastructures import MultiDict +from wtforms import Form, StringField, ValidationError -from webmentions_ssg.forms import AdminActionForm, EndpointForm, LoginForm +from webmentions_ssg.forms import ( + AdminActionForm, + AllowedHostname, + EndpointForm, + LoginForm, + NotEqualTo, + PublicIPLiteral, + URLMaxBytes, +) VALID_SOURCE = "https://source.example/post" VALID_TARGET = "https://dennisfink.me/blog/example/" +class ComparisonForm(Form): + source = StringField("Source") + target = StringField("Target") + + +class URLForm(Form): + url = StringField("URL") + + @pytest.mark.parametrize( ("form_data", "invalid_field", "expected_error"), [ @@ -113,7 +131,7 @@ def test_endpoint_does_not_do_dns_in_web_request( raise AssertionError("No DNS lookup belongs in form validation") monkeypatch.setattr( - "webmentions_ssg.url_security.dns.resolver.resolve_name", unexpected_dns_lookup + "webmentions_ssg.net.dns.resolver.resolve_name", unexpected_dns_lookup ) with app.test_request_context("/endpoint", method="POST"): for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"): @@ -203,3 +221,96 @@ def test_endpoint_accepts_encoded_path_space(app: Flask) -> None: meta={"csrf": False}, ) assert form.validate() + + +def test_not_equal_to_accepts_different_values() -> None: + form = ComparisonForm(MultiDict({"source": "source", "target": "target"})) + NotEqualTo("target")(form, form.source) + + +def test_not_equal_to_rejects_equal_values() -> None: + form = ComparisonForm(MultiDict({"source": "same", "target": "same"})) + with pytest.raises(ValidationError, match="Field must not be equal to target"): + NotEqualTo("target")(form, form.source) + + +def test_not_equal_to_uses_custom_message() -> None: + form = ComparisonForm(MultiDict({"source": "same", "target": "same"})) + with pytest.raises(ValidationError, match="Must differ from Target"): + NotEqualTo("target", "Must differ from %(other_label)s")(form, form.source) + + +def test_not_equal_to_rejects_unknown_field() -> None: + form = ComparisonForm(MultiDict({"source": "source", "target": "target"})) + with pytest.raises(ValidationError, match="Invalid field name 'missing'"): + NotEqualTo("missing")(form, form.source) + + +def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: + app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) + with app.app_context(): + AllowedHostname()(form, form.url) + + +def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: + app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedHostname()(form, form.url) + + +def test_allowed_hostname_uses_custom_message(app: Flask) -> None: + app.config["ALLOWED_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with ( + app.app_context(), + pytest.raises(ValidationError, match="Hostname is not allowed"), + ): + AllowedHostname("Hostname is not allowed")(form, form.url) + + +@pytest.mark.parametrize( + "address", + ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], +) +def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: + form = URLForm(MultiDict({"url": f"https://[{address}]/"})) + with pytest.raises(ValidationError, match="URL must use a public address"): + PublicIPLiteral()(form, form.url) + + +def test_public_ip_literal_accepts_public_ipv6_address() -> None: + form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) + PublicIPLiteral()(form, form.url) + + +def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: + form = URLForm(MultiDict({"url": "http://[::1"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedHostname()(form, form.url) + + +def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: + validator = URLMaxBytes() + form = URLForm(MultiDict({"url": "é" * 3})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 6 + validator(form, form.url) + app.config["MAX_WEBMENTION_URL_BYTES"] = 5 + with pytest.raises(ValidationError, match="5 bytes"): + validator(form, form.url) + + +def test_url_max_bytes_custom_message(app: Flask) -> None: + form = URLForm(MultiDict({"url": "too-long"})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 2 + with pytest.raises(ValidationError, match="URL too long"): + URLMaxBytes(message="URL too long")(form, form.url) + + +def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: + form = URLForm(MultiDict({"url": "\ud800"})) + with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): + URLMaxBytes()(form, form.url) diff --git a/tests/test_init.py b/tests/test_init.py index d690032..0a37786 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -6,8 +6,8 @@ from flask import Flask -from webmentions_ssg import VERSION, HUEY -from webmentions_ssg.url_security import is_http_url +from webmentions_ssg import HUEY, VERSION +from webmentions_ssg.net import is_http_url def test_create_app_registers_blueprint_and_extensions(app: Flask) -> None: diff --git a/tests/test_models.py b/tests/test_models.py index 392b246..26439b9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -7,12 +7,18 @@ from collections.abc import Callable from datetime import UTC, datetime import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError +from webmentions_ssg import DATABASE as db from webmentions_ssg.models import ( ReceivedWebmention, SentWebmention, Source, User, + received_url_digest, + sent_target_digest, uuid7_to_datetime, ) @@ -162,3 +168,144 @@ def test_user_session_id_changes_when_password_hash_changes() -> None: user.password = "a strong passphrase" assert user.password_hash != original_hash assert user.get_id() != original_id + + +def test_received_status_remains_text_with_enum_constants() -> None: + from sqlalchemy import Text + + from webmentions_ssg.models import ReceivedStatus + + assert isinstance(ReceivedWebmention.__table__.c.status.type, Text) + assert ReceivedStatus.VERIFIED == "verified" + assert ReceivedStatus.DELETED == "deleted" + + +def test_received_verification_reset_is_reusable_for_atomic_updates() -> None: + from webmentions_ssg.models import ReceivedStatus + + mention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/post", + status=ReceivedStatus.FAILED, + failure_reason="Old error", + recovery_attempts=3, + ) + assert mention.reset_for_verification() is mention + assert mention.status == ReceivedStatus.RECEIVED + assert mention.failure_reason is None + assert mention.recovery_attempts == 0 + assert mention.updated_at is not None + + +def test_reopen_preserves_delivery_history_unless_explicit_resend() -> None: + from webmentions_ssg.models import SentWebmentionStatus + + sent_at = datetime.now(UTC) + mention = SentWebmention( + target="https://example.com/post", + active=True, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + attempt_count=3, + next_retry_at=sent_at, + last_sent_at=sent_at, + ) + assert mention.reopen(3, False) is mention + assert mention.desired_revision == 3 + assert mention.active is False + assert mention.attempt_count == 0 + assert mention.next_retry_at is None + assert mention.status == SentWebmentionStatus.SENT + assert mention.sent_revision == 2 + + assert mention.reopen(3, True, resend=True) is mention + assert mention.processed_revision is None + assert mention.status is None + assert mention.sent_revision == 2 + assert mention.last_sent_at == sent_at + + +def test_received_pair_fingerprint_is_unambiguous() -> None: + assert received_url_digest("a", "bc") != received_url_digest("ab", "c") + assert received_url_digest("a", "b") != received_url_digest("b", "a") + + +def test_models_hash_url_fields_when_created_or_changed() -> None: + received = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/a", + target="https://site.example/b", + ) + assert received.url_digest == received_url_digest(received.source, received.target) + received.source = "https://source.example/c" + assert received.url_digest == received_url_digest(received.source, received.target) + received.target = "https://site.example/d" + assert received.url_digest == received_url_digest(received.source, received.target) + + sent = SentWebmention(target="https://site.example/a", desired_revision=1) + assert sent.target_digest == sent_target_digest(sent.target) + sent.target = "https://site.example/b" + assert sent.target_digest == sent_target_digest(sent.target) + + +def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: + long_source = "https://source.example/" + "a" * 4000 + long_target = "https://dennisfink.me/" + "b" * 4000 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + db.session.commit() + match = db.session.scalar( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.url_digest + == received_url_digest(long_source, long_target) + ) + ) + assert match is not None + assert match.source == long_source + assert match.target == long_target + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() + + +def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( + app: Flask, +) -> None: + now = datetime.now(UTC) + long_target = "https://destination.example/" + "a" * 4000 + with app.app_context(): + source = Source( + path="long/index.html", + url="https://example.com/long", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + db.session.add(source) + db.session.commit() + assert source.sent_webmentions[0].target_digest == sent_target_digest( + long_target + ) + + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/test_public_transport.py b/tests/test_net.py index df7a1c3..928a0d5 100644 --- a/tests/test_public_transport.py +++ b/tests/test_net.py @@ -4,20 +4,26 @@ """Security regression tests for DNS-pinned network connections.""" +import ipaddress import socket -from unittest.mock import Mock +from unittest.mock import Mock, PropertyMock, patch import httpcore import httpx import pytest +from flask import Flask -from webmentions_ssg.public_transport import ( +from webmentions_ssg.net import ( + AddressResolutionError, DeadlineStream, NetworkDeadline, PublicHTTPTransport, PublicNetworkBackend, + UnsafeAddressError, + is_http_url, + is_public_address, + public_addresses, ) -from webmentions_ssg.url_security import UnsafeAddressError @pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"]) @@ -34,7 +40,7 @@ def test_backend_rejects_private_ip_without_socket( def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None: """The address is resolved exactly once and the socket sees only that IP.""" dns = Mock(return_value=("93.184.216.34",)) - monkeypatch.setattr("webmentions_ssg.public_transport.public_addresses", dns) + monkeypatch.setattr("webmentions_ssg.net.public_addresses", dns) socket_instance = Mock() create_socket = Mock(return_value=socket_instance) monkeypatch.setattr(socket, "socket", create_socket) @@ -110,9 +116,7 @@ def test_deadline_clips_each_read_and_stops_trickle( monkeypatch: pytest.MonkeyPatch, ) -> None: clock = [100.0] - monkeypatch.setattr( - "webmentions_ssg.public_transport.time.monotonic", lambda: clock[0] - ) + monkeypatch.setattr("webmentions_ssg.net.time.monotonic", lambda: clock[0]) fake_stream = Mock() fake_stream.read.return_value = b"x" stream = DeadlineStream(fake_stream, NetworkDeadline(5.0)) @@ -154,3 +158,109 @@ def test_redirects_go_through_same_destination_guard( client.get("http://source.example/post") assert checked_hosts == ["source.example", "internal.example"] + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + pytest.param("http://example.com/", True, id="http"), + pytest.param("https://example.com/path", True, id="https"), + pytest.param("HTTP://EXAMPLE.COM/", True, id="scheme-case-insensitive"), + pytest.param("https://example.com:8443/path", True, id="port"), + pytest.param("http://[2001:db8::1]/", True, id="ipv6"), + pytest.param("ftp://example.com/", False, id="ftp"), + pytest.param("mailto:example@example.com", False, id="mailto"), + pytest.param("/relative/url", False, id="relative"), + pytest.param("//example.com/path", False, id="scheme-relative"), + pytest.param("https:///missing-host", False, id="missing-host"), + pytest.param("", False, id="empty"), + pytest.param("http://[::1", False, id="malformed"), + ], +) +def test_is_http_url(url: str, expected: bool) -> None: + assert is_http_url(url) is expected + + +def test_ip_literal_does_not_use_dns() -> None: + with patch("webmentions_ssg.net.dns.resolver.resolve_name") as dns_resolve: + with pytest.raises(UnsafeAddressError): + public_addresses("169.254.169.254") + with pytest.raises(UnsafeAddressError): + public_addresses("::1") + assert public_addresses("8.8.8.8") == ("8.8.8.8",) + dns_resolve.assert_not_called() + + +@pytest.mark.parametrize( + "address", + [ + pytest.param("64:ff9b::a9fe:a9fe", id="nat64-metadata"), + pytest.param("64:ff9b::808:808", id="nat64-public-ipv4"), + pytest.param("::10.0.0.1", id="ipv4-compatible-private"), + pytest.param("::8.8.8.8", id="ipv4-compatible-public"), + pytest.param("2002:a9fe:a9fe::1", id="6to4-metadata"), + pytest.param("2002:0a00:0001::1", id="6to4-private"), + pytest.param("::ffff:10.0.0.1", id="mapped-private"), + pytest.param("::ffff:169.254.169.254", id="mapped-metadata"), + ], +) +def test_public_addresses_rejects_ipv6_transition_literals(address: str) -> None: + with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve: + with pytest.raises(UnsafeAddressError, match="Non-public"): + public_addresses(address) + resolve.assert_not_called() + + +@pytest.mark.parametrize( + "address", ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1"] +) +def test_public_addresses_rejects_transition_addresses_in_dns(address: str) -> None: + with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve: + resolve.return_value.addresses.return_value = ["93.184.216.34", address] + with pytest.raises(UnsafeAddressError, match="Non-public"): + public_addresses("transition.example") + + +@pytest.mark.parametrize("address", ["2002:0a00:0001::1", "::ffff:10.0.0.1"]) +def test_private_embedded_ipv4_rejected_even_if_ipv6_is_global(address: str) -> None: + # Simulate a Python version treating the outer transition address as global. + with patch.object( + ipaddress.IPv6Address, "is_global", new_callable=PropertyMock + ) as global_: + global_.return_value = True + assert not is_public_address(ipaddress.ip_address(address)) + + +def test_public_addresses_rejects_mixed_dns_results() -> None: + with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve: + resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"] + with pytest.raises(UnsafeAddressError, match="Non-public"): + public_addresses("rebinding.example", lifetime=1.5) + resolve.assert_called_once_with("rebinding.example", lifetime=1.5) + + +def test_public_addresses_rejects_empty_dns_result() -> None: + with patch("webmentions_ssg.net.dns.resolver.resolve_name") as resolve: + resolve.return_value.addresses.return_value = [] + with pytest.raises(AddressResolutionError): + public_addresses("empty.example") + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + pytest.param(200, False, id="success"), + pytest.param(400, False, id="client-error"), + pytest.param(408, True, id="request-timeout"), + pytest.param(425, True, id="too-early"), + pytest.param(429, True, id="too-many-requests"), + pytest.param(500, True, id="server-error-lower-bound"), + pytest.param(599, True, id="server-error-upper-bound"), + pytest.param(600, False, id="outside-server-error-range"), + ], +) +def test_temporary_http_status(app: Flask, status: int, expected: bool) -> None: + _ = app + from webmentions_ssg.tasks import discovery + + assert discovery.is_temporary_status(status) is expected diff --git a/tests/test_optional_dependencies.py b/tests/test_packaging.py index 40c53c5..34c0d72 100644 --- a/tests/test_optional_dependencies.py +++ b/tests/test_packaging.py @@ -9,7 +9,6 @@ from pathlib import Path import pytest - BACKENDS = { "redis": ["flask-limiter[redis]>=4.1.1", "huey[redis]>=3.3.0"], "valkey": ["flask-limiter[valkey]>=4.1.1", "huey[redis]>=3.3.0"], diff --git a/tests/test_url_digest.py b/tests/test_url_digest.py deleted file mode 100644 index 0c04364..0000000 --- a/tests/test_url_digest.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Test the database-independent Webmention URL fingerprints.""" - -import uuid -from datetime import UTC, datetime - -import pytest -import sqlalchemy as sa -from flask import Flask -from sqlalchemy.exc import IntegrityError - -from webmentions_ssg import DATABASE as db -from webmentions_ssg.models import ReceivedWebmention, SentWebmention, Source -from webmentions_ssg.url_digest import received_url_digest, sent_target_digest - - -def test_received_pair_fingerprint_is_unambiguous() -> None: - assert received_url_digest("a", "bc") != received_url_digest("ab", "c") - assert received_url_digest("a", "b") != received_url_digest("b", "a") - - -def test_models_hash_url_fields_when_created_or_changed() -> None: - received = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/a", - target="https://site.example/b", - ) - assert received.url_digest == received_url_digest(received.source, received.target) - received.source = "https://source.example/c" - assert received.url_digest == received_url_digest(received.source, received.target) - received.target = "https://site.example/d" - assert received.url_digest == received_url_digest(received.source, received.target) - - sent = SentWebmention(target="https://site.example/a", desired_revision=1) - assert sent.target_digest == sent_target_digest(sent.target) - sent.target = "https://site.example/b" - assert sent.target_digest == sent_target_digest(sent.target) - - -def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: - long_source = "https://source.example/" + "a" * 4000 - long_target = "https://dennisfink.me/" + "b" * 4000 - with app.app_context(): - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - db.session.commit() - match = db.session.scalar( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.url_digest - == received_url_digest(long_source, long_target) - ) - ) - assert match is not None - assert match.source == long_source - assert match.target == long_target - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() - - -def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( - app: Flask, -) -> None: - now = datetime.now(UTC) - long_target = "https://destination.example/" + "a" * 4000 - with app.app_context(): - source = Source( - path="long/index.html", - url="https://example.com/long", - content_hash="0" * 32, - revision=1, - last_seen_at=now, - revised_at=now, - ) - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - db.session.add(source) - db.session.commit() - assert source.sent_webmentions[0].target_digest == sent_target_digest( - long_target - ) - - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() diff --git a/tests/test_url_security.py b/tests/test_url_security.py deleted file mode 100644 index 95c2bd9..0000000 --- a/tests/test_url_security.py +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -import ipaddress -from unittest.mock import Mock, PropertyMock, patch - -import dns.resolver -import pytest - -from webmentions_ssg.url_security import ( - AddressResolutionError, - UnsafeAddressError, - is_http_url, - is_public_address, - is_public_url, - is_safe_next_url, - public_addresses, -) - - -@pytest.mark.parametrize( - ("url", "expected"), - [ - pytest.param("http://example.com/", True, id="http"), - pytest.param("https://example.com/path", True, id="https"), - pytest.param("HTTP://EXAMPLE.COM/", True, id="scheme-case-insensitive"), - pytest.param("https://example.com:8443/path", True, id="port"), - pytest.param("http://[2001:db8::1]/", True, id="ipv6"), - pytest.param("ftp://example.com/", False, id="ftp"), - pytest.param("mailto:example@example.com", False, id="mailto"), - pytest.param("/relative/url", False, id="relative"), - pytest.param("//example.com/path", False, id="scheme-relative"), - pytest.param("https:///missing-host", False, id="missing-host"), - pytest.param("", False, id="empty"), - pytest.param("http://[::1", False, id="malformed"), - ], -) -def test_is_http_url(url: str, expected: bool) -> None: - assert is_http_url(url) is expected - - -@pytest.mark.parametrize( - ("next_url", "expected"), - [ - pytest.param("/", True, id="root"), - pytest.param("/received", True, id="local-path"), - pytest.param("/received?page=2", True, id="query-string"), - pytest.param("/received#details", True, id="fragment"), - pytest.param( - "/sent?next=https://evil.example", True, id="external-url-in-query" - ), - pytest.param("/foo//bar", True, id="double-slash-within-path"), - pytest.param("", False, id="empty"), - pytest.param("received", False, id="relative-path"), - pytest.param(" /received", False, id="leading-space"), - pytest.param("https://evil.example/phishing", False, id="absolute-https"), - pytest.param("http://evil.example", False, id="absolute-http"), - pytest.param("javascript:alert(1)", False, id="non-http-scheme"), - pytest.param("//evil.example/phishing", False, id="scheme-relative"), - pytest.param("///evil.example/phishing", False, id="triple-slash"), - pytest.param("////evil.example", False, id="four-slashes"), - pytest.param("//", False, id="two-slashes"), - pytest.param(r"/\evil.example", False, id="slash-backslash-host"), - pytest.param(r"/received\evil.example", False, id="backslash-in-path"), - pytest.param(r"\/evil.example", False, id="backslash-slash-host"), - ], -) -def test_is_safe_next_url(next_url: str, expected: bool) -> None: - assert is_safe_next_url(next_url) is expected - - -@patch("webmentions_ssg.url_security.dns.resolver.resolve_name") -@pytest.mark.parametrize( - ("url", "resolved_addresses"), - [ - ("http://127.0.0.1/", ["127.0.0.1"]), - ("http://127.0.0.1:8080/test", ["127.0.0.1"]), - ("http://[::1]/", ["::1"]), - ("http://10.0.0.1/", ["10.0.0.1"]), - ("http://172.16.0.1/", ["172.16.0.1"]), - ("http://192.168.1.1/", ["192.168.1.1"]), - ("http://169.254.169.254/", ["169.254.169.254"]), - ("http://localhost/", ["127.0.0.1", "::1"]), - ("http://internal.example/", ["192.168.1.10"]), - ("https://example.com/", ["93.184.216.34", "192.168.1.10"]), - ], -) -def test_is_public_url_returns_false_for_non_public_addresses( - resolve_name: Mock, url: str, resolved_addresses: list[str] -) -> None: - resolve_name.return_value.addresses.return_value = resolved_addresses - assert not is_public_url(url) - - -@patch("webmentions_ssg.url_security.dns.resolver.resolve_name") -@pytest.mark.parametrize( - ("url", "resolved_addresses"), - [ - ("https://example.com/", ["93.184.216.34"]), - ( - "https://example.com/", - ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"], - ), - ], -) -def test_is_public_url_returns_true_for_public_addresses( - resolve_name: Mock, url: str, resolved_addresses: list[str] -) -> None: - resolve_name.return_value.addresses.return_value = resolved_addresses - assert is_public_url(url) - - -@patch( - "webmentions_ssg.url_security.dns.resolver.resolve_name", - side_effect=dns.resolver.NXDOMAIN(), -) -def test_is_public_url_raises_for_resolution_failure(resolve_name: Mock) -> None: - with pytest.raises(AddressResolutionError, match="Could not resolve hostname"): - is_public_url("https://nonexistent.example/") - resolve_name.assert_called_once_with("nonexistent.example", lifetime=None) - - -@patch("webmentions_ssg.url_security.dns.resolver.resolve_name") -def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> None: - with pytest.raises(ValueError, match="No hostname was specified"): - is_public_url("/relative/url") - resolve_name.assert_not_called() - - -def test_ip_literal_does_not_use_dns() -> None: - with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as dns_resolve: - with pytest.raises(UnsafeAddressError): - public_addresses("169.254.169.254") - with pytest.raises(UnsafeAddressError): - public_addresses("::1") - assert public_addresses("8.8.8.8") == ("8.8.8.8",) - dns_resolve.assert_not_called() - - -@pytest.mark.parametrize( - "address", - [ - pytest.param("64:ff9b::a9fe:a9fe", id="nat64-metadata"), - pytest.param("64:ff9b::808:808", id="nat64-public-ipv4"), - pytest.param("::10.0.0.1", id="ipv4-compatible-private"), - pytest.param("::8.8.8.8", id="ipv4-compatible-public"), - pytest.param("2002:a9fe:a9fe::1", id="6to4-metadata"), - pytest.param("2002:0a00:0001::1", id="6to4-private"), - pytest.param("::ffff:10.0.0.1", id="mapped-private"), - pytest.param("::ffff:169.254.169.254", id="mapped-metadata"), - ], -) -def test_public_addresses_rejects_ipv6_transition_literals(address: str) -> None: - with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: - with pytest.raises(UnsafeAddressError, match="Non-public"): - public_addresses(address) - resolve.assert_not_called() - - -@pytest.mark.parametrize( - "address", ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1"] -) -def test_public_addresses_rejects_transition_addresses_in_dns(address: str) -> None: - with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: - resolve.return_value.addresses.return_value = ["93.184.216.34", address] - with pytest.raises(UnsafeAddressError, match="Non-public"): - public_addresses("transition.example") - - -@pytest.mark.parametrize("address", ["2002:0a00:0001::1", "::ffff:10.0.0.1"]) -def test_private_embedded_ipv4_rejected_even_if_ipv6_is_global(address: str) -> None: - # Simulate a Python version treating the outer transition address as global. - with patch.object( - ipaddress.IPv6Address, "is_global", new_callable=PropertyMock - ) as global_: - global_.return_value = True - assert not is_public_address(ipaddress.ip_address(address)) - - -def test_public_addresses_rejects_mixed_dns_results() -> None: - with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: - resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"] - with pytest.raises(UnsafeAddressError, match="Non-public"): - public_addresses("rebinding.example", lifetime=1.5) - resolve.assert_called_once_with("rebinding.example", lifetime=1.5) - - -def test_public_addresses_rejects_empty_dns_result() -> None: - with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: - resolve.return_value.addresses.return_value = [] - with pytest.raises(AddressResolutionError): - public_addresses("empty.example") diff --git a/tests/test_views.py b/tests/test_views.py deleted file mode 100644 index 129bdbf..0000000 --- a/tests/test_views.py +++ /dev/null @@ -1,1347 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -import uuid -from datetime import UTC, datetime, timedelta -from types import ModuleType -from unittest.mock import ANY, Mock - -import pytest -import sqlalchemy as sa -from flask import Flask -from flask.testing import FlaskClient -from sqlalchemy.exc import IntegrityError - -from webmentions_ssg import DATABASE as db -from webmentions_ssg.models import ( - ReceivedWebmention, - SentWebmention, - SentWebmentionStatus, - Source, - User, -) - -SOURCE_URL = "https://source.example/post" -TARGET_URL = "https://dennisfink.me/blog/example/" - - -@pytest.fixture -def views_module(app: Flask) -> ModuleType: - _ = app - from webmentions_ssg import views - - return views - - -def mock_endpoint_form( - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - *, - valid: bool, - source: str = SOURCE_URL, - target: str = TARGET_URL, - errors: dict[str, list[str]] | None = None, -) -> Mock: - form = Mock() - form.validate_on_submit.return_value = valid - form.source.data = source - form.target.data = target - form.errors = errors or {} - - monkeypatch.setattr(views_module.forms, "EndpointForm", Mock(return_value=form)) - return form - - -def log_in(app: Flask, client: FlaskClient) -> None: - with app.app_context(): - user = User(username="admin", password="test") - - db.session.add(user) - db.session.commit() - - user_id = user.get_id() - - with client.session_transaction() as session: - session["_user_id"] = str(user_id) - session["_fresh"] = True - - -def create_sent_source(app: Flask) -> uuid.UUID: - now = datetime.now(UTC) - - with app.app_context(): - source = Source( - path="blog/example/index.html", - url="https://dennisfink.me/blog/example/", - content_hash="0" * 32, - revision=2, - last_seen_at=now, - revised_at=now, - ) - - source.sent_webmentions.extend( - [ - SentWebmention( - target="https://example.com/b", - active=True, - desired_revision=2, - processed_revision=2, - sent_revision=2, - status=SentWebmentionStatus.SENT, - endpoint="https://example.com/webmention", - response_status=202, - ), - SentWebmention( - target="https://example.com/a", - active=True, - desired_revision=2, - processed_revision=2, - sent_revision=1, - status=SentWebmentionStatus.FAILED, - failure_reason="Webmention endpoint returned HTTP 400", - endpoint="https://example.com/webmention", - response_status=400, - ), - ] - ) - - db.session.add(source) - db.session.commit() - - return source.uuid - - -def test_endpoint_only_accepts_post(client: FlaskClient) -> None: - response = client.get("/endpoint") - assert response.status_code == 405 - - -def test_endpoint_returns_form_errors( - client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - errors = {"source": ["Invalid source"]} - mock_endpoint_form(views_module, monkeypatch, valid=False, errors=errors) - - response = client.post("/endpoint") - - assert response.status_code == 400 - assert response.get_json() == errors - - -def test_endpoint_creates_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - form = mock_endpoint_form(views_module, monkeypatch, valid=True) - verify_webmention = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) - - response = client.post("/endpoint") - - assert response.status_code == 201 - form.validate_on_submit.assert_called_once_with() - - with app.app_context(): - webmention = db.session.scalar(sa.select(ReceivedWebmention)) - - assert webmention is not None - assert webmention.source == SOURCE_URL - assert webmention.target == TARGET_URL - assert webmention.status == "received" - assert webmention.failure_reason is None - - identifier = webmention.uuid - - assert response.headers["Location"].endswith(str(identifier)) - verify_webmention.assert_called_once_with(identifier) - - -def test_endpoint_is_idempotent( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock_endpoint_form(views_module, monkeypatch, valid=True) - verify_webmention = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) - - first_response = client.post("/endpoint") - second_response = client.post("/endpoint") - - assert first_response.status_code == 201 - assert second_response.status_code == 201 - assert first_response.headers["Location"] == second_response.headers["Location"] - - with app.app_context(): - webmentions = db.session.scalars(sa.select(ReceivedWebmention)).all() - - assert len(webmentions) == 1 - - webmention = webmentions[0] - - assert webmention.source == SOURCE_URL - assert webmention.target == TARGET_URL - assert webmention.status == "received" - assert webmention.failure_reason is None - - identifier = webmention.uuid - - verify_webmention.assert_called_once_with(identifier) - - -def test_resending_resets_failure_state( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock_endpoint_form(views_module, monkeypatch, valid=True) - verify_webmention = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) - - with app.app_context(): - existing = ReceivedWebmention( - uuid=uuid.uuid7(), - source=SOURCE_URL, - target=TARGET_URL, - status="failed", - failure_reason="Previous failure", - updated_at=datetime.now(UTC) - timedelta(minutes=11), - ) - - db.session.add(existing) - db.session.commit() - - identifier = existing.uuid - - response = client.post("/endpoint") - - assert response.status_code == 201 - - with app.app_context(): - webmention = db.session.get(ReceivedWebmention, identifier) - - assert webmention is not None - assert webmention.status == "received" - assert webmention.failure_reason is None - - verify_webmention.assert_called_once_with(identifier) - - -@pytest.mark.parametrize("path", ["/sent", f"/sent/{uuid.uuid7()}"]) -def test_sent_views_require_login(client: FlaskClient, path: str) -> None: - response = client.get(path) - - assert response.status_code == 302 - assert "/login" in response.headers["Location"] - - -def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get("/sent") - - assert response.status_code == 200 - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - assert f"/sent/{identifier}" in html - - -def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get(f"/sent/{identifier}") - - assert response.status_code == 200 - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - - first_target = "https://example.com/a" - second_target = "https://example.com/b" - - assert first_target in html - assert second_target in html - assert html.index(first_target) < html.index(second_target) - - assert "Webmention endpoint returned HTTP 400" in html - assert "https://example.com/webmention" in html - - -def test_sent_source_returns_404_for_unknown_source( - app: Flask, client: FlaskClient -) -> None: - log_in(app, client) - response = client.get(f"/sent/{uuid.uuid7()}") - assert response.status_code == 404 - - -def mock_login_form( - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - *, - valid: bool, - username: str = "admin", - password: str | None = "test", -) -> Mock: - form = Mock() - form.validate_on_submit.return_value = valid - form.username.data = username - form.password.data = password - - monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form)) - return form - - -def mock_admin_action_form( - views_module: ModuleType, monkeypatch: pytest.MonkeyPatch, *, valid: bool -) -> Mock: - form = Mock() - form.validate_on_submit.return_value = valid - - monkeypatch.setattr(views_module.forms, "AdminActionForm", Mock(return_value=form)) - return form - - -def create_received_webmention( - app: Flask, - *, - source: str = SOURCE_URL, - target: str = TARGET_URL, - status: str = "verified", - failure_reason: str | None = None, -) -> uuid.UUID: - with app.app_context(): - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source=source, - target=target, - status=status, - failure_reason=failure_reason, - ) - - db.session.add(webmention) - db.session.commit() - - return webmention.uuid - - -def create_resendable_sent_webmention(app: Flask) -> tuple[uuid.UUID, uuid.UUID]: - now = datetime.now(UTC) - - with app.app_context(): - source = Source( - path="blog/resend/index.html", - url="https://dennisfink.me/blog/resend/", - content_hash="1" * 32, - revision=3, - last_seen_at=now, - revised_at=now, - ) - webmention = SentWebmention( - target="https://example.com/resend", - active=True, - desired_revision=3, - processed_revision=3, - sent_revision=2, - status=SentWebmentionStatus.FAILED, - failure_reason="Previous failure", - endpoint="https://example.com/webmention", - response_status=500, - status_url="https://example.com/webmention/status/123", - ) - source.sent_webmentions.append(webmention) - - db.session.add(source) - db.session.commit() - - return source.uuid, webmention.uuid - - -def test_login_renders_form(client: FlaskClient) -> None: - response = client.get("/login") - - assert response.status_code == 200 - assert "login" in response.get_data(as_text=True).lower() - - -@pytest.mark.parametrize("password", ("wrong-password", None)) -def test_login_rejects_invalid_credentials( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - password: str | None, -) -> None: - with app.app_context(): - db.session.add(User(username="admin", password="test")) - db.session.commit() - - mock_login_form( - views_module, monkeypatch, valid=True, username="admin", password=password - ) - - response = client.post("/login") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/login") - - with client.session_transaction() as session: - assert "_user_id" not in session - assert ("danger", "Invalid username or password") in session["_flashes"] - - -@pytest.mark.parametrize( - "next_url", - [ - pytest.param("/", id="root"), - pytest.param("/received", id="local-path"), - pytest.param("/received?page=2", id="query-string"), - pytest.param("/received#details", id="fragment"), - ], -) -def test_login_redirects_to_local_next_page( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - next_url: str, -) -> None: - with app.app_context(): - user = User(username="admin", password="test") - db.session.add(user) - db.session.commit() - user_id = user.get_id() - - mock_login_form(views_module, monkeypatch, valid=True) - - response = client.post("/login", query_string={"next": next_url}) - - assert response.status_code == 302 - assert response.headers["Location"] == next_url - - with client.session_transaction() as session: - assert session["_user_id"] == str(user_id) - - -@pytest.mark.parametrize( - "next_url", - [ - pytest.param(None, id="missing"), - pytest.param("", id="empty"), - pytest.param("received", id="relative-path"), - pytest.param("https://evil.example/phishing", id="absolute-https"), - pytest.param("http://evil.example", id="absolute-http"), - pytest.param("javascript:alert(1)", id="non-http-scheme"), - pytest.param("//evil.example/phishing", id="scheme-relative"), - pytest.param("///evil.example/phishing", id="triple-slash"), - pytest.param("////evil.example", id="four-slashes"), - pytest.param(r"/\evil.example", id="slash-backslash-host"), - pytest.param(r"/received\evil.example", id="backslash-in-path"), - pytest.param(r"\/evil.example", id="backslash-slash-host"), - ], -) -def test_login_rejects_unsafe_next_page( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - next_url: str | None, -) -> None: - with app.app_context(): - user = User(username="admin", password="test") - db.session.add(user) - db.session.commit() - user_id = user.get_id() - - mock_login_form(views_module, monkeypatch, valid=True) - - query_string = {} if next_url is None else {"next": next_url} - response = client.post("/login", query_string=query_string) - - assert response.status_code == 302 - assert response.headers["Location"] == "/" - - with client.session_transaction() as session: - assert session["_user_id"] == str(user_id) - - -def test_login_redirects_authenticated_user(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - - response = client.get("/login") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/") - - -def test_received_requires_login(client: FlaskClient) -> None: - response = client.get("/received") - - assert response.status_code == 302 - assert "/login" in response.headers["Location"] - - -def test_received_lists_webmentions(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - create_received_webmention(app) - - response = client.get("/received") - - assert response.status_code == 200 - html = response.get_data(as_text=True) - assert SOURCE_URL in html - assert TARGET_URL in html - - -@pytest.mark.parametrize("action", ["delete", "reverify"]) -def test_received_admin_actions_require_login(client: FlaskClient, action: str) -> None: - response = client.post(f"/received/{uuid.uuid7()}/{action}") - - assert response.status_code == 302 - assert "/login" in response.headers["Location"] - - -@pytest.mark.parametrize("action", ["delete", "reverify"]) -def test_received_admin_actions_reject_invalid_form( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - action: str, -) -> None: - log_in(app, client) - identifier = create_received_webmention(app) - form = mock_admin_action_form(views_module, monkeypatch, valid=False) - - response = client.post(f"/received/{identifier}/{action}") - - assert response.status_code == 400 - form.validate_on_submit.assert_called_once_with() - - -@pytest.mark.parametrize("action", ["delete", "reverify"]) -def test_received_admin_actions_return_404_for_unknown_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - action: str, -) -> None: - log_in(app, client) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post(f"/received/{uuid.uuid7()}/{action}") - - assert response.status_code == 404 - - -def test_delete_received_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - identifier = create_received_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post(f"/received/{identifier}/delete?page=3") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/received?page=3") - - with app.app_context(): - assert db.session.get(ReceivedWebmention, identifier) is None - - -def test_reverify_received_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - identifier = create_received_webmention( - app, status="failed", failure_reason="Previous failure" - ) - with app.app_context(): - row = db.session.get(ReceivedWebmention, identifier) - assert row is not None - row.recovery_attempts = 3 - db.session.commit() - mock_admin_action_form(views_module, monkeypatch, valid=True) - verify_webmention = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) - - response = client.post(f"/received/{identifier}/reverify?page=2") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/received?page=2") - - with app.app_context(): - webmention = db.session.get(ReceivedWebmention, identifier) - assert webmention is not None - assert webmention.status == "received" - assert webmention.failure_reason is None - assert webmention.recovery_attempts == 0 - - verify_webmention.assert_called_once_with(identifier) - - -def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: - response = client.post("/sent/rescan") - - assert response.status_code == 302 - assert "/login" in response.headers["Location"] - - -@pytest.mark.parametrize("valid", [False, True]) -def test_rescan_sent_sources( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - valid: bool, -) -> None: - log_in(app, client) - form = mock_admin_action_form(views_module, monkeypatch, valid=valid) - manual_scan_sources = Mock() - monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources) - - response = client.post("/sent/rescan") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/sent") - form.validate_on_submit.assert_called_once_with() - - if valid: - manual_scan_sources.assert_called_once_with() - else: - manual_scan_sources.assert_not_called() - - -def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: - response = client.post(f"/sent/{uuid.uuid7()}/{uuid.uuid7()}/resend") - - assert response.status_code == 302 - assert "/login" in response.headers["Location"] - - -def test_resend_sent_webmention_rejects_invalid_form( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=False) - - response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") - - assert response.status_code == 400 - - -def test_resend_sent_webmention_returns_404_for_unknown_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post(f"/sent/{source_identifier}/{uuid.uuid7()}/resend") - - assert response.status_code == 404 - - -def test_resend_sent_webmention_rejects_wrong_source( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - _, webmention_identifier = create_resendable_sent_webmention(app) - wrong_source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post( - f"/sent/{wrong_source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == 404 - - -def test_resend_sent_webmention_reopens_current_revision( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - send_webmention = Mock() - monkeypatch.setattr(views_module, "send_webmention", send_webmention) - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - webmention.attempt_count = 6 - webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) - db.session.commit() - - response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") - - assert response.status_code == 302 - assert response.headers["Location"].endswith(f"/sent/{source_identifier}") - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - assert webmention.desired_revision == 3 - assert webmention.processed_revision is None - assert webmention.attempt_count == 0 - assert webmention.next_retry_at is None - assert webmention.sent_revision == 2 - assert webmention.status is None - assert webmention.failure_reason is None - assert webmention.endpoint is None - assert webmention.response_status is None - assert webmention.status_url is None - - send_webmention.assert_called_once_with(webmention_identifier) - - -def test_status_returns_404_for_unknown_webmention(client: FlaskClient) -> None: - response = client.get(f"/status/{uuid.uuid7()}") - - assert response.status_code == 404 - - -def test_status_displays_webmention(app: Flask, client: FlaskClient) -> None: - identifier = create_received_webmention( - app, status="failed", failure_reason="Verification failed" - ) - - response = client.get(f"/status/{identifier}") - - assert response.status_code == 200 - html = response.get_data(as_text=True) - assert SOURCE_URL in html - assert TARGET_URL in html - assert "failed" in html - assert "Verification failed" in html - - -def test_endpoint_recovers_from_concurrent_insert( - client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - mock_endpoint_form(views_module, monkeypatch, valid=True) - - competing_identifier = uuid.uuid7() - competing_webmention = ReceivedWebmention( - uuid=competing_identifier, - source=SOURCE_URL, - target=TARGET_URL, - status="failed", - failure_reason="Competing request failed", - ) - - initial_result = Mock() - initial_result.scalar_one_or_none.return_value = None - competing_result = Mock() - competing_result.scalar_one_or_none.return_value = competing_webmention - - execute = Mock(side_effect=[initial_result, competing_result]) - add = Mock() - rollback = Mock() - commit = Mock( - side_effect=[ - IntegrityError( - "INSERT INTO received_webmention ...", - {}, - Exception("duplicate source/target"), - ), - None, - ] - ) - verify_webmention = Mock() - - monkeypatch.setattr(views_module.db.session, "execute", execute) - monkeypatch.setattr(views_module.db.session, "add", add) - monkeypatch.setattr(views_module.db.session, "rollback", rollback) - monkeypatch.setattr(views_module.db.session, "commit", commit) - monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) - - response = client.post("/endpoint") - - assert response.status_code == 201 - assert response.headers["Location"].endswith(f"/status/{competing_identifier}") - - assert execute.call_count == 2 - add.assert_called_once_with(ANY) - rollback.assert_called_once_with() - assert commit.call_count == 1 - - assert competing_webmention.status == "failed" - assert competing_webmention.failure_reason == "Competing request failed" - verify_webmention.assert_not_called() - - -def test_endpoint_rate_limits_requests_per_ip( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["ENDPOINT_RATE_LIMIT"] = "2/minute" - - form = mock_endpoint_form(views_module, monkeypatch, valid=True) - verify = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify) - - responses = [] - - for index in range(3): - form.source.data = f"https://source.example/post-{index}" - responses.append( - client.post("/endpoint", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) - ) - - assert [r.status_code for r in responses] == [201, 201, 429] - - assert verify.call_count == 2 - - with app.app_context(): - count = db.session.scalar( - sa.select(sa.func.count()).select_from(ReceivedWebmention) - ) - assert count == 2 - - -def test_endpoint_rate_limit_distinguishes_clients_behind_trusted_proxy( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from werkzeug.middleware.proxy_fix import ProxyFix - - app.config["ENDPOINT_RATE_LIMIT"] = "1/minute" - app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) - form = mock_endpoint_form(views_module, monkeypatch, valid=True) - monkeypatch.setattr(views_module, "verify_webmention", Mock()) - - def attempt(client_ip: str, index: int) -> int: - form.source.data = f"https://source.example/proxy-post-{index}" - return client.post( - "/endpoint", - environ_overrides={"REMOTE_ADDR": "192.0.2.200"}, - headers={"X-Forwarded-For": client_ip}, - ).status_code - - assert attempt("198.51.100.11", 0) == 201 - assert attempt("198.51.100.11", 1) == 429 - assert attempt("198.51.100.12", 2) == 201 - - -def test_endpoint_global_rate_limit( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["ENDPOINT_GLOBAL_RATE_LIMIT"] = "2/minute" - - form = mock_endpoint_form(views_module, monkeypatch, valid=True) - verify = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify) - - responses = [] - - for index in range(3): - form.source.data = f"https://source.example/post-{index}" - - responses.append( - client.post( - "/endpoint", environ_overrides={"REMOTE_ADDR": f"192.0.2.{index + 1}"} - ) - ) - - assert [r.status_code for r in responses] == [201, 201, 429] - assert verify.call_count == 2 - - -def test_endpoint_recent_duplicate_does_not_reverify( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock_endpoint_form(views_module, monkeypatch, valid=True) - - verify = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify) - - with app.app_context(): - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source=SOURCE_URL, - target=TARGET_URL, - status="failed", - failure_reason="Previous failure", - ) - db.session.add(webmention) - db.session.commit() - identifier = webmention.uuid - - response = client.post("/endpoint") - - assert response.status_code == 201 - assert response.headers["Location"].endswith(str(identifier)) - - with app.app_context(): - stored = db.session.get(ReceivedWebmention, identifier) - assert stored is not None - assert stored.status == "failed" - assert stored.failure_reason == "Previous failure" - - verify.assert_not_called() - - -def test_endpoint_reverifies_after_cooldown( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock_endpoint_form(views_module, monkeypatch, valid=True) - - verify = Mock() - monkeypatch.setattr(views_module, "verify_webmention", verify) - - with app.app_context(): - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source=SOURCE_URL, - target=TARGET_URL, - status="failed", - failure_reason="Previous failure", - recovery_attempts=3, - updated_at=datetime.now(UTC) - timedelta(minutes=11), - ) - db.session.add(webmention) - db.session.commit() - identifier = webmention.uuid - - response = client.post("/endpoint") - - assert response.status_code == 201 - - with app.app_context(): - stored = db.session.get(ReceivedWebmention, identifier) - assert stored is not None - assert stored.status == "received" - assert stored.failure_reason is None - assert stored.recovery_attempts == 0 - - verify.assert_called_once_with(identifier) - - -@pytest.mark.parametrize("username", ["admin", "unknown-user"]) -def test_login_always_checks_password( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - username: str, -) -> None: - with app.app_context(): - db.session.add(User(username="admin", password="test")) - db.session.commit() - - mock_login_form( - views_module, - monkeypatch, - valid=True, - username=username, - password="wrong-password", - ) - - original_check_password = User.check_password - checked_users: list[User] = [] - - def spy_check_password(self: User, password: str) -> bool: - checked_users.append(self) - return original_check_password(self, password) - - monkeypatch.setattr(User, "check_password", spy_check_password) - - response = client.post("/login") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/login") - assert len(checked_users) == 1 - - if username == "admin": - assert checked_users[0].username == "admin" - else: - assert checked_users[0] is views_module.DUMMY_USER - - with client.session_transaction() as session: - assert "_user_id" not in session - assert ("danger", "Invalid username or password") in session["_flashes"] - - -def test_login_rejects_dummy_user_even_if_password_matches( - client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - mock_login_form( - views_module, - monkeypatch, - valid=True, - username="unknown-user", - password="anything", - ) - - checked_users: list[User] = [] - - def mock_check_password(self: User, password: str) -> bool: - checked_users.append(self) - return True - - monkeypatch.setattr(User, "check_password", mock_check_password) - - response = client.post("/login") - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/login") - assert checked_users == [views_module.DUMMY_USER] - - with client.session_transaction() as session: - assert "_user_id" not in session - - -def test_login_rate_limit_is_independent_per_ip( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["LOGIN_RATE_LIMIT"] = "1/minute" - - mock_login_form( - views_module, - monkeypatch, - valid=True, - username="unknown-user", - password="wrong-password", - ) - - def attempt(ip: str) -> int: - response = client.post("/login", environ_overrides={"REMOTE_ADDR": ip}) - return response.status_code - - assert attempt("192.0.2.10") == 302 - assert attempt("192.0.2.10") == 429 - - # Another IP still has its own allowance. - assert attempt("192.0.2.11") == 302 - assert attempt("192.0.2.11") == 429 - - -def test_login_rate_limit_uses_forwarded_client_ip_when_proxy_is_trusted( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from werkzeug.middleware.proxy_fix import ProxyFix - - app.config["LOGIN_RATE_LIMIT"] = "1/minute" - app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) - mock_login_form( - views_module, - monkeypatch, - valid=True, - username="unknown-user", - password="wrong-password", - ) - - def attempt(client_ip: str) -> int: - response = client.post( - "/login", - environ_overrides={"REMOTE_ADDR": "192.0.2.200"}, - headers={"X-Forwarded-For": client_ip}, - ) - return response.status_code - - assert attempt("198.51.100.11") == 302 - # A forged earlier X-Forwarded-For entry cannot change the client bucket. - assert attempt("203.0.113.99, 198.51.100.11") == 429 - assert attempt("198.51.100.12") == 302 - - -def test_login_username_limit_throttles_across_ips( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["LOGIN_RATE_LIMIT"] = "100/minute" - app.config["LOGIN_USERNAME_RATE_LIMIT"] = "2/minute" - - with monkeypatch.context() as patcher: - mock_login_form( - views_module, - patcher, - valid=True, - username="unknown-user", - password="wrong-password", - ) - - def attempt(ip: str, username: str) -> int: - return client.post( - "/login", - data={"username": username, "password": "wrong-password"}, - environ_overrides={"REMOTE_ADDR": ip}, - ).status_code - - assert attempt("192.0.2.10", "Admin") == 302 - assert attempt("192.0.2.11", " admin ") == 302 - assert attempt("192.0.2.12", "ADMIN") == 429 - # An unrelated username is not locked by attempts against Admin. - assert attempt("192.0.2.12", "another-user") == 302 - - # Restore the real form: the POST-only limits must not block a GET, - # even from the client IP that submitted the throttled attempt. - response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.12"}) - assert response.status_code == 200 - - -def test_login_missing_username_uses_per_ip_bucket( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["LOGIN_RATE_LIMIT"] = "100/minute" - app.config["LOGIN_USERNAME_RATE_LIMIT"] = "1/minute" - mock_login_form( - views_module, - monkeypatch, - valid=True, - username="unknown-user", - password="wrong-password", - ) - - def attempt(ip: str) -> int: - return client.post("/login", environ_overrides={"REMOTE_ADDR": ip}).status_code - - assert attempt("192.0.2.10") == 302 - assert attempt("192.0.2.10") == 429 - assert attempt("192.0.2.11") == 302 - - -def test_login_rate_limits_requests_per_ip( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - app.config["LOGIN_RATE_LIMIT"] = "2/minute" - - with monkeypatch.context() as patcher: - form = mock_login_form( - views_module, - patcher, - valid=True, - username="unknown-user", - password="wrong-password", - ) - - responses = [ - client.post("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) - for _ in range(3) - ] - - assert [response.status_code for response in responses] == [302, 302, 429] - assert form.validate_on_submit.call_count == 2 - assert "Too many login attempts" in responses[-1].get_data(as_text=True) - - # The real LoginForm is restored here. - # GET requests must remain accessible after throttling. - response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) - assert response.status_code == 200 - - with client.session_transaction() as session: - assert "_user_id" not in session - - -def test_logout_rejects_get(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - - response = client.get("/logout") - - assert response.status_code == 405 - - with client.session_transaction() as session: - assert "_user_id" in session - - -@pytest.mark.parametrize( - "data", - [ - pytest.param({}, id="missing"), - pytest.param({"csrf_token": "invalid"}, id="invalid"), - ], -) -def test_logout_rejects_invalid_csrf( - app: Flask, client: FlaskClient, data: dict[str, str] -) -> None: - app.config["WTF_CSRF_ENABLED"] = True - log_in(app, client) - - response = client.post("/logout", data=data) - - assert response.status_code == 400 - - with client.session_transaction() as session: - assert "_user_id" in session - - -def test_logout_ends_session(app: Flask, client: FlaskClient) -> None: - import re - - app.config["WTF_CSRF_ENABLED"] = True - log_in(app, client) - - page = client.get("/received") - assert page.status_code == 200 - - html = page.get_data(as_text=True) - - assert 'action="/logout"' in html - assert 'method="post"' in html - - match = re.search(r'name="csrf_token"\s+value="([^"]+)"', html) - assert match is not None - - response = client.post("/logout", data={"csrf_token": match.group(1)}) - - assert response.status_code == 302 - assert response.headers["Location"].endswith("/") - - with client.session_transaction() as session: - assert "_user_id" not in session - - protected_response = client.get("/received") - assert protected_response.status_code == 302 - assert "/login" in protected_response.headers["Location"] - - -def create_outgoing( - app: Flask, *, status_url: str | None = None -) -> tuple[uuid.UUID, uuid.UUID]: - now = datetime.now(UTC) - with app.app_context(): - source = Source( - path="blog/post/index.html", - url="https://source.example/blog/post", - content_hash="0" * 32, - revision=1, - last_seen_at=now, - revised_at=now, - ) - mention = SentWebmention( - target="https://example.com/article", - active=True, - desired_revision=1, - status_url=status_url, - response_status=201 if status_url is not None else None, - ) - source.sent_webmentions.append(mention) - db.session.add(source) - db.session.commit() - return source.uuid, mention.uuid - - -@pytest.mark.parametrize( - ("status_url", "safe"), - [ - ("https://example.com/status", True), - ("http://example.com/status", True), - ("javascript:alert(1)", False), - ("data:text/html,evil", False), - ("ftp://example.com/status", False), - ], -) -def test_admin_only_links_safe_existing_status_urls( - app: Flask, client: FlaskClient, status_url: str, safe: bool -) -> None: - source_identifier, _ = create_outgoing(app, status_url=status_url) - with app.app_context(): - user = User(username="admin", password="test") - db.session.add(user) - db.session.commit() - user_id = user.get_id() - - with client.session_transaction() as session: - session["_user_id"] = str(user_id) - session["_fresh"] = True - - response = client.get(f"/sent/{source_identifier}") - assert response.status_code == 200 - html = response.get_data(as_text=True) - assert (f'href="{status_url}"' in html) is safe - assert status_url in html - - -def test_endpoint_rejects_request_bodies_over_configured_maximum( - app: Flask, client: FlaskClient -) -> None: - app.config["MAX_CONTENT_LENGTH"] = 128 - response = client.post( - "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} - ) - assert response.status_code == 413 - - -def test_endpoint_rejects_bad_url_before_inserting_row( - app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch -) -> None: - from webmentions_ssg import views - - verify_webmention = Mock() - monkeypatch.setattr(views, "verify_webmention", verify_webmention) - - response = client.post( - "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL} - ) - - assert response.status_code == 400 - assert "source" in response.get_json() - verify_webmention.assert_not_called() - with app.app_context(): - assert ( - db.session.scalar( - sa.select(sa.func.count()).select_from(ReceivedWebmention) - ) - == 0 - ) diff --git a/tests/cli/__init__.py b/tests/views/__init__.py index 25d78d5..25d78d5 100644 --- a/tests/cli/__init__.py +++ b/tests/views/__init__.py diff --git a/tests/views/helpers.py b/tests/views/helpers.py new file mode 100644 index 0000000..b32a066 --- /dev/null +++ b/tests/views/helpers.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ( + ReceivedWebmention, + SentWebmention, + SentWebmentionStatus, + Source, + User, +) + +SOURCE_URL = "https://source.example/post" +TARGET_URL = "https://dennisfink.me/blog/example/" + + +def mock_endpoint_form( + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + valid: bool, + source: str = SOURCE_URL, + target: str = TARGET_URL, + errors: dict[str, list[str]] | None = None, +) -> Mock: + form = Mock() + form.validate_on_submit.return_value = valid + form.source.data = source + form.target.data = target + form.errors = errors or {} + + monkeypatch.setattr(views_module.forms, "EndpointForm", Mock(return_value=form)) + return form + + +def log_in(app: Flask, client: FlaskClient) -> None: + with app.app_context(): + user = User(username="admin", password="test") + + db.session.add(user) + db.session.commit() + + user_id = user.get_id() + + with client.session_transaction() as session: + session["_user_id"] = str(user_id) + session["_fresh"] = True + + +def create_sent_source(app: Flask) -> uuid.UUID: + now = datetime.now(UTC) + + with app.app_context(): + source = Source( + path="blog/example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=2, + last_seen_at=now, + revised_at=now, + ) + + source.sent_webmentions.extend( + [ + SentWebmention( + target="https://example.com/b", + active=True, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + endpoint="https://example.com/webmention", + response_status=202, + ), + SentWebmention( + target="https://example.com/a", + active=True, + desired_revision=2, + processed_revision=2, + sent_revision=1, + status=SentWebmentionStatus.FAILED, + failure_reason="Webmention endpoint returned HTTP 400", + endpoint="https://example.com/webmention", + response_status=400, + ), + ] + ) + + db.session.add(source) + db.session.commit() + + return source.uuid + + +def mock_login_form( + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + valid: bool, + username: str = "admin", + password: str | None = "test", +) -> Mock: + form = Mock() + form.validate_on_submit.return_value = valid + form.username.data = username + form.password.data = password + + monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form)) + return form + + +def mock_admin_action_form( + views_module: ModuleType, monkeypatch: pytest.MonkeyPatch, *, valid: bool +) -> Mock: + form = Mock() + form.validate_on_submit.return_value = valid + + monkeypatch.setattr(views_module.forms, "AdminActionForm", Mock(return_value=form)) + return form + + +def create_received_webmention( + app: Flask, + *, + source: str = SOURCE_URL, + target: str = TARGET_URL, + status: str = "verified", + failure_reason: str | None = None, +) -> uuid.UUID: + with app.app_context(): + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source=source, + target=target, + status=status, + failure_reason=failure_reason, + ) + + db.session.add(webmention) + db.session.commit() + + return webmention.uuid + + +def create_resendable_sent_webmention(app: Flask) -> tuple[uuid.UUID, uuid.UUID]: + now = datetime.now(UTC) + + with app.app_context(): + source = Source( + path="blog/resend/index.html", + url="https://dennisfink.me/blog/resend/", + content_hash="1" * 32, + revision=3, + last_seen_at=now, + revised_at=now, + ) + webmention = SentWebmention( + target="https://example.com/resend", + active=True, + desired_revision=3, + processed_revision=3, + sent_revision=2, + status=SentWebmentionStatus.FAILED, + failure_reason="Previous failure", + endpoint="https://example.com/webmention", + response_status=500, + status_url="https://example.com/webmention/status/123", + ) + source.sent_webmentions.append(webmention) + + db.session.add(source) + db.session.commit() + + return source.uuid, webmention.uuid + + +def create_outgoing( + app: Flask, *, status_url: str | None = None +) -> tuple[uuid.UUID, uuid.UUID]: + now = datetime.now(UTC) + with app.app_context(): + source = Source( + path="blog/post/index.html", + url="https://source.example/blog/post", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + mention = SentWebmention( + target="https://example.com/article", + active=True, + desired_revision=1, + status_url=status_url, + response_status=201 if status_url is not None else None, + ) + source.sent_webmentions.append(mention) + db.session.add(source) + db.session.commit() + return source.uuid, mention.uuid diff --git a/tests/views/test_admin.py b/tests/views/test_admin.py new file mode 100644 index 0000000..26a13f8 --- /dev/null +++ b/tests/views/test_admin.py @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime, timedelta +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention, SentWebmention, User + +from .helpers import ( + SOURCE_URL, + TARGET_URL, + create_outgoing, + create_received_webmention, + create_resendable_sent_webmention, + create_sent_source, + log_in, + mock_admin_action_form, +) + + +@pytest.fixture +def views_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.views import admin as module + + return module + + +@pytest.mark.parametrize("path", ["/sent", f"/sent/{uuid.uuid7()}"]) +def test_sent_views_require_login(client: FlaskClient, path: str) -> None: + response = client.get(path) + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get("/sent") + + assert response.status_code == 200 + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + assert f"/sent/{identifier}" in html + + +def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get(f"/sent/{identifier}") + + assert response.status_code == 200 + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + + first_target = "https://example.com/a" + second_target = "https://example.com/b" + + assert first_target in html + assert second_target in html + assert html.index(first_target) < html.index(second_target) + + assert "Webmention endpoint returned HTTP 400" in html + assert "https://example.com/webmention" in html + + +def test_sent_source_returns_404_for_unknown_source( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.get(f"/sent/{uuid.uuid7()}") + assert response.status_code == 404 + + +def test_received_requires_login(client: FlaskClient) -> None: + response = client.get("/received") + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +def test_received_lists_webmentions(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + create_received_webmention(app) + + response = client.get("/received") + + assert response.status_code == 200 + html = response.get_data(as_text=True) + assert SOURCE_URL in html + assert TARGET_URL in html + + +@pytest.mark.parametrize("action", ["delete", "reverify"]) +def test_received_admin_actions_require_login(client: FlaskClient, action: str) -> None: + response = client.post(f"/received/{uuid.uuid7()}/{action}") + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +@pytest.mark.parametrize("action", ["delete", "reverify"]) +def test_received_admin_actions_reject_invalid_form( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + log_in(app, client) + identifier = create_received_webmention(app) + form = mock_admin_action_form(views_module, monkeypatch, valid=False) + + response = client.post(f"/received/{identifier}/{action}") + + assert response.status_code == 400 + form.validate_on_submit.assert_called_once_with() + + +@pytest.mark.parametrize("action", ["delete", "reverify"]) +def test_received_admin_actions_return_404_for_unknown_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + log_in(app, client) + mock_admin_action_form(views_module, monkeypatch, valid=True) + + response = client.post(f"/received/{uuid.uuid7()}/{action}") + + assert response.status_code == 404 + + +def test_delete_received_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + identifier = create_received_webmention(app) + mock_admin_action_form(views_module, monkeypatch, valid=True) + + response = client.post(f"/received/{identifier}/delete?page=3") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/received?page=3") + + with app.app_context(): + assert db.session.get(ReceivedWebmention, identifier) is None + + +def test_reverify_received_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + identifier = create_received_webmention( + app, status="failed", failure_reason="Previous failure" + ) + with app.app_context(): + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + row.recovery_attempts = 3 + db.session.commit() + mock_admin_action_form(views_module, monkeypatch, valid=True) + verify_webmention = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) + + response = client.post(f"/received/{identifier}/reverify?page=2") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/received?page=2") + + with app.app_context(): + webmention = db.session.get(ReceivedWebmention, identifier) + assert webmention is not None + assert webmention.status == "received" + assert webmention.failure_reason is None + assert webmention.recovery_attempts == 0 + + verify_webmention.assert_called_once_with(identifier) + + +def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: + response = client.post("/sent/rescan") + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +@pytest.mark.parametrize("valid", [False, True]) +def test_rescan_sent_sources( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + valid: bool, +) -> None: + log_in(app, client) + form = mock_admin_action_form(views_module, monkeypatch, valid=valid) + manual_scan_sources = Mock() + monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources) + + response = client.post("/sent/rescan") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/sent") + form.validate_on_submit.assert_called_once_with() + + if valid: + manual_scan_sources.assert_called_once_with() + else: + manual_scan_sources.assert_not_called() + + +def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: + response = client.post(f"/sent/{uuid.uuid7()}/{uuid.uuid7()}/resend") + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +def test_resend_sent_webmention_rejects_invalid_form( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(views_module, monkeypatch, valid=False) + + response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") + + assert response.status_code == 400 + + +def test_resend_sent_webmention_returns_404_for_unknown_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier = create_sent_source(app) + mock_admin_action_form(views_module, monkeypatch, valid=True) + + response = client.post(f"/sent/{source_identifier}/{uuid.uuid7()}/resend") + + assert response.status_code == 404 + + +def test_resend_sent_webmention_rejects_wrong_source( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + _, webmention_identifier = create_resendable_sent_webmention(app) + wrong_source_identifier = create_sent_source(app) + mock_admin_action_form(views_module, monkeypatch, valid=True) + + response = client.post( + f"/sent/{wrong_source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == 404 + + +def test_resend_sent_webmention_reopens_current_revision( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(views_module, monkeypatch, valid=True) + send_webmention = Mock() + monkeypatch.setattr(views_module, "send_webmention", send_webmention) + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + webmention.attempt_count = 6 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) + db.session.commit() + + response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") + + assert response.status_code == 302 + assert response.headers["Location"].endswith(f"/sent/{source_identifier}") + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + assert webmention.desired_revision == 3 + assert webmention.processed_revision is None + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None + assert webmention.sent_revision == 2 + assert webmention.status is None + assert webmention.failure_reason is None + assert webmention.endpoint is None + assert webmention.response_status is None + assert webmention.status_url is None + + send_webmention.assert_called_once_with(webmention_identifier) + + +@pytest.mark.parametrize( + ("status_url", "safe"), + [ + ("https://example.com/status", True), + ("http://example.com/status", True), + ("javascript:alert(1)", False), + ("data:text/html,evil", False), + ("ftp://example.com/status", False), + ], +) +def test_admin_only_links_safe_existing_status_urls( + app: Flask, client: FlaskClient, status_url: str, safe: bool +) -> None: + source_identifier, _ = create_outgoing(app, status_url=status_url) + with app.app_context(): + user = User(username="admin", password="test") + db.session.add(user) + db.session.commit() + user_id = user.get_id() + + with client.session_transaction() as session: + session["_user_id"] = str(user_id) + session["_fresh"] = True + + response = client.get(f"/sent/{source_identifier}") + assert response.status_code == 200 + html = response.get_data(as_text=True) + assert (f'href="{status_url}"' in html) is safe + assert status_url in html diff --git a/tests/views/test_endpoint.py b/tests/views/test_endpoint.py new file mode 100644 index 0000000..652250f --- /dev/null +++ b/tests/views/test_endpoint.py @@ -0,0 +1,422 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime, timedelta +from types import ModuleType +from unittest.mock import ANY, Mock + +import pytest +import sqlalchemy as sa +from flask import Flask +from flask.testing import FlaskClient +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention + +from .helpers import ( + SOURCE_URL, + TARGET_URL, + create_received_webmention, + mock_endpoint_form, +) + + +@pytest.fixture +def views_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.views import endpoint as module + + return module + + +def test_endpoint_only_accepts_post(client: FlaskClient) -> None: + response = client.get("/endpoint") + assert response.status_code == 405 + + +def test_endpoint_returns_form_errors( + client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + errors = {"source": ["Invalid source"]} + mock_endpoint_form(views_module, monkeypatch, valid=False, errors=errors) + + response = client.post("/endpoint") + + assert response.status_code == 400 + assert response.get_json() == errors + + +def test_endpoint_creates_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + form = mock_endpoint_form(views_module, monkeypatch, valid=True) + verify_webmention = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) + + response = client.post("/endpoint") + + assert response.status_code == 201 + form.validate_on_submit.assert_called_once_with() + + with app.app_context(): + webmention = db.session.scalar(sa.select(ReceivedWebmention)) + + assert webmention is not None + assert webmention.source == SOURCE_URL + assert webmention.target == TARGET_URL + assert webmention.status == "received" + assert webmention.failure_reason is None + + identifier = webmention.uuid + + assert response.headers["Location"].endswith(str(identifier)) + verify_webmention.assert_called_once_with(identifier) + + +def test_endpoint_is_idempotent( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + mock_endpoint_form(views_module, monkeypatch, valid=True) + verify_webmention = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) + + first_response = client.post("/endpoint") + second_response = client.post("/endpoint") + + assert first_response.status_code == 201 + assert second_response.status_code == 201 + assert first_response.headers["Location"] == second_response.headers["Location"] + + with app.app_context(): + webmentions = db.session.scalars(sa.select(ReceivedWebmention)).all() + + assert len(webmentions) == 1 + + webmention = webmentions[0] + + assert webmention.source == SOURCE_URL + assert webmention.target == TARGET_URL + assert webmention.status == "received" + assert webmention.failure_reason is None + + identifier = webmention.uuid + + verify_webmention.assert_called_once_with(identifier) + + +def test_resending_resets_failure_state( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + mock_endpoint_form(views_module, monkeypatch, valid=True) + verify_webmention = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) + + with app.app_context(): + existing = ReceivedWebmention( + uuid=uuid.uuid7(), + source=SOURCE_URL, + target=TARGET_URL, + status="failed", + failure_reason="Previous failure", + updated_at=datetime.now(UTC) - timedelta(minutes=11), + ) + + db.session.add(existing) + db.session.commit() + + identifier = existing.uuid + + response = client.post("/endpoint") + + assert response.status_code == 201 + + with app.app_context(): + webmention = db.session.get(ReceivedWebmention, identifier) + + assert webmention is not None + assert webmention.status == "received" + assert webmention.failure_reason is None + + verify_webmention.assert_called_once_with(identifier) + + +def test_status_returns_404_for_unknown_webmention(client: FlaskClient) -> None: + response = client.get(f"/status/{uuid.uuid7()}") + + assert response.status_code == 404 + + +def test_status_displays_webmention(app: Flask, client: FlaskClient) -> None: + identifier = create_received_webmention( + app, status="failed", failure_reason="Verification failed" + ) + + response = client.get(f"/status/{identifier}") + + assert response.status_code == 200 + html = response.get_data(as_text=True) + assert SOURCE_URL in html + assert TARGET_URL in html + assert "failed" in html + assert "Verification failed" in html + + +def test_endpoint_recovers_from_concurrent_insert( + client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + mock_endpoint_form(views_module, monkeypatch, valid=True) + + competing_identifier = uuid.uuid7() + competing_webmention = ReceivedWebmention( + uuid=competing_identifier, + source=SOURCE_URL, + target=TARGET_URL, + status="failed", + failure_reason="Competing request failed", + ) + + initial_result = Mock() + initial_result.scalar_one_or_none.return_value = None + competing_result = Mock() + competing_result.scalar_one_or_none.return_value = competing_webmention + + execute = Mock(side_effect=[initial_result, competing_result]) + add = Mock() + rollback = Mock() + commit = Mock( + side_effect=[ + IntegrityError( + "INSERT INTO received_webmention ...", + {}, + Exception("duplicate source/target"), + ), + None, + ] + ) + verify_webmention = Mock() + + monkeypatch.setattr(views_module.db.session, "execute", execute) + monkeypatch.setattr(views_module.db.session, "add", add) + monkeypatch.setattr(views_module.db.session, "rollback", rollback) + monkeypatch.setattr(views_module.db.session, "commit", commit) + monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) + + response = client.post("/endpoint") + + assert response.status_code == 201 + assert response.headers["Location"].endswith(f"/status/{competing_identifier}") + + assert execute.call_count == 2 + add.assert_called_once_with(ANY) + rollback.assert_called_once_with() + assert commit.call_count == 1 + + assert competing_webmention.status == "failed" + assert competing_webmention.failure_reason == "Competing request failed" + verify_webmention.assert_not_called() + + +def test_endpoint_rate_limits_requests_per_ip( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["ENDPOINT_RATE_LIMIT"] = "2/minute" + + form = mock_endpoint_form(views_module, monkeypatch, valid=True) + verify = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify) + + responses = [] + + for index in range(3): + form.source.data = f"https://source.example/post-{index}" + responses.append( + client.post("/endpoint", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) + ) + + assert [r.status_code for r in responses] == [201, 201, 429] + + assert verify.call_count == 2 + + with app.app_context(): + count = db.session.scalar( + sa.select(sa.func.count()).select_from(ReceivedWebmention) + ) + assert count == 2 + + +def test_endpoint_rate_limit_distinguishes_clients_behind_trusted_proxy( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from werkzeug.middleware.proxy_fix import ProxyFix + + app.config["ENDPOINT_RATE_LIMIT"] = "1/minute" + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) + form = mock_endpoint_form(views_module, monkeypatch, valid=True) + monkeypatch.setattr(views_module, "verify_webmention", Mock()) + + def attempt(client_ip: str, index: int) -> int: + form.source.data = f"https://source.example/proxy-post-{index}" + return client.post( + "/endpoint", + environ_overrides={"REMOTE_ADDR": "192.0.2.200"}, + headers={"X-Forwarded-For": client_ip}, + ).status_code + + assert attempt("198.51.100.11", 0) == 201 + assert attempt("198.51.100.11", 1) == 429 + assert attempt("198.51.100.12", 2) == 201 + + +def test_endpoint_global_rate_limit( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["ENDPOINT_GLOBAL_RATE_LIMIT"] = "2/minute" + + form = mock_endpoint_form(views_module, monkeypatch, valid=True) + verify = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify) + + responses = [] + + for index in range(3): + form.source.data = f"https://source.example/post-{index}" + + responses.append( + client.post( + "/endpoint", environ_overrides={"REMOTE_ADDR": f"192.0.2.{index + 1}"} + ) + ) + + assert [r.status_code for r in responses] == [201, 201, 429] + assert verify.call_count == 2 + + +def test_endpoint_recent_duplicate_does_not_reverify( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + mock_endpoint_form(views_module, monkeypatch, valid=True) + + verify = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify) + + with app.app_context(): + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source=SOURCE_URL, + target=TARGET_URL, + status="failed", + failure_reason="Previous failure", + ) + db.session.add(webmention) + db.session.commit() + identifier = webmention.uuid + + response = client.post("/endpoint") + + assert response.status_code == 201 + assert response.headers["Location"].endswith(str(identifier)) + + with app.app_context(): + stored = db.session.get(ReceivedWebmention, identifier) + assert stored is not None + assert stored.status == "failed" + assert stored.failure_reason == "Previous failure" + + verify.assert_not_called() + + +def test_endpoint_reverifies_after_cooldown( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + mock_endpoint_form(views_module, monkeypatch, valid=True) + + verify = Mock() + monkeypatch.setattr(views_module, "verify_webmention", verify) + + with app.app_context(): + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source=SOURCE_URL, + target=TARGET_URL, + status="failed", + failure_reason="Previous failure", + recovery_attempts=3, + updated_at=datetime.now(UTC) - timedelta(minutes=11), + ) + db.session.add(webmention) + db.session.commit() + identifier = webmention.uuid + + response = client.post("/endpoint") + + assert response.status_code == 201 + + with app.app_context(): + stored = db.session.get(ReceivedWebmention, identifier) + assert stored is not None + assert stored.status == "received" + assert stored.failure_reason is None + assert stored.recovery_attempts == 0 + + verify.assert_called_once_with(identifier) + + +def test_endpoint_rejects_request_bodies_over_configured_maximum( + app: Flask, client: FlaskClient +) -> None: + app.config["MAX_CONTENT_LENGTH"] = 128 + response = client.post( + "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} + ) + assert response.status_code == 413 + + +def test_endpoint_rejects_bad_url_before_inserting_row( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg.views import endpoint as views + + verify_webmention = Mock() + monkeypatch.setattr(views, "verify_webmention", verify_webmention) + + response = client.post( + "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL} + ) + + assert response.status_code == 400 + assert "source" in response.get_json() + verify_webmention.assert_not_called() + with app.app_context(): + assert ( + db.session.scalar( + sa.select(sa.func.count()).select_from(ReceivedWebmention) + ) + == 0 + ) diff --git a/tests/views/test_session.py b/tests/views/test_session.py new file mode 100644 index 0000000..6a96032 --- /dev/null +++ b/tests/views/test_session.py @@ -0,0 +1,431 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import User + +from .helpers import log_in, mock_login_form + + +@pytest.fixture +def views_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.views import session as module + + return module + + +def test_login_renders_form(client: FlaskClient) -> None: + response = client.get("/login") + + assert response.status_code == 200 + assert "login" in response.get_data(as_text=True).lower() + + +@pytest.mark.parametrize("password", ("wrong-password", None)) +def test_login_rejects_invalid_credentials( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + password: str | None, +) -> None: + with app.app_context(): + db.session.add(User(username="admin", password="test")) + db.session.commit() + + mock_login_form( + views_module, monkeypatch, valid=True, username="admin", password=password + ) + + response = client.post("/login") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/login") + + with client.session_transaction() as session: + assert "_user_id" not in session + assert ("danger", "Invalid username or password") in session["_flashes"] + + +@pytest.mark.parametrize( + "next_url", + [ + pytest.param("/", id="root"), + pytest.param("/received", id="local-path"), + pytest.param("/received?page=2", id="query-string"), + pytest.param("/received#details", id="fragment"), + ], +) +def test_login_redirects_to_local_next_page( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + next_url: str, +) -> None: + with app.app_context(): + user = User(username="admin", password="test") + db.session.add(user) + db.session.commit() + user_id = user.get_id() + + mock_login_form(views_module, monkeypatch, valid=True) + + response = client.post("/login", query_string={"next": next_url}) + + assert response.status_code == 302 + assert response.headers["Location"] == next_url + + with client.session_transaction() as session: + assert session["_user_id"] == str(user_id) + + +@pytest.mark.parametrize( + "next_url", + [ + pytest.param(None, id="missing"), + pytest.param("", id="empty"), + pytest.param("received", id="relative-path"), + pytest.param("https://evil.example/phishing", id="absolute-https"), + pytest.param("http://evil.example", id="absolute-http"), + pytest.param("javascript:alert(1)", id="non-http-scheme"), + pytest.param("//evil.example/phishing", id="scheme-relative"), + pytest.param("///evil.example/phishing", id="triple-slash"), + pytest.param("////evil.example", id="four-slashes"), + pytest.param(r"/\evil.example", id="slash-backslash-host"), + pytest.param(r"/received\evil.example", id="backslash-in-path"), + pytest.param(r"\/evil.example", id="backslash-slash-host"), + ], +) +def test_login_rejects_unsafe_next_page( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + next_url: str | None, +) -> None: + with app.app_context(): + user = User(username="admin", password="test") + db.session.add(user) + db.session.commit() + user_id = user.get_id() + + mock_login_form(views_module, monkeypatch, valid=True) + + query_string = {} if next_url is None else {"next": next_url} + response = client.post("/login", query_string=query_string) + + assert response.status_code == 302 + assert response.headers["Location"] == "/" + + with client.session_transaction() as session: + assert session["_user_id"] == str(user_id) + + +def test_login_redirects_authenticated_user(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + + response = client.get("/login") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/") + + +@pytest.mark.parametrize("username", ["admin", "unknown-user"]) +def test_login_always_checks_password( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + username: str, +) -> None: + with app.app_context(): + db.session.add(User(username="admin", password="test")) + db.session.commit() + + mock_login_form( + views_module, + monkeypatch, + valid=True, + username=username, + password="wrong-password", + ) + + original_check_password = User.check_password + checked_users: list[User] = [] + + def spy_check_password(self: User, password: str) -> bool: + checked_users.append(self) + return original_check_password(self, password) + + monkeypatch.setattr(User, "check_password", spy_check_password) + + response = client.post("/login") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/login") + assert len(checked_users) == 1 + + if username == "admin": + assert checked_users[0].username == "admin" + else: + assert checked_users[0] is views_module.DUMMY_USER + + with client.session_transaction() as session: + assert "_user_id" not in session + assert ("danger", "Invalid username or password") in session["_flashes"] + + +def test_login_rejects_dummy_user_even_if_password_matches( + client: FlaskClient, views_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + mock_login_form( + views_module, + monkeypatch, + valid=True, + username="unknown-user", + password="anything", + ) + + checked_users: list[User] = [] + + def mock_check_password(self: User, password: str) -> bool: + checked_users.append(self) + return True + + monkeypatch.setattr(User, "check_password", mock_check_password) + + response = client.post("/login") + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/login") + assert checked_users == [views_module.DUMMY_USER] + + with client.session_transaction() as session: + assert "_user_id" not in session + + +def test_login_rate_limit_is_independent_per_ip( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["LOGIN_RATE_LIMIT"] = "1/minute" + + mock_login_form( + views_module, + monkeypatch, + valid=True, + username="unknown-user", + password="wrong-password", + ) + + def attempt(ip: str) -> int: + response = client.post("/login", environ_overrides={"REMOTE_ADDR": ip}) + return response.status_code + + assert attempt("192.0.2.10") == 302 + assert attempt("192.0.2.10") == 429 + + # Another IP still has its own allowance. + assert attempt("192.0.2.11") == 302 + assert attempt("192.0.2.11") == 429 + + +def test_login_rate_limit_uses_forwarded_client_ip_when_proxy_is_trusted( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from werkzeug.middleware.proxy_fix import ProxyFix + + app.config["LOGIN_RATE_LIMIT"] = "1/minute" + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) + mock_login_form( + views_module, + monkeypatch, + valid=True, + username="unknown-user", + password="wrong-password", + ) + + def attempt(client_ip: str) -> int: + response = client.post( + "/login", + environ_overrides={"REMOTE_ADDR": "192.0.2.200"}, + headers={"X-Forwarded-For": client_ip}, + ) + return response.status_code + + assert attempt("198.51.100.11") == 302 + # A forged earlier X-Forwarded-For entry cannot change the client bucket. + assert attempt("203.0.113.99, 198.51.100.11") == 429 + assert attempt("198.51.100.12") == 302 + + +def test_login_username_limit_throttles_across_ips( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["LOGIN_RATE_LIMIT"] = "100/minute" + app.config["LOGIN_USERNAME_RATE_LIMIT"] = "2/minute" + + with monkeypatch.context() as patcher: + mock_login_form( + views_module, + patcher, + valid=True, + username="unknown-user", + password="wrong-password", + ) + + def attempt(ip: str, username: str) -> int: + return client.post( + "/login", + data={"username": username, "password": "wrong-password"}, + environ_overrides={"REMOTE_ADDR": ip}, + ).status_code + + assert attempt("192.0.2.10", "Admin") == 302 + assert attempt("192.0.2.11", " admin ") == 302 + assert attempt("192.0.2.12", "ADMIN") == 429 + # An unrelated username is not locked by attempts against Admin. + assert attempt("192.0.2.12", "another-user") == 302 + + # Restore the real form: the POST-only limits must not block a GET, + # even from the client IP that submitted the throttled attempt. + response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.12"}) + assert response.status_code == 200 + + +def test_login_missing_username_uses_per_ip_bucket( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["LOGIN_RATE_LIMIT"] = "100/minute" + app.config["LOGIN_USERNAME_RATE_LIMIT"] = "1/minute" + mock_login_form( + views_module, + monkeypatch, + valid=True, + username="unknown-user", + password="wrong-password", + ) + + def attempt(ip: str) -> int: + return client.post("/login", environ_overrides={"REMOTE_ADDR": ip}).status_code + + assert attempt("192.0.2.10") == 302 + assert attempt("192.0.2.10") == 429 + assert attempt("192.0.2.11") == 302 + + +def test_login_rate_limits_requests_per_ip( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + app.config["LOGIN_RATE_LIMIT"] = "2/minute" + + with monkeypatch.context() as patcher: + form = mock_login_form( + views_module, + patcher, + valid=True, + username="unknown-user", + password="wrong-password", + ) + + responses = [ + client.post("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) + for _ in range(3) + ] + + assert [response.status_code for response in responses] == [302, 302, 429] + assert form.validate_on_submit.call_count == 2 + assert "Too many login attempts" in responses[-1].get_data(as_text=True) + + # The real LoginForm is restored here. + # GET requests must remain accessible after throttling. + response = client.get("/login", environ_overrides={"REMOTE_ADDR": "192.0.2.10"}) + assert response.status_code == 200 + + with client.session_transaction() as session: + assert "_user_id" not in session + + +def test_logout_rejects_get(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + + response = client.get("/logout") + + assert response.status_code == 405 + + with client.session_transaction() as session: + assert "_user_id" in session + + +@pytest.mark.parametrize( + "data", + [ + pytest.param({}, id="missing"), + pytest.param({"csrf_token": "invalid"}, id="invalid"), + ], +) +def test_logout_rejects_invalid_csrf( + app: Flask, client: FlaskClient, data: dict[str, str] +) -> None: + app.config["WTF_CSRF_ENABLED"] = True + log_in(app, client) + + response = client.post("/logout", data=data) + + assert response.status_code == 400 + + with client.session_transaction() as session: + assert "_user_id" in session + + +def test_logout_ends_session(app: Flask, client: FlaskClient) -> None: + import re + + app.config["WTF_CSRF_ENABLED"] = True + log_in(app, client) + + page = client.get("/received") + assert page.status_code == 200 + + html = page.get_data(as_text=True) + + assert 'action="/logout"' in html + assert 'method="post"' in html + + match = re.search(r'name="csrf_token"\s+value="([^"]+)"', html) + assert match is not None + + response = client.post("/logout", data={"csrf_token": match.group(1)}) + + assert response.status_code == 302 + assert response.headers["Location"].endswith("/") + + with client.session_transaction() as session: + assert "_user_id" not in session + + protected_response = client.get("/received") + assert protected_response.status_code == 302 + assert "/login" in protected_response.headers["Location"] diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 8fed1a7..63e9f13 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -8,47 +8,20 @@ import logging import logging.handlers import os from pathlib import Path -from urllib.parse import urlsplit import coloredlogs from flask import Flask -from flask_bootstrap import Bootstrap5 -from flask_limiter import Limiter -from flask_limiter.util import get_remote_address -from flask_mail import Mail -from flask_migrate import Migrate -from flask_sqlalchemy import SQLAlchemy -from flask_wtf.csrf import CSRFProtect -from sqlalchemy.orm import DeclarativeBase from werkzeug.middleware.proxy_fix import ProxyFix -from webmentions_ssg.config import ( - DefaultConfig, - DevelopmentConfig, - ProductionConfig, - normalize_allowed_hostnames, - normalize_ignored_hostnames, -) - -from .security import validate_secret_key -from .tasks import Huey -from .url_security import is_http_url - APP_NAME = "webmentions-ssg" VERSION = "0.0.2" - -class Base(DeclarativeBase): - pass - - -CSRF = CSRFProtect() -BOOTSTRAP = Bootstrap5() -DATABASE = SQLAlchemy(model_class=Base) -MIGRATE = Migrate() -HUEY = Huey() -MAIL = Mail() -LIMITER = Limiter(key_func=get_remote_address) +from .auth import AUTH +from .cli import DB_CLI, MAIL_CLI, USERS_CLI +from .config import DefaultConfig, DevelopmentConfig, ProductionConfig, validate_config +from .extensions import BOOTSTRAP, CSRF, DATABASE, HUEY, LIMITER, MAIL, MIGRATE +from .net import is_http_url +from .views import register_routes def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: @@ -91,71 +64,15 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.config.from_prefixed_env(prefix="FLASK") app.config.from_prefixed_env(prefix="WEBMENTIONS_SSG") - # A fixed key is convenient only for local debugging. All other - # deployments must supply their own persistent, random signing key. - using_local_dev_key = ( - issubclass(config_class, DevelopmentConfig) - and app.debug - and app.config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY - ) - if not app.testing and not using_local_dev_key: - validate_secret_key(app.config.get("SECRET_KEY")) + app.config.update(validate_config(app.config, config_class=config_class)) - app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames( - app.config["ALLOWED_HOSTNAMES"] - ) - app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames( - app.config["IGNORED_HOSTNAMES"] - ) - proxy_hops = app.config["PROXY_FIX_TRUSTED_HOPS"] - if type(proxy_hops) is not int or proxy_hops < 0: - raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.") - if proxy_hops: - # Trust only the configured number of proxies; never trust forwarded - # host, port or URL prefix headers for rate limiting. + if proxy_hops := app.config["PROXY_FIX_TRUSTED_HOPS"]: + # Do not trust forwarded host, port, or URL prefixes. app.wsgi_app = ProxyFix(app.wsgi_app, x_for=proxy_hops, x_proto=proxy_hops) - for key in ( - "MAX_CONTENT_LENGTH", - "MAX_WEBMENTION_URL_BYTES", - "RECEIVED_RETENTION_DAYS", - ): - value = app.config[key] - if type(value) is not int or value <= 0: - raise ValueError(f"{key} must be a positive integer.") - if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]: - raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.") - - if ( - not app.testing - and not app.debug - and app.config["RATELIMIT_STORAGE_URI"] == "memory://" - ): - raise RuntimeError( - "Configure RATELIMIT_STORAGE_URI with shared storage before running in production." - ) - - if not app.testing and not app.debug: - # HUEY_URL overrides the legacy HUEY.url option; absent either, Huey - # falls back to an in-process queue that cannot serve a separate worker. - huey_url = app.config.get( - "HUEY_URL", app.config.get("HUEY", {}).get("url", "memory://") - ) - if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in { - "memory", - "blackhole", - }: - raise RuntimeError( - "Configure HUEY_URL with a shared, persistent queue before " - "running in production (memory:// and blackhole:// are unsafe)." - ) - - # Local development and tests should start from a fresh checkout: their - # default log directory and development SQLite instance are generated. - if app.testing or app.debug: - Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True) - if app.debug: + if app.debug or app.testing: Path(app.instance_path).mkdir(parents=True, exist_ok=True) + Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True) default_logging_formatter = logging.Formatter(app.config["LOG_FORMAT"]) @@ -197,8 +114,6 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: LIMITER.init_app(app) MAIL.init_app(app) - from .auth import AUTH - AUTH.init_app(app) BOOTSTRAP.init_app(app) HUEY.init_app(app) @@ -214,20 +129,12 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.add_template_filter(is_http_url) - from .views import root_page - - app.register_blueprint(root_page) - - from .cli.users import USERS_CLI + app.register_blueprint(register_routes()) app.cli.add_command(USERS_CLI) - from .cli.mail import MAIL_CLI - app.cli.add_command(MAIL_CLI) - from .cli.database import DB_CLI - app.cli.add_command(DB_CLI) return app diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index 147220e..633aa82 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -4,17 +4,43 @@ """Configure user authentication and session loading.""" +import hashlib +import secrets from hmac import compare_digest +from urllib.parse import urlsplit +from flask import request +from flask_limiter.util import get_remote_address from flask_login import LoginManager -from . import DATABASE as db +from .extensions import DATABASE as db from .models import User AUTH = LoginManager() AUTH.login_view = "root.login" # pyright: ignore[reportAttributeAccessIssue] +MIN_PASSWORD_LENGTH = 8 + +DUMMY_USER = User(username=secrets.token_urlsafe()) +DUMMY_USER.password = secrets.token_urlsafe() + + +def validate_password(password: str) -> None: + """Validate newly chosen passwords without arbitrary composition rules. + + Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases, + but reject empty passwords and control characters. + """ + if not password: + raise ValueError("Password must not be empty.") + if len(password) < MIN_PASSWORD_LENGTH: + raise ValueError( + f"Password must be at least {MIN_PASSWORD_LENGTH} characters long." + ) + if any(ord(char) < 32 or ord(char) == 127 for char in password): + raise ValueError("Password must not contain control characters.") + @AUTH.user_loader def load_user(session_id: str) -> User | None: @@ -44,3 +70,28 @@ def load_user(session_id: str) -> User | None: if not compare_digest(session_id, user.get_id()): return None return user + + +def login_username_limit_key() -> str: + """Group usernames across IPs without exposing plaintext to rate-limit storage.""" + username = request.form.get("username", "").strip().casefold() + if not username: + return f"missing-username:{get_remote_address()}" + digest = hashlib.sha256(username.encode("utf-8")).hexdigest() + return f"username:{digest}" + + +def is_safe_next_url(next_url: str) -> bool: + """Allow redirects only to paths on this site.""" + if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"): + return False + try: + target = urlsplit(next_url) + except ValueError: + return False + return ( + not target.scheme + and not target.netloc + and target.path.startswith("/") + and not target.path.startswith("//") + ) diff --git a/webmentions_ssg/cli/users.py b/webmentions_ssg/cli.py index a48fc71..0387ba6 100644 --- a/webmentions_ssg/cli/users.py +++ b/webmentions_ssg/cli.py @@ -2,22 +2,26 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Provide commands for managing application user accounts.""" +"""Manage users, mail settings, and database migrations.""" import click import sqlalchemy as sa +from flask import current_app from flask.cli import AppGroup +from flask_mail import Message +from flask_migrate import upgrade as migrate_upgrade +from sqlalchemy import inspect from sqlalchemy.exc import IntegrityError -from .. import DATABASE -from ..security import validate_password +from .auth import validate_password +from .extensions import DATABASE, MAIL +from .models import User USERS_CLI = AppGroup("users", help="Manage users.") def _require_user(username: str): """Return a user by username or report a CLI-friendly error.""" - from ..models import User user = DATABASE.session.scalar(sa.select(User).where(User.username == username)) if user is None: @@ -36,7 +40,6 @@ def create_user(username: str, password: str) -> None: :param password: Plain-text password to hash and store. :raises click.ClickException: If the credentials are invalid or the user exists. """ - from ..models import User username = username.strip() if not username: @@ -63,7 +66,6 @@ def create_user(username: str, password: str) -> None: @USERS_CLI.command("list", help="List registered users.") def list_users() -> None: """List user IDs and usernames in alphabetical order.""" - from ..models import User users = DATABASE.session.scalars(sa.select(User).order_by(User.username)).all() if not users: @@ -115,3 +117,83 @@ def delete_user(username: str, yes: bool) -> None: DATABASE.session.delete(user) DATABASE.session.commit() click.echo(f"Deleted user {username!r}.") + + +MAIL_CLI = AppGroup("mail", help="Handle mail") + + +@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") +@click.option( + "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)." +) +def send_test_mail(recipient: str | None) -> None: + """ + Send a test email using the configured SMTP server. + + :param recipient: Optional recipient overriding NOTIFICATION_EMAIL. + :raises click.ClickException: If configuration is missing or sending fails. + """ + recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL") + if not recipient: + raise click.ClickException( + "No recipient specified. Set NOTIFICATION_EMAIL or pass --to." + ) + + sender = current_app.config.get("MAIL_DEFAULT_SENDER") + if not sender: + raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.") + + message = Message( + subject="webmentions-ssg: test email", + sender=sender, + recipients=[recipient], + body=( + "This is a test email from webmentions-ssg.\n\n" + "If you received this message, the outgoing mail configuration works.\n" + ), + ) + + try: + MAIL.send(message) + except Exception as exc: + raise click.ClickException(f"Could not send test email: {exc}") from exc + + click.echo(f"Test email sent to {recipient}.") + + +DB_CLI = AppGroup("db", help="Handle database") + + +def _table_names() -> set[str]: + """Return existing table names without modifying the database schema.""" + with DATABASE.engine.connect() as connection: + return set(inspect(connection).get_table_names()) + + +@DB_CLI.command("create-db", help="Create a new database using all migrations") +def create_db() -> None: + """Initialize an empty database and record its Alembic revision.""" + if _table_names(): + raise click.ClickException( + "The database already contains tables. " + "Use 'flask db upgrade' for an existing migrated database." + ) + + # Applying migrations (rather than create_all()) keeps alembic_version in sync. + migrate_upgrade() + click.echo("Database created at the latest migration revision.") + + +@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision") +def upgrade_db() -> None: + """Apply pending migrations to a previously initialized database.""" + if "alembic_version" not in _table_names(): + raise click.ClickException( + "No Flask-Migrate revision table found. " + "For a new, empty database run 'flask db create-db'. " + "For a legacy database, inspect its schema and use " + "'flask migrate stamp' only if appropriate." + ) + + migrate_upgrade() + click.echo("Database upgraded to the latest migration revision.") diff --git a/webmentions_ssg/cli/__init__.py b/webmentions_ssg/cli/__init__.py deleted file mode 100644 index accaaa3..0000000 --- a/webmentions_ssg/cli/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Group the application command-line management modules.""" diff --git a/webmentions_ssg/cli/database.py b/webmentions_ssg/cli/database.py deleted file mode 100644 index e918600..0000000 --- a/webmentions_ssg/cli/database.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Simple database commands built on Flask-Migrate.""" - -import click -from flask.cli import AppGroup -from flask_migrate import upgrade as migrate_upgrade -from sqlalchemy import inspect - -from .. import DATABASE - -DB_CLI = AppGroup("db", help="Handle database") - - -def _table_names() -> set[str]: - """Return existing table names without modifying the database schema.""" - with DATABASE.engine.connect() as connection: - return set(inspect(connection).get_table_names()) - - -@DB_CLI.command("create-db", help="Create a new database using all migrations") -def create_db() -> None: - """Initialize an empty database and record its Alembic revision.""" - if _table_names(): - raise click.ClickException( - "The database already contains tables. " - "Use 'flask db upgrade' for an existing migrated database." - ) - - # Applying migrations (rather than create_all()) keeps alembic_version in sync. - migrate_upgrade() - click.echo("Database created at the latest migration revision.") - - -@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision") -def upgrade_db() -> None: - """Apply pending migrations to a previously initialized database.""" - if "alembic_version" not in _table_names(): - raise click.ClickException( - "No Flask-Migrate revision table found. " - "For a new, empty database run 'flask db create-db'. " - "For a legacy database, inspect its schema and use " - "'flask migrate stamp' only if appropriate." - ) - - migrate_upgrade() - click.echo("Database upgraded to the latest migration revision.") diff --git a/webmentions_ssg/cli/mail.py b/webmentions_ssg/cli/mail.py deleted file mode 100644 index 47687e2..0000000 --- a/webmentions_ssg/cli/mail.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Provide a command for testing outgoing email configuration.""" - -import click -from flask import current_app -from flask.cli import AppGroup -from flask_mail import Message - -from .. import MAIL - -MAIL_CLI = AppGroup("mail", help="Handle mail") - - -@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") -@click.option( - "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)." -) -def send_test_mail(recipient: str | None) -> None: - """ - Send a test email using the configured SMTP server. - - :param recipient: Optional recipient overriding NOTIFICATION_EMAIL. - :raises click.ClickException: If configuration is missing or sending fails. - """ - recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL") - if not recipient: - raise click.ClickException( - "No recipient specified. Set NOTIFICATION_EMAIL or pass --to." - ) - - sender = current_app.config.get("MAIL_DEFAULT_SENDER") - if not sender: - raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.") - - message = Message( - subject="webmentions-ssg: test email", - sender=sender, - recipients=[recipient], - body=( - "This is a test email from webmentions-ssg.\n\n" - "If you received this message, the outgoing mail configuration works.\n" - ), - ) - - try: - MAIL.send(message) - except Exception as exc: - raise click.ClickException(f"Could not send test email: {exc}") from exc - - click.echo(f"Test email sent to {recipient}.") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 8a3e58d..8251da3 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -5,39 +5,11 @@ """Define application settings for production, development, and testing.""" import logging +from urllib.parse import urlsplit +from flask.config import Config -def normalize_allowed_hostnames(value: object) -> set[str]: - """Validate and normalize the configured collection of allowed hostnames.""" - if not isinstance(value, (list, tuple, set, frozenset)): - raise TypeError( - "ALLOWED_HOSTNAMES must be a list, tuple, or set of hostnames, not a string." - ) - - hostnames: set[str] = set() - for hostname in value: - if not isinstance(hostname, str) or not hostname.strip(): - raise ValueError("ALLOWED_HOSTNAMES must contain non-empty strings only.") - hostnames.add(hostname.strip().lower()) - return hostnames - - -def normalize_ignored_hostnames(value: object) -> tuple[str, ...]: - """Validate and normalize hostname patterns ignored by the source scanner.""" - if not isinstance(value, (list, tuple, set, frozenset)): - raise TypeError( - "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, " - "not a string." - ) - - patterns: set[str] = set() - for pattern in value: - if not isinstance(pattern, str) or not ( - normalized := pattern.strip().lower().rstrip(".") - ): - raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.") - patterns.add(normalized) - return tuple(sorted(patterns)) +MIN_SECRET_KEY_LENGTH = 32 class DefaultConfig: @@ -69,6 +41,7 @@ class DefaultConfig: RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 + MAX_TARGET_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 # Absolute wall-clock budget for each verification or delivery attempt. REQUEST_TOTAL_TIMEOUT = 30.0 @@ -144,3 +117,107 @@ class TestingConfig(DefaultConfig): HUEY_URL = "memory://" ALLOWED_HOSTNAMES = {"dennisfink.me"} + + +def normalize_hostnames(value: object) -> set[str]: + """Validate and normalize a collection of hostnames.""" + if not isinstance(value, (list, tuple, set, frozenset)): + raise TypeError("Expected a list, tuple, or set of hostnames.") + + hostnames: set[str] = set() + for hostname in value: + if not isinstance(hostname, str) or not ( + normalized := hostname.strip().lower().rstrip(".") + ): + raise ValueError("Hostnames must be non-empty strings.") + + hostnames.add(normalized) + + return hostnames + + +def validate_secret_key(value: object) -> str | bytes: + """Require a non-placeholder, sufficiently long deployment signing key.""" + if not isinstance(value, (str, bytes)): + raise TypeError( + "Configure a persistent SECRET_KEY of at least 32 random bytes " + "before starting." + ) + + # Flask accepts either text or bytes as its signing key. Reject common + # example/development values even when their lengths look sufficient. + key = value.encode("utf-8") if isinstance(value, str) else value + if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip(): + raise RuntimeError( + "Configure a persistent SECRET_KEY of at least 32 random bytes " + "before starting." + ) + + normalized = key.strip().lower() + if normalized.startswith( + (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_") + ): + raise RuntimeError( + "SECRET_KEY is an example/development placeholder; " + "generate and configure a real random key." + ) + + return value + + +def validate_config( + config: Config, *, config_class: type[DefaultConfig] +) -> dict[str, object]: + """Validate runtime configuration and return normalized values. + + The caller applies the returned values once; validation does not mutate + the Flask application or install extensions. + """ + normalized: dict[str, object] = {} + for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"): + try: + normalized[key] = normalize_hostnames(config[key]) + except (TypeError, ValueError) as exc: + raise type(exc)(f"{key}: {exc}") from exc + + proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] + if type(proxy_hops) is not int or proxy_hops < 0: + raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.") + + for key in ( + "MAX_CONTENT_LENGTH", + "MAX_WEBMENTION_URL_BYTES", + "RECEIVED_RETENTION_DAYS", + ): + value = config[key] + if type(value) is not int or value <= 0: + raise ValueError(f"{key} must be a positive integer.") + + if not config["TESTING"] and not ( + issubclass(config_class, DevelopmentConfig) + and config["DEBUG"] + and config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY + ): + normalized["SECRET_KEY"] = validate_secret_key(config.get("SECRET_KEY")) + + if not config["DEBUG"] and not config["TESTING"]: + if not normalized["ALLOWED_HOSTNAMES"]: + raise RuntimeError( + "Configure ALLOWED_HOSTNAMES before running in production." + ) + if config["RATELIMIT_STORAGE_URI"] == "memory://": + raise RuntimeError( + "Configure RATELIMIT_STORAGE_URI with shared storage before running in production." + ) + huey_url = config.get( + "HUEY_URL", config.get("HUEY", {}).get("url", "memory://") + ) + if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in { + "memory", + "blackhole", + }: + raise RuntimeError( + "Configure HUEY_URL with a shared, persistent queue before " + "running in production (memory:// and blackhole:// are unsafe)." + ) + return normalized diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py new file mode 100644 index 0000000..0afec9c --- /dev/null +++ b/webmentions_ssg/extensions.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unbound extensions shared across Flask application instances.""" + +from flask_bootstrap import Bootstrap5 +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address +from flask_mail import Mail +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect +from sqlalchemy.orm import DeclarativeBase + +from .tasks.extension import Huey + + +class Base(DeclarativeBase): + pass + + +CSRF = CSRFProtect() +BOOTSTRAP = Bootstrap5() +DATABASE = SQLAlchemy(model_class=Base) +MIGRATE = Migrate() +HUEY = Huey() +MAIL = Mail() +LIMITER = Limiter(key_func=get_remote_address) diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms.py index 384c7a1..a721dc6 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms.py @@ -2,19 +2,21 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Provide validators for URL safety and form field constraints.""" +"""Webmention forms and URL validation.""" import ipaddress +import re import unicodedata from urllib.parse import urlsplit import httpx from flask import current_app -from wtforms import Field, ValidationError -from wtforms.validators import StopValidation +from flask_wtf import FlaskForm +from wtforms import Field, PasswordField, StringField, SubmitField, ValidationError from wtforms.form import BaseForm +from wtforms.validators import URL, InputRequired, Regexp, StopValidation -from ..url_security import AddressResolutionError, is_public_address, is_public_url +from .net import is_public_address class URLMaxBytes: @@ -143,7 +145,7 @@ class AllowedHostname: :raises ValidationError: If the URL hostname is not configured as allowed. """ try: - hostname = urlsplit(field.data).hostname + hostname = (urlsplit(field.data).hostname or "").rstrip(".") except ValueError, TypeError: hostname = None if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]: @@ -151,42 +153,6 @@ class AllowedHostname: raise ValidationError(self.message or field.gettext("Invalid input.")) -class PublicURL: - """ - Validate that a URL resolves exclusively to public IP addresses. - - :param message: Optional validation error message. - """ - - def __init__(self, message: str | None = None) -> None: - """ - Initialize the validator. - - :param message: Optional validation error message. - """ - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - """ - Validate that a URL resolves exclusively to public IP addresses. - - :param form: Form containing the field. - :param field: Field containing the URL to validate. - :raises ValidationError: If the URL cannot be resolved, has no hostname, or - resolves to a non-public address. - """ - message = self.message - - if message is None: - message = field.gettext("URL must resolve to a public address.") - - try: - if not is_public_url(field.data): - raise ValidationError(message) - except (AddressResolutionError, ValueError) as exc: - raise ValidationError(message) from exc - - class PublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" @@ -206,3 +172,62 @@ class PublicIPLiteral: if not is_public_address(address): raise ValidationError(field.gettext("URL must use a public address.")) + + +class LoginForm(FlaskForm): + """ + Form for authenticating a user. + """ + + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") + + +class EndpointForm(FlaskForm): + """ + Form for receiving a Webmention. + + The source must be a public HTTP or HTTPS URL distinct from the target. + The target must be an HTTP or HTTPS URL using an allowed hostname. + """ + + source = StringField( + "source", + validators=[ + InputRequired(), + URLMaxBytes(), + HTTPXURL(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="source must begin with http or https", + ), + NotEqualTo("target"), + PublicIPLiteral(), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + URLMaxBytes(), + HTTPXURL(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="target must begin with http or https", + ), + AllowedHostname(), + ], + ) + + +class AdminActionForm(FlaskForm): + """ + Form for confirming an administrative action. + """ + + submit = SubmitField("Confirm") diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py deleted file mode 100644 index 589cbd9..0000000 --- a/webmentions_ssg/forms/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Define forms for login, Webmention submission, and administration.""" - -import re - -from flask_wtf import FlaskForm -from wtforms import PasswordField, StringField, SubmitField -from wtforms.validators import URL, InputRequired, Regexp - -from .validators import ( - AllowedHostname, - HTTPXURL, - NotEqualTo, - PublicIPLiteral, - URLMaxBytes, -) - - -class LoginForm(FlaskForm): - """ - Form for authenticating a user. - """ - - username = StringField("Username", validators=[InputRequired()]) - password = PasswordField("Password", validators=[InputRequired()]) - submit = SubmitField("Sign In") - - -class EndpointForm(FlaskForm): - """ - Form for receiving a Webmention. - - The source must be a public HTTP or HTTPS URL distinct from the target. - The target must be an HTTP or HTTPS URL using an allowed hostname. - """ - - source = StringField( - "source", - validators=[ - InputRequired(), - URLMaxBytes(), - HTTPXURL(), - URL(), - Regexp( - "^https?://.*", - flags=re.IGNORECASE, - message="source must begin with http or https", - ), - NotEqualTo("target"), - PublicIPLiteral(), - ], - ) - target = StringField( - "target", - validators=[ - InputRequired(), - URLMaxBytes(), - HTTPXURL(), - URL(), - Regexp( - "^https?://.*", - flags=re.IGNORECASE, - message="target must begin with http or https", - ), - AllowedHostname(), - ], - ) - - -class AdminActionForm(FlaskForm): - """ - Form for confirming an administrative action. - """ - - submit = SubmitField("Confirm") diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py index 905b2c2..a263a2f 100644 --- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py +++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Track webmention digest notifications Revision ID: 408fc2eca9ce diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py index c4eac52..ee6ad09 100644 --- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Add Webmention retry backoff Revision ID: 85f2b226ec99 diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py index cbb40ef..08dd45f 100644 --- a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py +++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Index received webmentions for expired verification recovery. Revision ID: c0272f25ce17 diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 0157df3..41c5376 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,7 +11,7 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.url_digest import received_url_digest, sent_target_digest +from webmentions_ssg.models import received_url_digest, sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 31516d5..9442309 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import uuid from datetime import UTC, datetime from enum import StrEnum @@ -28,8 +29,17 @@ from sqlalchemy import ( from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from werkzeug.security import check_password_hash, generate_password_hash -from . import Base -from .url_digest import received_url_digest, sent_target_digest +from .extensions import Base + + +class ReceivedStatus(StrEnum): + """Persisted lifecycle state of an incoming Webmention.""" + + RECEIVED = "received" + VERIFYING = "verifying" + VERIFIED = "verified" + FAILED = "failed" + DELETED = "deleted" class SentWebmentionStatus(StrEnum): @@ -120,7 +130,9 @@ class ReceivedWebmention(Base): self.url_digest = received_url_digest(source, target) return value - status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + status: Mapped[str] = mapped_column( + Text(), nullable=False, default=ReceivedStatus.RECEIVED + ) # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh # explicit submission resets this counter. recovery_attempts: Mapped[int] = mapped_column( @@ -154,6 +166,24 @@ class ReceivedWebmention(Base): """ return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" + @staticmethod + def verification_reset_values( + *, at: datetime, recoveries: int = 0 + ) -> dict[str, object]: + """Return the columns to reset when verification is explicitly reopened.""" + return { + "status": ReceivedStatus.RECEIVED, + "failure_reason": None, + "recovery_attempts": recoveries, + "updated_at": at, + } + + def reset_for_verification(self) -> ReceivedWebmention: + """Reopen an existing row for manual verification and return it.""" + for key, value in self.verification_reset_values(at=datetime.now(UTC)).items(): + setattr(self, key, value) + return self + @property def verified(self) -> bool: """ @@ -161,7 +191,7 @@ class ReceivedWebmention(Base): :return: Whether the Webmention status is ``verified``. """ - return self.status == "verified" + return self.status == ReceivedStatus.VERIFIED @property def created_at(self) -> datetime: @@ -308,6 +338,23 @@ class SentWebmention(Base): """ return f"<SentWebmention {self.target!r}>" + def reopen( + self, revision: int, active: bool, *, resend: bool = False + ) -> SentWebmention: + """Reopen delivery for a revision while preserving send history.""" + self.desired_revision = revision + self.active = active + self.attempt_count = 0 + self.next_retry_at = None + if resend: + self.processed_revision = None + self.status = None + self.failure_reason = None + self.endpoint = None + self.response_status = None + self.status_url = None + return self + @property def pending(self) -> bool: """ @@ -338,3 +385,16 @@ def uuid7_to_datetime(identifier: uuid.UUID) -> datetime: :return: Timestamp as a timezone-aware UTC datetime. """ return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) + + +def received_url_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair.""" + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def sent_target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID).""" + return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/net.py index 131cf55..01a9a80 100644 --- a/webmentions_ssg/public_transport.py +++ b/webmentions_ssg/net.py @@ -2,23 +2,31 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""HTTP transport that connects only to validated DNS answers. - -httpcore receives the original host for Host and TLS SNI/certificate checks, -but the socket is opened to an IP from a single, bounded DNS lookup. This -protects the receiver and sender, including redirects and advertised URLs. -""" +"""Public-only HTTP transport, URL checks, and bounded response reads.""" import ipaddress import socket import time from collections.abc import Iterable +from urllib.parse import urlsplit +import dns.exception +import dns.resolver import httpcore import httpx -from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation +from flask import current_app +from httpcore._backends.sync import SyncStream + +NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") +IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") + + +class AddressResolutionError(Exception): + """A hostname could not be resolved (potentially transient).""" -from .url_security import public_addresses + +class UnsafeAddressError(ValueError): + """A hostname or IP literal identifies a non-public network address.""" class NetworkDeadline: @@ -142,3 +150,114 @@ class PublicHTTPTransport(httpx.HTTPTransport): ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)), ) + + +def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client: + """Return the hardened HTTP client shared by receiver and sender.""" + headers = {"User-Agent": user_agent} + if accept is not None: + headers["Accept"] = accept + return httpx.Client( + headers=headers, + timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]), + follow_redirects=True, + max_redirects=current_app.config["MAX_REDIRECTS"], + trust_env=False, + transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]), + ) + + +def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """Reject non-public addresses, including IPv4 hidden in IPv6 forms. + + ``is_global`` alone does not reliably exclude IPv6 transition addresses + across Python versions or networks with NAT64/6to4 routing. + """ + if isinstance(address, ipaddress.IPv6Address): + # The well-known NAT64 prefix can translate an otherwise global IPv6 + # address into a private IPv4 destination. IPv4-compatible addresses + # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. + if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX: + return False + + # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 + # destination through ipaddress; validate that destination as well. + for embedded in (address.sixtofour, address.ipv4_mapped): + if embedded is not None and not embedded.is_global: + return False + + return address.is_global + + +def public_addresses( + hostname: str, *, lifetime: float | None = None +) -> tuple[str, ...]: + """Resolve *once* and return only checked, globally routable IP addresses. + + A DNS name with a mixture of public and private answers is rejected in its + entirety. IP literals are handled without DNS, including IPv6 literals. + The returned strings must be used for the actual TCP connection; checking + them and then connecting to ``hostname`` reintroduces DNS rebinding. + """ + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + + if literal is not None: + addresses = (literal,) + else: + try: + answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) + addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) + except (dns.exception.DNSException, ValueError) as exc: + raise AddressResolutionError( + f"Could not resolve hostname {hostname!r}" + ) from exc + + if not addresses: + raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") + + if any(not is_public_address(address) for address in addresses): + raise UnsafeAddressError(f"Non-public network address for {hostname!r}") + + return tuple(str(address) for address in addresses) + + +def is_http_url(url: str) -> bool: + """ + Check whether a URL is an absolute HTTP or HTTPS URL. + + :param url: URL to validate. + :return: Whether the URL uses HTTP or HTTPS and contains a hostname. + """ + try: + parsed = urlsplit(url) + except ValueError: + return False + + return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None + + +def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes: + """Read a streaming response, rejecting oversized declared or actual bodies.""" + if (content_length := response.headers.get("Content-Length")) is not None: + try: + declared_length = int(content_length) + except ValueError: + pass + else: + if declared_length > max_bytes: + raise ValueError(f"{label} document is too large") + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) + if len(body) > max_bytes: + raise ValueError(f"{label} document is too large") + return bytes(body) + + +def is_temporary_status(status: int) -> bool: + """Return whether an HTTP status is normally retryable.""" + return status in {408, 425, 429} or 500 <= status <= 599 diff --git a/webmentions_ssg/security.py b/webmentions_ssg/security.py deleted file mode 100644 index f4841a4..0000000 --- a/webmentions_ssg/security.py +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Validate persistent signing keys and administrator passwords.""" - -MIN_PASSWORD_LENGTH = 8 -MIN_SECRET_KEY_LENGTH = 32 - - -def validate_secret_key(value: object) -> None: - """Require a non-placeholder, sufficiently long deployment signing key.""" - if not isinstance(value, (str, bytes)): - raise TypeError( - "Configure a persistent SECRET_KEY of at least 32 random bytes " - "before starting." - ) - - # Flask accepts either text or bytes as its signing key. Reject common - # example/development values even when their lengths look sufficient. - key = value.encode("utf-8") if isinstance(value, str) else value - if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip(): - raise RuntimeError( - "Configure a persistent SECRET_KEY of at least 32 random bytes " - "before starting." - ) - - normalized = key.strip().lower() - if normalized.startswith( - (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_") - ): - raise RuntimeError( - "SECRET_KEY is an example/development placeholder; " - "generate and configure a real random key." - ) - - -def validate_password(password: str) -> None: - """Validate newly chosen passwords without arbitrary composition rules. - - Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases, - but reject empty passwords and control characters. - """ - if not password: - raise ValueError("Password must not be empty.") - if len(password) < MIN_PASSWORD_LENGTH: - raise ValueError( - f"Password must be at least {MIN_PASSWORD_LENGTH} characters long." - ) - if any(ord(char) < 32 or ord(char) == 127 for char in password): - raise ValueError("Password must not contain control characters.") diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index fb1d46d..bf7189f 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,7 +21,7 @@ def create_consumer() -> Huey: """ app = create_app() - from . import notifications, receiver, retention, scanner, sender # noqa: F401 + from . import notifications, receiver, retention, scanner schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py new file mode 100644 index 0000000..98f3fd5 --- /dev/null +++ b/webmentions_ssg/tasks/discovery.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Discover a target Webmention endpoint and submit a POST.""" + +import re +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup +from flask import current_app + +from ..net import is_http_url, is_temporary_status, read_limited_body + +LINK_SPLIT = re.compile(r",\s*(?=<)") +DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} + + +class SenderError(Exception): + """A Webmention cannot be sent.""" + + +class TemporarySenderError(SenderError): + """A Webmention send failed for a potentially temporary reason.""" + + +class PermanentSenderError(SenderError): + """A Webmention send failed permanently for this source revision.""" + + +def resolve_endpoint(response: httpx.Response, href: str) -> str: + """ + Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :return: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or + HTTPS URL. + """ + try: + endpoint = urljoin(str(response.url), href.strip()) + except ValueError as exc: + raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc + + if not is_http_url(endpoint): + raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") + + return endpoint + + +def parse_link_value(value: str) -> tuple[str, set[str]] | None: + """ + Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :return: Link target and relation names, or ``None`` if the value is invalid. + """ + value = value.strip() + + if not value.startswith("<"): + return None + + href, separator, parameters = value[1:].partition(">") + + if not separator: + return None + + relations: set[str] = set() + + for parameter in parameters.split(";"): + name, separator, value = parameter.partition("=") + + if separator and name.strip().lower() == "rel": + relations.update( + relation.lower() for relation in value.strip().strip("\"'").split() + ) + + return href, relations + + +def endpoint_from_headers(response: httpx.Response) -> str | None: + """ + Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + for header in response.headers.get_list("Link"): + for value in LINK_SPLIT.split(header): + if (link := parse_link_value(value)) is None: + continue + + href, relations = link + + if "webmention" in relations: + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: + """ + Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + document = BeautifulSoup(body, "html5lib") + + for element in document.find_all(["link", "a"], href=True): + relations = element.get("rel") + + if isinstance(relations, str): + relations = relations.split() + + if not relations: + continue + + if not any( + isinstance(relation, str) and relation.lower() == "webmention" + for relation in relations + ): + continue + + href = element.get("href") + + if not isinstance(href, str): + continue + + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def read_target_body(response: httpx.Response) -> bytes: + """Return a bounded target body, using the shared HTTP size guard.""" + try: + return read_limited_body( + response, current_app.config["MAX_TARGET_BYTES"], label="Target" + ) + except ValueError as exc: + raise PermanentSenderError(str(exc)) from exc + + +def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: + """ + Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises TemporarySenderError: If the target returns a temporary HTTP failure. + :raises PermanentSenderError: If GET returns a permanent HTTP failure. + """ + try: + head_response = client.head(target, headers=DISCOVERY_HEADERS) + except httpx.RequestError, TemporarySenderError: + # HEAD is only an optimization. Servers and intermediaries may reject + # or drop it even when fetching the document with GET would succeed. + pass + else: + if endpoint := endpoint_from_headers(head_response): + return endpoint + + with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: + if endpoint := endpoint_from_headers(response): + return endpoint + + if is_temporary_status(response.status_code): + raise TemporarySenderError(f"Target returned HTTP {response.status_code}") + + if not response.is_success: + raise PermanentSenderError(f"Target returned HTTP {response.status_code}") + + media_type = ( + response.headers.get("Content-Type", "").partition(";")[0].strip().lower() + ) + + if media_type not in {"text/html", "application/xhtml+xml"}: + return None + + return endpoint_from_html(response, read_target_body(response)) + + +def post_webmention( + client: httpx.Client, *, endpoint: str, source: str, target: str +) -> tuple[int, str | None]: + """ + Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :return: HTTP response status code and optional status URL. + """ + with client.stream( + "POST", endpoint, data={"source": source, "target": target} + ) as response: + status_url = None + + if response.status_code == 201 and ( + location := response.headers.get("Location") + ): + try: + candidate = urljoin(str(response.url), location) + except ValueError: + # A malformed Location must not fail an otherwise successful send. + pass + else: + if is_http_url(candidate): + status_url = candidate + + return (response.status_code, status_url) diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index df6cb21..9f53f3b 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -93,54 +93,37 @@ class Huey: ) return self._huey + def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]: + """Return a task wrapper that executes with the configured Flask app.""" + + @wraps(func) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + if self.app is None: + raise RuntimeError("Flask app is not available.") + with self.app.app_context(): + return func(*args, **kwargs) + + return wrapped + def task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's task decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's task decorator. - :return: Decorator that registers the wrapped function as a Huey task. - """ + """Return a Huey task decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.task(*task_args, **task_kwargs)(wrapper) + return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func)) return decorator def periodic_task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a periodic Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's periodic task - decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's periodic task - decorator. - :return: Decorator that registers the wrapped function as a periodic Huey - task. - """ + """Return a periodic Huey decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper) + return self.huey.periodic_task(*task_args, **task_kwargs)( + self._in_app_context(func) + ) return decorator diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py index 473a544..7e51358 100644 --- a/webmentions_ssg/tasks/json_serializer.py +++ b/webmentions_ssg/tasks/json_serializer.py @@ -11,7 +11,7 @@ from typing import Any from huey.registry import Message from huey.serializer import Serializer -from huey.utils import Error, SKIPPED +from huey.utils import SKIPPED, Error class JSONSerializer(Serializer): @@ -39,66 +39,87 @@ class JSONSerializer(Serializer): @classmethod def _encode(cls, value: Any) -> Any: - if value is None or type(value) in {str, int, float, bool}: - return value - if isinstance(value, uuid.UUID): - return {"type": "uuid", "value": str(value)} - if isinstance(value, datetime): - return {"type": "datetime", "value": value.isoformat()} - if isinstance(value, Message): - return {"type": "message", "value": [cls._encode(v) for v in value]} - if isinstance(value, Error): - return {"type": "error", "value": cls._encode(value.metadata)} - if value is SKIPPED: - return {"type": "skipped", "value": None} - if isinstance(value, tuple): - return {"type": "tuple", "value": [cls._encode(v) for v in value]} - if isinstance(value, list): - return {"type": "list", "value": [cls._encode(v) for v in value]} - if isinstance(value, dict) and all(type(k) is str for k in value): - return { - "type": "dict", - "value": [[k, cls._encode(v)] for k, v in value.items()], - } - raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") + match value: + case _ if value is None or type(value) in {str, int, float, bool}: + return value + + case uuid.UUID(): + return {"type": "uuid", "value": str(value)} + + case datetime(): + return {"type": "datetime", "value": value.isoformat()} + + case Message(): + return {"type": "message", "value": [cls._encode(v) for v in value]} + + case Error(): + return {"type": "error", "value": cls._encode(value.metadata)} + + case _ if value is SKIPPED: + return {"type": "skipped", "value": None} + + case tuple(): + return {"type": "tuple", "value": [cls._encode(v) for v in value]} + + case list(): + return {"type": "list", "value": [cls._encode(v) for v in value]} + + case dict() if all(isinstance(k, str) for k in value): + return { + "type": "dict", + "value": [[k, cls._encode(v)] for k, v in value.items()], + } + + case _: + raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") @classmethod def _decode(cls, value: Any) -> Any: if value is None or type(value) in {str, int, float, bool}: return value + if not isinstance(value, dict) or set(value) != {"type", "value"}: raise ValueError("Invalid Huey JSON data") - kind, payload = value["type"], value["value"] - if kind == "uuid" and isinstance(payload, str): - return uuid.UUID(payload) - if kind == "datetime" and isinstance(payload, str): - return datetime.fromisoformat(payload) - if kind == "message" and isinstance(payload, list): - if len(payload) != len(Message._fields): - raise ValueError("Invalid Huey JSON message field count") - return Message(*(cls._decode(v) for v in payload)) - if kind == "error": - metadata = cls._decode(payload) - if not isinstance(metadata, dict): - raise ValueError("Invalid Huey JSON error metadata") - return Error(metadata) - if kind == "skipped" and payload is None: - return SKIPPED - if kind in {"tuple", "list"} and isinstance(payload, list): - items = [cls._decode(v) for v in payload] - return tuple(items) if kind == "tuple" else items - if kind == "dict" and isinstance(payload, list): - result: dict[str, Any] = {} - for pair in payload: - if ( - not isinstance(pair, list) - or len(pair) != 2 - or type(pair[0]) is not str - ): - raise ValueError("Invalid Huey JSON dictionary entry") - if pair[0] in result: - raise ValueError("Duplicate Huey JSON dictionary key") - result[pair[0]] = cls._decode(pair[1]) - return result - raise ValueError("Unsupported Huey JSON type") + match value["type"], value["value"]: + case "uuid", str() as payload: + return uuid.UUID(payload) + + case "datetime", str() as payload: + return datetime.fromisoformat(payload) + + case "message", list() as payload: + if len(payload) != len(Message._fields): + raise ValueError("Invalid Huey JSON message field count") + return Message(*(cls._decode(v) for v in payload)) + + case "error", payload: + metadata = cls._decode(payload) + if not isinstance(metadata, dict): + raise TypeError("Invalid Huey JSON error metadata") + return Error(metadata) + + case "skipped", None: + return SKIPPED + + case ("tuple" | "list") as kind, list() as payload: + items = [cls._decode(v) for v in payload] + return tuple(items) if kind == "tuple" else items + + case "dict", list() as payload: + result: dict[str, Any] = {} + + for pair in payload: + match pair: + case list([str() as key, encoded]): + if key in result: + raise ValueError("Duplicate Huey JSON dictionary key") + result[key] = cls._decode(encoded) + + case _: + raise ValueError("Invalid Huey JSON dictionary entry") + + return result + + case _: + raise ValueError("Unsupported Huey JSON type") diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py new file mode 100644 index 0000000..4e5a5ca --- /dev/null +++ b/webmentions_ssg/tasks/mentions.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure HTML and text matching for received Webmention targets.""" + +from urllib.parse import urljoin + +import rfc3987 +from bs4 import BeautifulSoup + +IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") +TRAILING_IRI_PUNCTUATION = ".,;:!?" + +HTML_URL_ATTRIBUTES = { + "href": {"a", "area", "link"}, + "src": { + "audio", + "embed", + "iframe", + "img", + 'input[type="image" i]', + "script", + "audio source", + "video source", + "track", + "video", + }, + "cite": {"blockquote", "del", "ins", "q"}, +} + + +def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: + """ + Check whether an HTML document mentions the target URL. + + URL references are resolved against the document's ``base`` element when + present, or against the source URL otherwise. + + :param body: HTML document body. + :param source_url: URL from which the document was retrieved. + :param target_url: Exact URL to look for. + :return: Whether a supported HTML URL attribute references the target URL. + """ + + document = BeautifulSoup(body, "html.parser") + base_url = source_url + + if (base_element := document.select_one("base[href]")) is not None and isinstance( + base_href := base_element.get("href"), str + ): + base_url = urljoin(source_url, base_href.strip()) + + for attribute, selectors in HTML_URL_ATTRIBUTES.items(): + selector = ", ".join( + [f"{selector_string}[{attribute}]" for selector_string in selectors] + ) + + for element in document.select(selector): + if not isinstance(reference := element.get(attribute), str): + continue + + if urljoin(base_url, reference.strip()) == target_url: + return True + + return False + + +def text_mentions_target(body: str, target_url: str) -> bool: + """ + Check whether plain text contains the target IRI, ignoring trailing prose + punctuation but not extra URL path or query characters. + + :param body: Plain-text document body. + :param target_url: Exact IRI to look for. + :return: Whether the target IRI occurs in the document. + """ + for match in IRI_PATTERN.finditer(body): + candidate = match.group() + + # A URL in prose can be followed by punctuation that the IRI pattern + # consumes as part of its match. Only remove trailing punctuation, + # leaving longer URLs (e.g. ``target/more``) unmatched. + if not candidate.startswith(target_url): + continue + + while candidate != target_url: + if candidate[-1] in TRAILING_IRI_PUNCTUATION: + candidate = candidate[:-1] + elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): + # Keep balanced parentheses that belong to the URL itself. + candidate = candidate[:-1] + else: + break + + if candidate == target_url: + return True + + return False diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index 475fad1..889030c 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -10,10 +10,10 @@ import sqlalchemy as sa from flask import current_app from flask_mail import Message -from .. import DATABASE as db -from .. import HUEY as huey -from .. import MAIL as mail -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..extensions import MAIL as mail +from ..models import ReceivedStatus, ReceivedWebmention @huey.lock_task("send-webmention-digest") @@ -27,7 +27,7 @@ def send_webmention_digest() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == "verified", + ReceivedWebmention.status == ReceivedStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) @@ -53,7 +53,6 @@ def send_webmention_digest() -> None: ), ) - # If SMTP fails, the records remain pending for the next digest. mail.send(message) db.session.execute( diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index acd66a1..bed923c 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -2,27 +2,27 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Fetch and verify incoming Webmentions against their source pages.""" +"""Fetch and verify incoming Webmentions with transactional state changes.""" import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx -import rfc3987 import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey -from ..models import ReceivedWebmention -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..models import ReceivedStatus, ReceivedWebmention +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_temporary_status, + read_limited_body, +) +from .mentions import html_mentions_target, text_mentions_target class VerificationError(Exception): @@ -37,95 +37,7 @@ class TemporaryFetchError(Exception): """Fetching the source may succeed when retried later.""" -IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") -TRAILING_IRI_PUNCTUATION = ".,;:!?" - - -HTML_URL_ATTRIBUTES = { - "href": {"a", "area", "link"}, - "src": { - "audio", - "embed", - "iframe", - "img", - 'input[type="image" i]', - "script", - "audio source", - "video source", - "track", - "video", - }, - "cite": {"blockquote", "del", "ins", "q"}, -} - - -def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """ - Check whether an HTML document mentions the target URL. - - URL references are resolved against the document's ``base`` element when - present, or against the source URL otherwise. - - :param body: HTML document body. - :param source_url: URL from which the document was retrieved. - :param target_url: Exact URL to look for. - :return: Whether a supported HTML URL attribute references the target URL. - """ - - document = BeautifulSoup(body, "html.parser") - base_url = source_url - - if (base_element := document.select_one("base[href]")) is not None and isinstance( - base_href := base_element.get("href"), str - ): - base_url = urljoin(source_url, base_href.strip()) - - for attribute, selectors in HTML_URL_ATTRIBUTES.items(): - selector = ", ".join( - [f"{selector_string}[{attribute}]" for selector_string in selectors] - ) - - for element in document.select(selector): - if not isinstance(reference := element.get(attribute), str): - continue - - if urljoin(base_url, reference.strip()) == target_url: - return True - - return False - - -def text_mentions_target(body: str, target_url: str) -> bool: - """ - Check whether plain text contains the target IRI, ignoring trailing prose - punctuation but not extra URL path or query characters. - - :param body: Plain-text document body. - :param target_url: Exact IRI to look for. - :return: Whether the target IRI occurs in the document. - """ - for match in IRI_PATTERN.finditer(body): - candidate = match.group() - - # A URL in prose can be followed by punctuation that the IRI pattern - # consumes as part of its match. Only remove trailing punctuation, - # leaving longer URLs (e.g. ``target/more``) unmatched. - if not candidate.startswith(target_url): - continue - - while candidate != target_url: - if candidate[-1] in TRAILING_IRI_PUNCTUATION: - candidate = candidate[:-1] - elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): - # Keep balanced parentheses that belong to the URL itself. - candidate = candidate[:-1] - else: - break - - if candidate == target_url: - return True - - return False +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: @@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ try: - return _fetch_source(source_url) + with ( + build_client( + f"{APP_NAME}/{VERSION} ReceivedWebmention", + accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if is_temporary_status(status): + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") + + try: + return response, read_limited_body( + response, current_app.config["MAX_SOURCE_BYTES"], label="Source" + ) + except ValueError as exc: + raise VerificationError(str(exc)) from exc + except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc -def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - with ( - httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client, - client.stream("GET", source_url) as response, - ): - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") - - max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass - - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_source_bytes: - raise VerificationError("Source document is too large") - - return response, bytes(body) - - def source_mentions_target(source_url: str, target_url: str) -> bool: """ Fetch a source and check whether it mentions the target URL. @@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - # Failed tasks may be retried by Huey. Completed and already-running tasks - # should not start another concurrent verification. - if webmention.status not in {"received", "failed"}: + if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: return - # A reaper-exhausted row must not be revived by a late Huey retry. + if ( - webmention.status == "failed" + webmention.status == ReceivedStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return @@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ReceivedWebmention.status == previous_status, ReceivedWebmention.updated_at == previous_updated_at, ) - .values(status="verifying", failure_reason=None, updated_at=started_at) + .values( + status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at + ) ) + if claimed.rowcount != 1: db.session.rollback() return - db.session.commit() - def finish(status: str, failure_reason: str | None) -> None: - # A timeout or a newer verification may have superseded this attempt. - # Do not let a late worker overwrite its state. - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == "verifying", - ReceivedWebmention.updated_at == started_at, - ) - .values( - status=status, - failure_reason=failure_reason, - updated_at=datetime.now(UTC), - ) - ) - db.session.commit() + db.session.commit() + status = None + failure_reason = None try: mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - finish("deleted", str(exc)) + status = ReceivedStatus.DELETED + failure_reason = str(exc) except VerificationError as exc: - finish("failed", str(exc)) + status = ReceivedStatus.FAILED + failure_reason = str(exc) except (TemporaryFetchError, httpx.RequestError) as exc: - finish("failed", str(exc) or "Source could not be fetched") + status = ReceivedStatus.FAILED + failure_reason = str(exc) or "Source could not be fetched" raise except Exception as exc: # InvalidURL is not a RequestError; other unexpected source errors must @@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", webmention_uuid ) - finish("failed", str(exc) or type(exc).__name__) + + status = ReceivedStatus.FAILED + failure_reason = str(exc) or type(exc).__name__ else: if mentions_target: - finish("verified", None) + status = ReceivedStatus.VERIFIED else: - finish("deleted", "Source does not mention target") + status = ReceivedStatus.DELETED + failure_reason = "Source does not mention target" + finally: + if status is not None: + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() def recover_stale_webmentions() -> int: @@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int: """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( sa.select( ReceivedWebmention.uuid, @@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.updated_at, ReceivedWebmention.recovery_attempts, ).where( - ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.status.in_( + (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + ), ReceivedWebmention.updated_at <= cutoff, ) ).all() @@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int: recovered = 0 exhausted = 0 max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( @@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.recovery_attempts == recovery_attempts, ) .values( - status="failed" if at_limit else "received", - failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, - updated_at=now, - recovery_attempts=( - recovery_attempts if at_limit else recovery_attempts + 1 - ), + **( + { + "status": ReceivedStatus.FAILED, + "failure_reason": RECOVERY_EXHAUSTED_REASON, + "updated_at": now, + "recovery_attempts": recovery_attempts, + } + if at_limit + else ReceivedWebmention.verification_reset_values( + at=now, recoveries=recovery_attempts + 1 + ) + ) ) ) + if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: exhausted += 1 continue diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 727b186..bf79512 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,8 +11,8 @@ import sqlalchemy as sa from flask import current_app from sqlalchemy.engine import CursorResult -from .. import DATABASE as db -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..models import ReceivedStatus, ReceivedWebmention def purge_old_received_webmentions() -> int: @@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int: cutoff = datetime.now(UTC) - timedelta( days=current_app.config["RECEIVED_RETENTION_DAYS"] ) + result = cast( CursorResult[Any], db.session.execute( sa.delete(ReceivedWebmention).where( - ReceivedWebmention.status.in_(("failed", "deleted")), + ReceivedWebmention.status.in_( + (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ), ReceivedWebmention.updated_at < cutoff, ) ), ) db.session.commit() + removed = result.rowcount if removed: current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + return removed diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 666eba7..b2d6def 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -2,404 +2,22 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Scan generated site pages to track outgoing Webmention targets.""" +"""Reconcile scanned static-site sources with the database.""" -import json -from collections.abc import Iterator -from dataclasses import dataclass from datetime import UTC, datetime -from fnmatch import fnmatchcase -from itertools import chain from pathlib import Path -from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit -import mf2py import sqlalchemy as sa -import xxhash -from bs4 import BeautifulSoup, NavigableString, Tag -from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload -from .. import DATABASE as db -from .. import HUEY as huey -from ..config import normalize_ignored_hostnames +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, Source -from ..url_security import is_http_url +from ..net import is_http_url from .sender import send_webmention - -REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") - -CONTENT_BLOCK_TAGS = frozenset( - { - "address", - "article", - "aside", - "blockquote", - "br", - "dd", - "div", - "dl", - "dt", - "figcaption", - "figure", - "footer", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "header", - "hr", - "li", - "main", - "ol", - "p", - "pre", - "section", - "table", - "td", - "th", - "tr", - "ul", - } -) - - -class SourceScanError(Exception): - """A source document cannot safely be processed.""" - - -@dataclass(frozen=True) -class ScannedSource: - path: str - url: str - content_hash: str - targets: frozenset[str] - - -def source_url_for_path(relative_path: Path, base_url: str) -> str: - """ - Derive the public source URL from a relative filesystem path. - - :param relative_path: Path of the source document relative to the source root. - :param base_url: Base URL under which source documents are published. - :return: Public URL corresponding to the source document. - """ - directory = relative_path.parent.as_posix() - - return urljoin(base_url, f"{quote(directory, safe='/')}/") - - -def canonical_url( - document: BeautifulSoup, *, relative_path: Path, base_url: str | None -) -> str | None: - """ - Return the canonical URL declared by a source document. - - Relative canonical URLs are resolved against the configured source base URL. - - :param document: Parsed HTML source document. - :param relative_path: Path of the source document relative to the source root. - :param base_url: Configured source base URL, if available. - :return: Canonical URL, or ``None`` if none is declared. - :raises SourceScanError: If the canonical URL is empty, cannot be resolved, - or does not resolve to an HTTP or HTTPS URL. - """ - if (link := document.select_one('link[rel~="canonical"][href]')) is not None: - href = link.get("href") - - if not isinstance(href, str) or not (href := href.strip()): - raise SourceScanError("Document contains an empty canonical URL") - - if is_http_url(href): - return href - - if base_url is None: - raise SourceScanError( - "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" - ) - - resolved = urljoin(source_url_for_path(relative_path, base_url), href) - - if not is_http_url(resolved): - raise SourceScanError( - f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" - ) - - return resolved - - -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """ - Yield URL values from a microformats property. - - :param entry: Parsed microformats entry. - :param property_name: Name of the property to inspect. - :return: Iterator over string values of the property. - """ - properties = entry.get("properties") - - if not isinstance(properties, dict): - return - - values = properties.get(property_name) - - if not isinstance(values, list): - return - - yield from (value for value in values if isinstance(value, str)) - - -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """ - Parse and validate an h-entry with mf2py. - - :param element: HTML element containing the h-entry. - :param base_url: Base URL used when parsing the microformats data. - :return: Parsed h-entry. - :raises SourceScanError: If parsing does not produce exactly one valid h-entry. - """ - parsed = mf2py.parse(doc=str(element), url=base_url) - - if not isinstance(parsed, dict): - raise SourceScanError("Microformats parser did not return a document object") - - items = parsed.get("items") - - if not isinstance(items, list) or len(items) != 1: - raise SourceScanError("Expected exactly one parsed h-entry") - - entry = items[0] - - if not isinstance(entry, dict): - raise SourceScanError("Parsed h-entry is not an object") - - types = entry.get("type") - - if not isinstance(types, list) or "h-entry" not in types: - raise SourceScanError("Parsed microformats item is not an h-entry") - - return entry - - -def primary_entry( - document: BeautifulSoup, source_url: str -) -> tuple[Tag, dict[str, Any]]: - """ - Return and validate the primary h-entry of a source document. - - The document must contain exactly one h-entry whose ``u-url`` matches the - source URL. - - :param document: Parsed HTML source document. - :param source_url: Public URL of the source document. - :return: h-entry element and its parsed microformats representation. - :raises SourceScanError: If the document does not contain exactly one h-entry - or its ``u-url`` does not match the source URL. - """ - entries = document.find_all(class_="h-entry") - - if len(entries) != 1: - raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") - - entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) - - urls = tuple(property_urls(mf2_entry, "url")) - - if source_url not in urls: - raise SourceScanError( - f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" - ) - - return entry_element, mf2_entry - - -def content_element(entry: Tag) -> Tag: - """ - Return the e-content element of an h-entry. - - :param entry: HTML element containing the h-entry. - :return: The h-entry's e-content element. - :raises SourceScanError: If the h-entry does not contain exactly one - e-content element. - """ - contents = entry.find_all(class_="e-content") - - if len(contents) != 1: - raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") - - return contents[0] - - -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """ - Resolve and validate a possible Webmention target URL. - - Empty values, non-HTTP URLs, and URLs referring to the source itself are - discarded. - - :param value: URL reference to normalize. - :param base_url: Base URL against which relative references are resolved. - :param source_url: URL of the source document. - :return: Normalized target URL, or ``None`` if the value is not a valid - Webmention target. - """ - value = value.strip() - - if not value: - return None - - target = urljoin(base_url, value) - - if not is_http_url(target): - return None - - if urldefrag(target)[0] == urldefrag(source_url)[0]: - return None - - return target - - -def iter_targets( - content: Tag, - mf2_entry: dict[str, Any], - *, - base_url: str, - source_url: str, - ignored_hostnames: tuple[str, ...] = (), -) -> Iterator[str]: - """ - Yield outgoing Webmention targets from an h-entry. - - Targets are collected from links in the entry content and supported - microformats reaction properties. Invalid, self-referencing, and ignored - targets are excluded. - - :param content: e-content element of the h-entry. - :param mf2_entry: Parsed microformats representation of the h-entry. - :param base_url: Base URL against which relative targets are resolved. - :param source_url: URL of the source document. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Iterator over outgoing Webmention target URLs. - """ - hrefs = ( - href - for element in content.find_all("a", href=True) - if isinstance(href := element.get("href"), str) - ) - - reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES - ) - - for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) - if target is not None: - hostname = urlsplit(target).hostname or "" - if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): - yield target - - -def normalized_content(content: Tag) -> str: - """Extract stable reader-facing content without generated HTML attributes.""" - - def extract(node: PageElement) -> str: - if isinstance(node, NavigableString): - return str(node) - if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: - return "" - - # A relative-time label may change on every site build even though its - # machine-readable date stays the same. - if node.name == "time" and isinstance(value := node.get("datetime"), str): - return value.strip() - - # Image-only posts also have meaningful content, despite having no text. - if node.name == "img": - return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " - - text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text - - return " ".join(extract(content).split()) - - -def scan_source_file( - path: Path, - *, - root: Path, - base_url: str | None, - ignored_hostnames: tuple[str, ...] = (), -) -> ScannedSource: - """ - Scan one generated source document for outgoing Webmentions. - - :param path: Path of the generated HTML document. - :param root: Root directory containing generated source documents. - :param base_url: Configured source base URL, if available. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Scanned source metadata and discovered targets. - :raises SourceScanError: If the document cannot be safely interpreted as a - Webmention source. - :raises OSError: If the source document cannot be read. - """ - # Use the path as published for its URL, but read only files whose - # resolved target stays inside the configured source directory. Resolving - # before reading also protects against symlinked index.html files. - resolved_root = root.resolve() - resolved_path = path.resolve() - if not resolved_path.is_relative_to(resolved_root): - raise SourceScanError(f"Source document escapes source directory: {path}") - - relative_path = path.relative_to(root) - document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) - - if source_url is None: - if base_url is None: - raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" - ) - - source_url = source_url_for_path(relative_path, base_url) - - entry_element, mf2_entry = primary_entry(document, source_url) - - content = content_element(entry_element) - - targets = frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ) - - # Fingerprint the meaningful post text and its outgoing targets rather than - # the full h-entry markup. Relative timestamps, classes, and other volatile - # metadata outside e-content must not cause repeat Webmentions. - stable_content = normalized_content(content) - fingerprint = json.dumps( - {"content": stable_content, "targets": sorted(targets)}, - ensure_ascii=False, - separators=(",", ":"), - ) - - return ScannedSource( - path=relative_path.as_posix(), - url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), - targets=targets, - ) +from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: @@ -466,10 +84,9 @@ def update_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.active = webmention.target in scanned.targets - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen( + source.revision, webmention.target in scanned.targets + ) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -523,10 +140,6 @@ def scan_sources_task() -> None: raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" - ignored_hostnames = normalize_ignored_hostnames( - current_app.config["IGNORED_HOSTNAMES"] - ) - sources_by_path = { source.path: source for source in db.session.scalars( @@ -555,7 +168,7 @@ def scan_sources_task() -> None: index_path, root=root, base_url=base_url, - ignored_hostnames=ignored_hostnames, + ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -592,10 +205,6 @@ def scan_sources_task() -> None: except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - # A static-site generator can temporarily empty its output directory while - # rebuilding. With no successfully scanned pages, absence is not evidence - # of deletion; do not generate removals or send pending Webmentions that - # might be verified while the site is unavailable. if scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " @@ -612,10 +221,7 @@ def scan_sources_task() -> None: source.deleted_at = scan_time for webmention in source.sent_webmentions: - webmention.active = False - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 9330247..3c4c717 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -2,272 +2,33 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Discover endpoints and deliver outgoing Webmentions.""" +"""Claim, send, and persist outgoing Webmention attempts.""" -import re import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from huey.exceptions import TaskLockedException from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, SentWebmentionStatus -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url - -LINK_SPLIT = re.compile(r",\s*(?=<)") - -DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} - - -class SenderError(Exception): - """A Webmention cannot be sent.""" - - -class TemporarySenderError(SenderError): - """A Webmention send failed for a potentially temporary reason.""" - - -class PermanentSenderError(SenderError): - """A Webmention send failed permanently for this source revision.""" - - -def temporary_http_status(status: int) -> bool: - """ - Return whether an HTTP status indicates a temporary failure. - - :param status: HTTP response status code. - :return: Whether the request should be retried. - """ - return status in {408, 425, 429} or 500 <= status <= 599 - - -def resolve_endpoint(response: httpx.Response, href: str) -> str: - """ - Resolve and validate a discovered Webmention endpoint. - - :param response: Response from which the endpoint was discovered. - :param href: Endpoint reference to resolve. - :return: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or - HTTPS URL. - """ - try: - endpoint = urljoin(str(response.url), href.strip()) - except ValueError as exc: - raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc - - if not is_http_url(endpoint): - raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") - - return endpoint - - -def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """ - Parse a Link header value into its target and relations. - - :param value: Link header value to parse. - :return: Link target and relation names, or ``None`` if the value is invalid. - """ - value = value.strip() - - if not value.startswith("<"): - return None - - href, separator, parameters = value[1:].partition(">") - - if not separator: - return None - - relations: set[str] = set() - - for parameter in parameters.split(";"): - name, separator, value = parameter.partition("=") - - if separator and name.strip().lower() == "rel": - relations.update( - relation.lower() for relation in value.strip().strip("\"'").split() - ) - - return href, relations - - -def endpoint_from_headers(response: httpx.Response) -> str | None: - """ - Find a Webmention endpoint in the response Link headers. - - :param response: HTTP response whose headers should be inspected. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - for header in response.headers.get_list("Link"): - for value in LINK_SPLIT.split(header): - if (link := parse_link_value(value)) is None: - continue - - href, relations = link - - if "webmention" in relations: - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """ - Find a Webmention endpoint in an HTML document. - - ``link`` and ``a`` elements are inspected in document order. - - :param response: HTTP response from which the document was retrieved. - :param body: HTML document body. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - document = BeautifulSoup(body, "html5lib") - - for element in document.find_all(["link", "a"], href=True): - relations = element.get("rel") - - if isinstance(relations, str): - relations = relations.split() - - if not relations: - continue - - if not any( - isinstance(relation, str) and relation.lower() == "webmention" - for relation in relations - ): - continue - - href = element.get("href") - - if not isinstance(href, str): - continue - - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def read_target_body(response: httpx.Response) -> bytes: - """ - Read a target document up to the configured size limit. - - :param response: Streaming HTTP response to read. - :return: Response body. - :raises PermanentSenderError: If the document exceeds the configured maximum - size. - """ - max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_bytes: - raise PermanentSenderError("Target document is too large") - except ValueError: - pass - - body = bytearray() - - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_bytes: - raise PermanentSenderError("Target document is too large") - - return bytes(body) - - -def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """ - Discover the Webmention endpoint advertised by a target URL. - - Endpoint discovery first checks the response to a HEAD request and then falls - back to a GET request, inspecting both HTTP Link headers and supported HTML - documents. - - :param client: HTTP client to use for discovery requests. - :param target: Target URL whose Webmention endpoint should be discovered. - :return: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If GET returns a permanent HTTP failure. - """ - try: - head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError, TemporarySenderError: - # HEAD is only an optimization. Servers and intermediaries may reject - # or drop it even when fetching the document with GET would succeed. - pass - else: - if endpoint := endpoint_from_headers(head_response): - return endpoint - - with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := endpoint_from_headers(response): - return endpoint - - if temporary_http_status(response.status_code): - raise TemporarySenderError(f"Target returned HTTP {response.status_code}") - - if not response.is_success: - raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) - - if media_type not in {"text/html", "application/xhtml+xml"}: - return None - - return endpoint_from_html(response, read_target_body(response)) - - -def post_webmention( - client: httpx.Client, *, endpoint: str, source: str, target: str -) -> tuple[int, str | None]: - """ - Send a Webmention to a discovered endpoint. - - :param client: HTTP client to use for the request. - :param endpoint: Webmention endpoint URL. - :param source: Source URL of the Webmention. - :param target: Target URL of the Webmention. - :return: HTTP response status code and optional status URL. - """ - with client.stream( - "POST", endpoint, data={"source": source, "target": target} - ) as response: - status_url = None - - if response.status_code == 201 and ( - location := response.headers.get("Location") - ): - try: - candidate = urljoin(str(response.url), location) - except ValueError: - # A malformed Location must not fail an otherwise successful send. - pass - else: - if is_http_url(candidate): - status_url = candidate - - return (response.status_code, status_url) +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_http_url, + is_temporary_status, +) +from .discovery import ( + PermanentSenderError, + TemporarySenderError, + discover_webmention_endpoint, + post_webmention, +) def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: now = datetime.now(UTC) deadline = webmention.next_retry_at - # SQLite returns timezone-naive datetimes even for timezone-aware columns. if deadline is not None: aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) if aware_deadline > now: @@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: endpoint: str | None = None response_status: int | None = None + status_url: str | None = None + reason: str | None = None + terminal = True + temporary_failure = False try: if not is_http_url(target): raise PermanentSenderError(f"Invalid target URL: {target!r}") - with httpx.Client( - headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")}, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client: + with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.UNSUPPORTED - webmention.failure_reason = "No Webmention endpoint discovered" - webmention.endpoint = None - webmention.response_status = None - - db.session.commit() - return - - (response_status, status_url) = post_webmention( - client, endpoint=endpoint, source=source, target=target - ) - - if not (200 <= response_status <= 299): - message = f"Webmention endpoint returned HTTP {response_status}" - - if temporary_http_status(response_status): - raise TemporarySenderError(message) - - raise PermanentSenderError(message) + outcome = SentWebmentionStatus.UNSUPPORTED + reason = "No Webmention endpoint discovered" + else: + response_status, status_url = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + if not 200 <= response_status <= 299: + message = f"Webmention endpoint returned HTTP {response_status}" + if is_temporary_status(response_status): + raise TemporarySenderError(message) + raise PermanentSenderError(message) + outcome = SentWebmentionStatus.SENT except ( PermanentSenderError, @@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: ValueError, UnicodeError, ) as exc: - # Malformed URLs and local request/protocol errors cannot be repaired - # by retrying this source revision. InvalidURL is not a RequestError; - # protocol errors are, so handle them before the retryable branch. - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or type(exc).__name__ - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - return - + # Invalid URLs and local protocol errors cannot improve with retries. + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or type(exc).__name__ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: - if not attempt_is_current(webmention, revision): - return - - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or "Webmention request failed" - if attempt >= max_attempts: - webmention.processed_revision = revision - webmention.next_retry_at = None - current_app.logger.warning( - "Webmention %s exhausted %d send attempts: %s", - webmention_uuid, - attempt, - webmention.failure_reason, - ) - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - - # No Huey retries: only the scanner may requeue after next_retry_at. - return + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or "Webmention request failed" + terminal = attempt >= max_attempts + temporary_failure = True + # A scanner may have reopened or superseded the revision during I/O. + # Refresh only once, and persist one outcome atomically with its metadata. if not attempt_is_current(webmention, revision): return - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.sent_revision = revision - - webmention.status = SentWebmentionStatus.SENT - webmention.failure_reason = None - + webmention.status = outcome + webmention.failure_reason = reason webmention.endpoint = endpoint webmention.response_status = response_status - webmention.status_url = ( - status_url if status_url is not None and is_http_url(status_url) else None - ) - webmention.last_sent_at = datetime.now(UTC) + if terminal: + webmention.processed_revision = revision + webmention.next_retry_at = None + + if outcome == SentWebmentionStatus.SENT: + webmention.sent_revision = revision + webmention.status_url = ( + status_url if status_url is not None and is_http_url(status_url) else None + ) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() + + if temporary_failure and not terminal: + # Huey does not retry: scanner requeues after next_retry_at. + current_app.logger.info( + "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt + ) + elif temporary_failure and terminal: + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + reason, + ) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py new file mode 100644 index 0000000..cb15ab6 --- /dev/null +++ b/webmentions_ssg/tasks/source_parser.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Extract canonical source URLs, content fingerprints, and outgoing links.""" + +import json +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from fnmatch import fnmatchcase +from itertools import chain +from pathlib import Path +from typing import Any +from urllib.parse import quote, urldefrag, urljoin, urlsplit + +import mf2py +import xxhash +from bs4 import BeautifulSoup, Tag +from bs4.element import NavigableString, PageElement + +from ..net import is_http_url + +REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") + +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + + +class SourceScanError(Exception): + """A source document cannot safely be processed.""" + + +@dataclass(frozen=True) +class ScannedSource: + path: str + url: str + content_hash: str + targets: frozenset[str] + + +def source_url_for_path(relative_path: Path, base_url: str) -> str: + """ + Derive the public source URL from a relative filesystem path. + + :param relative_path: Path of the source document relative to the source root. + :param base_url: Base URL under which source documents are published. + :return: Public URL corresponding to the source document. + """ + directory = relative_path.parent.as_posix() + + return urljoin(base_url, f"{quote(directory, safe='/')}/") + + +def canonical_url( + document: BeautifulSoup, *, relative_path: Path, base_url: str | None +) -> str | None: + """ + Return the canonical URL declared by a source document. + + Relative canonical URLs are resolved against the configured source base URL. + + :param document: Parsed HTML source document. + :param relative_path: Path of the source document relative to the source root. + :param base_url: Configured source base URL, if available. + :return: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, + or does not resolve to an HTTP or HTTPS URL. + """ + if (link := document.select_one('link[rel~="canonical"][href]')) is not None: + href = link.get("href") + + if not isinstance(href, str) or not (href := href.strip()): + raise SourceScanError("Document contains an empty canonical URL") + + if is_http_url(href): + return href + + if base_url is None: + raise SourceScanError( + "Document contains a relative canonical URL, but " + "SOURCE_BASE_URL is not configured" + ) + + resolved = urljoin(source_url_for_path(relative_path, base_url), href) + + if not is_http_url(resolved): + raise SourceScanError( + f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" + ) + + return resolved + + +def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: + """ + Yield URL values from a microformats property. + + :param entry: Parsed microformats entry. + :param property_name: Name of the property to inspect. + :return: Iterator over string values of the property. + """ + properties = entry.get("properties") + + if not isinstance(properties, dict): + return + + values = properties.get(property_name) + + if not isinstance(values, list): + return + + yield from (value for value in values if isinstance(value, str)) + + +def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: + """ + Parse and validate an h-entry with mf2py. + + :param element: HTML element containing the h-entry. + :param base_url: Base URL used when parsing the microformats data. + :return: Parsed h-entry. + :raises SourceScanError: If parsing does not produce exactly one valid h-entry. + """ + parsed = mf2py.parse(doc=str(element), url=base_url) + + if not isinstance(parsed, dict): + raise SourceScanError("Microformats parser did not return a document object") + + items = parsed.get("items") + + if not isinstance(items, list) or len(items) != 1: + raise SourceScanError("Expected exactly one parsed h-entry") + + entry = items[0] + + if not isinstance(entry, dict): + raise SourceScanError("Parsed h-entry is not an object") + + types = entry.get("type") + + if not isinstance(types, list) or "h-entry" not in types: + raise SourceScanError("Parsed microformats item is not an h-entry") + + return entry + + +def primary_entry( + document: BeautifulSoup, source_url: str +) -> tuple[Tag, dict[str, Any]]: + """ + Return and validate the primary h-entry of a source document. + + The document must contain exactly one h-entry whose ``u-url`` matches the + source URL. + + :param document: Parsed HTML source document. + :param source_url: Public URL of the source document. + :return: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry + or its ``u-url`` does not match the source URL. + """ + entries = document.find_all(class_="h-entry") + + if len(entries) != 1: + raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") + + entry_element = entries[0] + mf2_entry = parse_entry(entry_element, source_url) + + urls = tuple(property_urls(mf2_entry, "url")) + + if source_url not in urls: + raise SourceScanError( + f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" + ) + + return entry_element, mf2_entry + + +def content_element(entry: Tag) -> Tag: + """ + Return the e-content element of an h-entry. + + :param entry: HTML element containing the h-entry. + :return: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one + e-content element. + """ + contents = entry.find_all(class_="e-content") + + if len(contents) != 1: + raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") + + return contents[0] + + +def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: + """ + Resolve and validate a possible Webmention target URL. + + Empty values, non-HTTP URLs, and URLs referring to the source itself are + discarded. + + :param value: URL reference to normalize. + :param base_url: Base URL against which relative references are resolved. + :param source_url: URL of the source document. + :return: Normalized target URL, or ``None`` if the value is not a valid + Webmention target. + """ + value = value.strip() + + if not value: + return None + + target = urljoin(base_url, value) + + if not is_http_url(target): + return None + + if urldefrag(target)[0] == urldefrag(source_url)[0]: + return None + + return target + + +def iter_targets( + content: Tag, + mf2_entry: dict[str, Any], + *, + base_url: str, + source_url: str, + ignored_hostnames: Iterable[str] = (), +) -> Iterator[str]: + """ + Yield outgoing Webmention targets from an h-entry. + + Targets are collected from links in the entry content and supported + microformats reaction properties. Invalid, self-referencing, and ignored + targets are excluded. + + :param content: e-content element of the h-entry. + :param mf2_entry: Parsed microformats representation of the h-entry. + :param base_url: Base URL against which relative targets are resolved. + :param source_url: URL of the source document. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Iterator over outgoing Webmention target URLs. + """ + hrefs = ( + href + for element in content.find_all("a", href=True) + if isinstance(href := element.get("href"), str) + ) + + reactions = chain.from_iterable( + property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + ) + + for value in chain(hrefs, reactions): + target = normalize_target(value, base_url=base_url, source_url=source_url) + if target is not None: + hostname = (urlsplit(target).hostname or "").rstrip(".") + if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): + yield target + + +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + +def scan_source_file( + path: Path, + *, + root: Path, + base_url: str | None, + ignored_hostnames: Iterable[str] = (), +) -> ScannedSource: + """ + Scan one generated source document for outgoing Webmentions. + + :param path: Path of the generated HTML document. + :param root: Root directory containing generated source documents. + :param base_url: Configured source base URL, if available. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Scanned source metadata and discovered targets. + :raises SourceScanError: If the document cannot be safely interpreted as a + Webmention source. + :raises OSError: If the source document cannot be read. + """ + # Use the path as published for its URL, but read only files whose + # resolved target stays inside the configured source directory. Resolving + # before reading also protects against symlinked index.html files. + resolved_root = root.resolve() + resolved_path = path.resolve() + if not resolved_path.is_relative_to(resolved_root): + raise SourceScanError(f"Source document escapes source directory: {path}") + + relative_path = path.relative_to(root) + document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") + + source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + + if source_url is None: + if base_url is None: + raise SourceScanError( + "Document has no canonical URL and SOURCE_BASE_URL is not configured" + ) + + source_url = source_url_for_path(relative_path, base_url) + + entry_element, mf2_entry = primary_entry(document, source_url) + + content = content_element(entry_element) + + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + + return ScannedSource( + path=relative_path.as_posix(), + url=source_url, + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, + ) diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index 1ff3690..3dbb896 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -6,6 +6,33 @@ SPDX-License-Identifier: BSD-3-Clause {% extends "base.html" %} {% from "bootstrap5/pagination.html" import render_pagination %} +{% set received_status_presentation = { + "received": { + "row_class": "table-info", + "badge_class": "text-bg-info", + "label": "Pending" + }, + "verifying": { + "row_class": "table-warning", + "badge_class": "text-bg-warning", + "label": "Verifying" + }, + "verified": { + "row_class": "table-success", + "badge_class": "text-bg-success", + "label": "Verified" + }, + "failed": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Declined" + }, + "deleted": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Declined" + } +} %} {% block content %} <div class="d-flex justify-content-between align-items-center mb-3"> @@ -27,27 +54,9 @@ SPDX-License-Identifier: BSD-3-Clause </thead> <tbody> {% for webmention in webmentions %} - {% if webmention.status == "received" %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% set display_status = "Pending" %} - {% elif webmention.status == "verifying" %} - {% set row_class = "table-warning" %} - {% set badge_class = "text-bg-warning" %} - {% set display_status = "Verifying" %} - {% elif webmention.status == "verified" %} - {% set row_class = "table-success" %} - {% set badge_class = "text-bg-success" %} - {% set display_status = "Verified" %} - {% else %} - {% set row_class = "table-danger" %} - {% set badge_class = "text-bg-danger" %} - {% set display_status = "Declined" %} - {% endif %} - <tr class="{{ row_class }}"> - <td> - <span class="badge {{ badge_class }}">{{ display_status }}</span> - </td> + {% set style = received_status_presentation.get(webmention.status, {"row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined"}) %} + <tr class="{{ style.row_class }}"> + <td><span class="badge {{ style.badge_class }}">{{ style.label }}</span></td> <td class="text-break"> <a href="{{ webmention.source }}">{{ webmention.source }}</a> </td> diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 689c651..974c7b3 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -6,6 +6,23 @@ SPDX-License-Identifier: BSD-3-Clause {% extends "base.html" %} {% from "bootstrap5/pagination.html" import render_pagination %} +{% set sent_status_presentation = { + "sent": { + "row_class": "table-success", + "badge_class": "text-bg-success", + "label": "Sent" + }, + "unsupported": { + "row_class": "table-warning", + "badge_class": "text-bg-warning", + "label": "Unsupported" + }, + "failed": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Failed" + } +} %} {% block content %} <div class="mb-3"> @@ -85,32 +102,13 @@ SPDX-License-Identifier: BSD-3-Clause </thead> <tbody> {% for webmention in webmentions %} - {% if webmention.status is none %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% elif webmention.status.value == "sent" %} - {% set row_class = "table-success" %} - {% set badge_class = "text-bg-success" %} - {% elif webmention.status.value == "unsupported" %} - {% set row_class = "table-warning" %} - {% set badge_class = "text-bg-warning" %} - {% elif webmention.status.value == "failed" %} - {% set row_class = "table-danger" %} - {% set badge_class = "text-bg-danger" %} - {% else %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% endif %} - <tr class="{{ row_class }}"> + {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %} + <tr class="{{ style.row_class }}"> <td class="text-break"> <a href="{{ webmention.target }}">{{ webmention.target }}</a> </td> <td> - {% if webmention.status is none %} - <span class="badge {{ badge_class }}">Pending</span> - {% else %} - <span class="badge {{ badge_class }}">{{ webmention.status.value.title() }}</span> - {% endif %} + <span class="badge {{ style.badge_class }}">{{ style.label }}</span> {% if webmention.failure_reason is not none %} <div class="small mt-2">{{ webmention.failure_reason }}</div> {% endif %} diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py deleted file mode 100644 index 256d037..0000000 --- a/webmentions_ssg/url_digest.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Portable SHA-256 fingerprints for indexed Webmention URLs.""" - -import hashlib -import json - - -def received_url_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair.""" - encoded = json.dumps( - [source, target], ensure_ascii=False, separators=(",", ":") - ).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def sent_target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID).""" - return hashlib.sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py deleted file mode 100644 index c4324e0..0000000 --- a/webmentions_ssg/url_security.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Validate URL schemes and publicly routable network destinations.""" - -import ipaddress -from urllib.parse import urlsplit - -import dns.exception -import dns.resolver - - -class AddressResolutionError(Exception): - """A hostname could not be resolved (potentially transient).""" - - -class UnsafeAddressError(ValueError): - """A hostname or IP literal identifies a non-public network address.""" - - -_NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") -_IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") - - -def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: - """Reject non-public addresses, including IPv4 hidden in IPv6 forms. - - ``is_global`` alone does not reliably exclude IPv6 transition addresses - across Python versions or networks with NAT64/6to4 routing. - """ - if isinstance(address, ipaddress.IPv6Address): - # The well-known NAT64 prefix can translate an otherwise global IPv6 - # address into a private IPv4 destination. IPv4-compatible addresses - # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. - if address in _NAT64_PREFIX or address in _IPV4_COMPATIBLE_PREFIX: - return False - - # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 - # destination through ipaddress; validate that destination as well. - for embedded in (address.sixtofour, address.ipv4_mapped): - if embedded is not None and not embedded.is_global: - return False - - return address.is_global - - -def public_addresses( - hostname: str, *, lifetime: float | None = None -) -> tuple[str, ...]: - """Resolve *once* and return only checked, globally routable IP addresses. - - A DNS name with a mixture of public and private answers is rejected in its - entirety. IP literals are handled without DNS, including IPv6 literals. - The returned strings must be used for the actual TCP connection; checking - them and then connecting to ``hostname`` reintroduces DNS rebinding. - """ - try: - literal = ipaddress.ip_address(hostname) - except ValueError: - literal = None - - if literal is not None: - addresses = (literal,) - else: - try: - answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) - addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) - except (dns.exception.DNSException, ValueError) as exc: - raise AddressResolutionError( - f"Could not resolve hostname {hostname!r}" - ) from exc - - if not addresses: - raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") - - if any(not is_public_address(address) for address in addresses): - raise UnsafeAddressError(f"Non-public network address for {hostname!r}") - - return tuple(str(address) for address in addresses) - - -def is_public_url(url: str) -> bool: - """Return whether the URL resolves entirely to public IP addresses. - - This is useful for diagnostics, but **not** sufficient to authorize a - request: network connections must use the exact validated IP address. - """ - hostname = urlsplit(url).hostname - - if hostname is None: - raise ValueError("No hostname was specified") - - try: - public_addresses(hostname) - except UnsafeAddressError: - return False - return True - - -def is_http_url(url: str) -> bool: - """ - Check whether a URL is an absolute HTTP or HTTPS URL. - - :param url: URL to validate. - :return: Whether the URL uses HTTP or HTTPS and contains a hostname. - """ - try: - parsed = urlsplit(url) - except ValueError: - return False - - return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None - - -def is_safe_next_url(next_url: str) -> bool: - """Allow redirects only to paths on this site.""" - # Check the original string too: urlsplit("///evil.example").path is - # "/evil.example", even though browsers can treat the original as external. - if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"): - return False - - try: - target = urlsplit(next_url) - except ValueError: - return False - - return ( - not target.scheme - and not target.netloc - and target.path.startswith("/") - and not target.path.startswith("//") - ) diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py deleted file mode 100644 index 5b563f0..0000000 --- a/webmentions_ssg/views.py +++ /dev/null @@ -1,454 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Provide routes for authentication and Webmention administration.""" - -import hashlib -import secrets -import uuid -from datetime import UTC, datetime, timedelta -from typing import Any, cast - -import sqlalchemy as sa -import sqlalchemy.exc as sa_exc -from flask import ( - Blueprint, - Response, - abort, - current_app, - flash, - redirect, - render_template, - request, - url_for, -) -from flask.typing import ResponseReturnValue -from flask_limiter.util import get_remote_address -from flask_login import current_user, login_required, login_user, logout_user -from sqlalchemy.engine import CursorResult -from sqlalchemy.orm import selectinload - -from . import CSRF, LIMITER, forms -from . import DATABASE as db -from .models import ReceivedWebmention, SentWebmention, Source, User -from .tasks.receiver import verify_webmention -from .tasks.scanner import manual_scan_sources -from .tasks.sender import send_webmention -from .url_digest import received_url_digest -from .url_security import is_safe_next_url - -root_page = Blueprint("root", __name__) - -DUMMY_USER = User(username=secrets.token_urlsafe()) -DUMMY_USER.password = secrets.token_urlsafe() - - -def login_username_limit_key() -> str: - """Group submitted usernames across IPs without storing raw account names.""" - username = request.form.get("username", "").strip().casefold() - if not username: - # A missing username must not put all clients into one global bucket. - return f"missing-username:{get_remote_address()}" - digest = hashlib.sha256(username.encode("utf-8")).hexdigest() - return f"username:{digest}" - - -@root_page.route("/") -def index() -> ResponseReturnValue: - """ - Redirect to the received Webmentions view. - - :return: Redirect response to the received Webmentions view. - """ - return redirect(url_for("root.received")) - - -@root_page.route("/login", methods=["GET", "POST"]) -@LIMITER.limit( - lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"], - key_func=login_username_limit_key, - methods=["POST"], - error_message="Too many login attempts. Please try again later.", -) -@LIMITER.limit( - lambda: current_app.config["LOGIN_RATE_LIMIT"], - methods=["POST"], - error_message="Too many login attempts. Please try again later.", -) -def login() -> ResponseReturnValue: - """ - Authenticate a user and start a login session. - - Authenticated users are redirected to the application index. After a - successful login, the user is redirected to the requested local URL when - provided. - - :return: Rendered login page or redirect response. - """ - if current_user.is_authenticated: - return redirect(url_for("root.index")) - - form = forms.LoginForm() - - if form.validate_on_submit(): - user = db.session.scalar( - sa.select(User).where(User.username == form.username.data) - ) - - user_exists = user is not None - - if user is None: - user = DUMMY_USER - - if ( - form.password.data is None - or not user.check_password(form.password.data) - or not user_exists - ): - flash("Invalid username or password", "danger") - return redirect(url_for("root.login")) - login_user(user) - - next_url = request.args.get("next") - if not next_url or not is_safe_next_url(next_url): - next_url = url_for("root.index") - return redirect(next_url) - - return render_template("login.html", form=form) - - -@root_page.route("/logout", methods=("POST",)) -@login_required -def logout() -> ResponseReturnValue: - """ - End the current user's login session. - - :return: Redirect response to the application index. - """ - logout_user() - return redirect(url_for("root.index")) - - -@root_page.route("/received") -@login_required -def received() -> ResponseReturnValue: - """ - Display received Webmentions. - - :return: Rendered page containing the paginated received Webmentions. - """ - webmentions = db.paginate( - sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), - per_page=25, - ) - - return render_template( - "received.html", - title="Received Webmentions", - webmentions=webmentions, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/received/<uuid:identifier>/delete") -@login_required -def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Delete a received Webmention. - - :param identifier: Identifier of the Webmention to delete. - :return: Redirect response to the received Webmentions view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - db.session.delete(webmention) - db.session.commit() - - flash(f"Webmention {webmention.uuid} deleted.", "success") - - return redirect( - url_for("root.received", page=request.args.get("page", 1, type=int)) - ) - - -@root_page.post("/received/<uuid:identifier>/reverify") -@login_required -def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Queue a received Webmention for reverification. - - The Webmention status is reset before a new verification task is queued. - - :param identifier: Identifier of the Webmention to reverify. - :return: Redirect response to the received Webmentions view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - webmention.status = "received" - webmention.failure_reason = None - webmention.recovery_attempts = 0 - - db.session.commit() - - verify_webmention(identifier) - - flash(f"Webmention {webmention.uuid} queued for reverification.", "success") - - return redirect( - url_for("root.received", page=request.args.get("page", 1, type=int)) - ) - - -@root_page.route("/sent") -@login_required -def sent() -> ResponseReturnValue: - """ - Display sources with sent Webmentions. - - :return: Rendered page containing the paginated source list. - """ - sources = db.paginate( - sa.select(Source) - .options(selectinload(Source.sent_webmentions)) - .order_by(Source.last_seen_at.desc()), - per_page=25, - ) - return render_template( - "sent.html", - title="Sent Webmentions", - sources=sources, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/sent/rescan") -@login_required -def rescan_sent_sources() -> ResponseReturnValue: - """ - Queue a manual scan of sent Webmention sources. - - :return: Redirect response to the sent Webmentions view. - """ - form = forms.AdminActionForm() - if form.validate_on_submit(): - manual_scan_sources() - flash("Source rescan queued.", "success") - return redirect(url_for("root.sent")) - - -@root_page.route("/sent/<uuid:source_identifier>") -@login_required -def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: - """ - Display the sent Webmentions associated with a source. - - :param source_identifier: Identifier of the source to display. - :return: Rendered source details page. - """ - source = db.session.scalar( - sa.select(Source).where(Source.uuid == source_identifier) - ) - - if source is None: - return abort(404) - - webmentions = db.paginate( - sa.select(SentWebmention) - .where(SentWebmention.source_id == source.uuid) - .order_by(SentWebmention.target, SentWebmention.uuid), - per_page=25, - ) - - return render_template( - "sent_source.html", - title="Sent Webmentions", - source=source, - webmentions=webmentions, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend") -@login_required -def resend_sent_webmention( - source_identifier: uuid.UUID, webmention_identifier: uuid.UUID -) -> ResponseReturnValue: - """ - Queue an outgoing Webmention to be sent again. - - The current desired source revision is reopened for processing so the - sender task does not treat the Webmention as already completed. Historical - successful-send fields are preserved, while result fields from the previous - attempt are cleared. - - :param source_identifier: Identifier of the source owning the Webmention. - :param webmention_identifier: Identifier of the sent Webmention to resend. - :return: Redirect response to the source details view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(SentWebmention, webmention_identifier) - - if webmention is None or webmention.source_id != source_identifier: - return abort(404) - - # Reopen the existing desired revision instead of inventing a new source - # revision. send_webmention() otherwise exits when this revision has already - # been processed. - webmention.processed_revision = None - webmention.attempt_count = 0 - webmention.next_retry_at = None - webmention.status = None - webmention.failure_reason = None - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None - - db.session.commit() - - send_webmention(webmention_identifier) - - flash(f"Webmention {webmention.uuid} queued for resend.", "success") - - return redirect(url_for("root.sent_source", source_identifier=source_identifier)) - - -@root_page.post("/endpoint") -@LIMITER.limit( - lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"], - key_func=lambda: "all-clients", -) -@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"]) -@CSRF.exempt -def endpoint() -> ResponseReturnValue: - """ - Receive and queue a Webmention for verification. - - Existing Webmentions with the same source and target are reset for - reverification. Concurrent insertion of the same Webmention is handled by - retrieving and updating the row created by the competing request. - - :return: HTTP 201 response with the Webmention status URL in the - ``Location`` header, or validation errors with HTTP 400. - """ - form = forms.EndpointForm(meta={"csrf": False}) - - if not form.validate_on_submit(): - return form.errors, 400 - - source = cast(str, form.source.data) - target = cast(str, form.target.data) - - url_digest = received_url_digest(source, target) - webmention = db.session.execute( - sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) - ).scalar_one_or_none() - - # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. - if webmention is not None and (webmention.source, webmention.target) != ( - source, - target, - ): - return {"error": "Webmention URL fingerprint conflict"}, 409 - - should_verify = False - - if webmention is None: - identifier = uuid.uuid7() - - webmention = ReceivedWebmention(uuid=identifier, source=source, target=target) - - db.session.add(webmention) - - try: - db.session.commit() - should_verify = True - except sa_exc.IntegrityError: - db.session.rollback() - - # Another request may have inserted the same source/target pair - # after our SELECT but before our COMMIT. - webmention = db.session.execute( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.url_digest == url_digest - ) - ).scalar_one_or_none() - if webmention is None or (webmention.source, webmention.target) != ( - source, - target, - ): - return {"error": "Webmention URL fingerprint conflict"}, 409 - - else: - now = datetime.now(UTC) - cutoff = now - timedelta( - seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] - ) - - # Atomic conditional update: only one concurrent request can reopen an expired verification. - result = cast( - CursorResult[Any], - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention.uuid, - ReceivedWebmention.updated_at <= cutoff, - ) - .values( - status="received", - failure_reason=None, - recovery_attempts=0, - updated_at=now, - ) - .execution_options(synchronize_session=False) - ), - ) - should_verify = result.rowcount == 1 - db.session.commit() - - identifier = webmention.uuid - - if should_verify: - verify_webmention(identifier) - - status_url = url_for("root.status", identifier=str(identifier), _external=True) - - return Response(status=201, headers={"Location": status_url}) - - -@root_page.route("/status/<uuid:identifier>") -def status(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Display the verification status of a received Webmention. - - :param identifier: Identifier of the Webmention to display. - :return: Rendered Webmention status page. - """ - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - return render_template( - "status.html", title="Webmention status", webmention=webmention - ) diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py new file mode 100644 index 0000000..91966d4 --- /dev/null +++ b/webmentions_ssg/views/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Register all views under the historic root Blueprint.""" + +from flask import Blueprint + +root_page = Blueprint("root", __name__) + + +def register_routes() -> Blueprint: + """Import task-backed view handlers after Huey initialization and return the Blueprint.""" + from . import admin, endpoint, session # noqa: F401 + + return root_page diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py new file mode 100644 index 0000000..46e3b18 --- /dev/null +++ b/webmentions_ssg/views/admin.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated incoming and outgoing Webmention management.""" + +import uuid + +import sqlalchemy as sa +from flask import abort, flash, redirect, render_template, request, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.orm import selectinload + +from .. import forms +from ..extensions import DATABASE as db +from ..models import ReceivedWebmention, SentWebmention, Source +from ..tasks.receiver import verify_webmention +from ..tasks.scanner import manual_scan_sources +from ..tasks.sender import send_webmention +from . import root_page + + +@root_page.route("/received") +@login_required +def received() -> ResponseReturnValue: + """ + Display received Webmentions. + + :return: Rendered page containing the paginated received Webmentions. + """ + webmentions = db.paginate( + sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), + per_page=25, + ) + + return render_template( + "received.html", + title="Received Webmentions", + webmentions=webmentions, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/received/<uuid:identifier>/delete") +@login_required +def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Delete a received Webmention. + + :param identifier: Identifier of the Webmention to delete. + :return: Redirect response to the received Webmentions view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + db.session.delete(webmention) + db.session.commit() + + flash(f"Webmention {webmention.uuid} deleted.", "success") + + return redirect( + url_for("root.received", page=request.args.get("page", 1, type=int)) + ) + + +@root_page.post("/received/<uuid:identifier>/reverify") +@login_required +def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Queue a received Webmention for reverification. + + The Webmention status is reset before a new verification task is queued. + + :param identifier: Identifier of the Webmention to reverify. + :return: Redirect response to the received Webmentions view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + webmention = webmention.reset_for_verification() + + db.session.commit() + + verify_webmention(identifier) + + flash(f"Webmention {webmention.uuid} queued for reverification.", "success") + + return redirect( + url_for("root.received", page=request.args.get("page", 1, type=int)) + ) + + +@root_page.route("/sent") +@login_required +def sent() -> ResponseReturnValue: + """ + Display sources with sent Webmentions. + + :return: Rendered page containing the paginated source list. + """ + sources = db.paginate( + sa.select(Source) + .options(selectinload(Source.sent_webmentions)) + .order_by(Source.last_seen_at.desc()), + per_page=25, + ) + return render_template( + "sent.html", + title="Sent Webmentions", + sources=sources, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/sent/rescan") +@login_required +def rescan_sent_sources() -> ResponseReturnValue: + """ + Queue a manual scan of sent Webmention sources. + + :return: Redirect response to the sent Webmentions view. + """ + form = forms.AdminActionForm() + if form.validate_on_submit(): + manual_scan_sources() + flash("Source rescan queued.", "success") + return redirect(url_for("root.sent")) + + +@root_page.route("/sent/<uuid:source_identifier>") +@login_required +def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: + """ + Display the sent Webmentions associated with a source. + + :param source_identifier: Identifier of the source to display. + :return: Rendered source details page. + """ + source = db.session.scalar( + sa.select(Source).where(Source.uuid == source_identifier) + ) + + if source is None: + return abort(404) + + webmentions = db.paginate( + sa.select(SentWebmention) + .where(SentWebmention.source_id == source.uuid) + .order_by(SentWebmention.target, SentWebmention.uuid), + per_page=25, + ) + + return render_template( + "sent_source.html", + title="Sent Webmentions", + source=source, + webmentions=webmentions, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend") +@login_required +def resend_sent_webmention( + source_identifier: uuid.UUID, webmention_identifier: uuid.UUID +) -> ResponseReturnValue: + """ + Queue an outgoing Webmention to be sent again. + + The current desired source revision is reopened for processing so the + sender task does not treat the Webmention as already completed. Historical + successful-send fields are preserved, while result fields from the previous + attempt are cleared. + + :param source_identifier: Identifier of the source owning the Webmention. + :param webmention_identifier: Identifier of the sent Webmention to resend. + :return: Redirect response to the source details view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(SentWebmention, webmention_identifier) + + if webmention is None or webmention.source_id != source_identifier: + return abort(404) + + # Reopen the existing desired revision instead of inventing a new source + # revision. send_webmention() otherwise exits when this revision has already + # been processed. + webmention = webmention.reopen( + webmention.desired_revision, webmention.active, resend=True + ) + + db.session.commit() + + send_webmention(webmention_identifier) + + flash(f"Webmention {webmention.uuid} queued for resend.", "success") + + return redirect(url_for("root.sent_source", source_identifier=source_identifier)) diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py new file mode 100644 index 0000000..1c2fe13 --- /dev/null +++ b/webmentions_ssg/views/endpoint.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Public Webmention endpoint and verification status view.""" + +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +import sqlalchemy as sa +import sqlalchemy.exc as sa_exc +from flask import Response, abort, current_app, render_template, url_for +from flask.typing import ResponseReturnValue +from sqlalchemy.engine import CursorResult + +from .. import forms +from ..extensions import CSRF, LIMITER +from ..extensions import DATABASE as db +from ..models import ReceivedWebmention, received_url_digest +from ..tasks.receiver import verify_webmention +from . import root_page + + +def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None: + """Upsert a submission atomically and return (identifier, enqueue flag). + + ``None`` means that the URL fingerprint matches a different source/target. + No background task is enqueued until the transaction is committed. + """ + url_digest = received_url_digest(source, target) + webmention = db.session.execute( + sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) + ).scalar_one_or_none() + + # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. + if webmention is not None and (webmention.source, webmention.target) != ( + source, + target, + ): + return None + + should_verify = False + + if webmention is None: + identifier = uuid.uuid7() + webmention = ReceivedWebmention(uuid=identifier, source=source, target=target) + db.session.add(webmention) + + try: + db.session.commit() + should_verify = True + except sa_exc.IntegrityError: + db.session.rollback() + + # Another request may have inserted the same source/target pair + # after our SELECT but before our COMMIT. + webmention = db.session.execute( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.url_digest == url_digest + ) + ).scalar_one_or_none() + if webmention is None or (webmention.source, webmention.target) != ( + source, + target, + ): + return None + + else: + now = datetime.now(UTC) + cutoff = now - timedelta( + seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] + ) + + # Atomic conditional update: only one concurrent request can reopen an expired verification. + result = cast( + CursorResult[Any], + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention.uuid, + ReceivedWebmention.updated_at <= cutoff, + ) + .values(ReceivedWebmention.verification_reset_values(at=now)) + .execution_options(synchronize_session=False) + ), + ) + should_verify = result.rowcount == 1 + db.session.commit() + + return webmention.uuid, should_verify + + +@root_page.post("/endpoint") +@LIMITER.limit( + lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"], + key_func=lambda: "all-clients", +) +@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"]) +@CSRF.exempt +def endpoint() -> ResponseReturnValue: + """ + Receive and queue a Webmention for verification. + + Existing Webmentions with the same source and target are reset for + reverification. Concurrent insertion of the same Webmention is handled by + retrieving and updating the row created by the competing request. + + :return: HTTP 201 response with the Webmention status URL in the + ``Location`` header, or validation errors with HTTP 400. + """ + form = forms.EndpointForm(meta={"csrf": False}) + + if not form.validate_on_submit(): + return form.errors, 400 + + source = cast(str, form.source.data) + target = cast(str, form.target.data) + + result = submit_received(source, target) + + if result is None: + return {"error": "Webmention URL fingerprint conflict"}, 409 + + identifier, should_verify = result + + if should_verify: + verify_webmention(identifier) + + status_url = url_for("root.status", identifier=str(identifier), _external=True) + + return Response(status=201, headers={"Location": status_url}) + + +@root_page.route("/status/<uuid:identifier>") +def status(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Display the verification status of a received Webmention. + + :param identifier: Identifier of the Webmention to display. + :return: Rendered Webmention status page. + """ + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + return render_template( + "status.html", title="Webmention status", webmention=webmention + ) diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py new file mode 100644 index 0000000..7bd8a10 --- /dev/null +++ b/webmentions_ssg/views/session.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authentication and session routes.""" + +import sqlalchemy as sa +from flask import current_app, flash, redirect, render_template, request, url_for +from flask.typing import ResponseReturnValue +from flask_login import current_user, login_required, login_user, logout_user + +from .. import forms +from ..auth import DUMMY_USER, is_safe_next_url, login_username_limit_key +from ..extensions import DATABASE as db +from ..extensions import LIMITER +from ..models import User +from . import root_page + + +@root_page.route("/") +def index() -> ResponseReturnValue: + """ + Redirect to the received Webmentions view. + + :return: Redirect response to the received Webmentions view. + """ + return redirect(url_for("root.received")) + + +@root_page.route("/login", methods=["GET", "POST"]) +@LIMITER.limit( + lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"], + key_func=login_username_limit_key, + methods=["POST"], + error_message="Too many login attempts. Please try again later.", +) +@LIMITER.limit( + lambda: current_app.config["LOGIN_RATE_LIMIT"], + methods=["POST"], + error_message="Too many login attempts. Please try again later.", +) +def login() -> ResponseReturnValue: + """ + Authenticate a user and start a login session. + + Authenticated users are redirected to the application index. After a + successful login, the user is redirected to the requested local URL when + provided. + + :return: Rendered login page or redirect response. + """ + if current_user.is_authenticated: + return redirect(url_for("root.index")) + + form = forms.LoginForm() + + if form.validate_on_submit(): + user = db.session.scalar( + sa.select(User).where(User.username == form.username.data) + ) + + user_exists = user is not None + + if user is None: + user = DUMMY_USER + + if ( + form.password.data is None + or not user.check_password(form.password.data) + or not user_exists + ): + flash("Invalid username or password", "danger") + return redirect(url_for("root.login")) + login_user(user) + + next_url = request.args.get("next") + if not next_url or not is_safe_next_url(next_url): + next_url = url_for("root.index") + return redirect(next_url) + + return render_template("login.html", form=form) + + +@root_page.route("/logout", methods=("POST",)) +@login_required +def logout() -> ResponseReturnValue: + """ + End the current user's login session. + + :return: Redirect response to the application index. + """ + logout_user() + return redirect(url_for("root.index")) |
