diff options
| author | Dennis Fink | 2026-11-17 17:09:59 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-17 17:09:59 +0100 |
| commit | 6d6d079a3a99d3d3a922a640b6925eb2adf044de (patch) | |
| tree | 33cc19241f4ce86ea43b2f194087c58e3fdc8a4d | |
| parent | 227092b57acc886469e16bcc4c0d3256e86d3d1e (diff) | |
| download | webmentions-ssg-6d6d079a3a99d3d3a922a640b6925eb2adf044de.tar.gz webmentions-ssg-6d6d079a3a99d3d3a922a640b6925eb2adf044de.zip | |
refactor(core)!: split forms and models by feature
Replace monolithic form and model modules with focused packages for
receiver, moderation, source, user, and shared validation concerns.
Rename source policy concepts to domain moderation, including routes,
models, tables, configuration, templates, and tests.
BREAKING CHANGE: form and model import paths, moderation routes, and
`SOURCE_ALLOWLIST_ENABLED` have changed; use `ALLOWLIST_ENABLED` and the
new feature-specific modules.
63 files changed, 2235 insertions, 1949 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 882c86d..9be2cf0 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -95,12 +95,12 @@ MAX_REDIRECTS = 20 # also used for outgoing requests # Webmention. Include each hostname you serve; wildcards are not supported. ALLOWED_TARGET_HOSTNAMES = {"example.com", "www.example.com"} -# Optional source allowlist mode. By default, every valid public source is +# Optional allowlist mode. By default, every valid public source is # accepted unless its hostname appears in the admin-managed blocklist. # When enabled, a source must match an allowed domain; blocked entries always -# override allowed entries. Manage both lists in the authenticated "Source rules" +# override allowed entries. Manage both lists in the authenticated "Moderation" # page. Entries include subdomains; an empty allowlist rejects all sources. -SOURCE_ALLOWLIST_ENABLED = False +ALLOWLIST_ENABLED = False # Maximum size of an incoming HTTP request body, including form encoding. This # global Flask limit also applies to login and administration routes. diff --git a/tests/forms/__init__.py b/tests/forms/__init__.py new file mode 100644 index 0000000..25d78d5 --- /dev/null +++ b/tests/forms/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/forms/test_admin.py b/tests/forms/test_admin.py new file mode 100644 index 0000000..cb011ee --- /dev/null +++ b/tests/forms/test_admin.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from flask import Flask + +from webmentions_ssg.forms.admin import ConfirmActionForm + + +def test_admin_action_form_accepts_submission(app: Flask) -> None: + with app.test_request_context(method="POST"): + form = ConfirmActionForm(meta={"csrf": False}) + assert form.validate() diff --git a/tests/test_forms.py b/tests/forms/test_endpoint.py index c6a8b25..11f233d 100644 --- a/tests/test_forms.py +++ b/tests/forms/test_endpoint.py @@ -5,26 +5,13 @@ import pytest from flask import Flask from werkzeug.datastructures import MultiDict -from wtforms import Form, StringField, ValidationError -from webmentions_ssg.forms import ( - AllowedTargetHostname, - ConfirmActionForm, - LoginForm, - MaxURLBytes, - ReceivedWebmentionFilterForm, - RejectNonPublicIPLiteral, - WebmentionSubmissionForm, -) +from webmentions_ssg.forms.endpoint import WebmentionSubmissionForm VALID_SOURCE = "https://source.example/post" VALID_TARGET = "https://dennisfink.me/blog/example/" -class URLForm(Form): - url = StringField("URL") - - @pytest.mark.parametrize( ("form_data", "invalid_field", "expected_error"), [ @@ -118,122 +105,6 @@ def test_endpoint_form_accepts_valid_data(app: Flask) -> None: assert form.errors == {} -@pytest.mark.parametrize( - ("form_data", "invalid_field"), - [ - pytest.param({"password": "secret"}, "username", id="username-required"), - pytest.param({"username": "admin"}, "password", id="password-required"), - ], -) -def test_login_form_requires_credentials( - app: Flask, form_data: dict[str, str], invalid_field: str -) -> None: - with app.test_request_context("/login", method="POST"): - form = LoginForm(formdata=MultiDict(form_data), meta={"csrf": False}) - - assert not form.validate() - assert "This field is required." in form.errors[invalid_field] - - -def test_login_form_accepts_credentials(app: Flask) -> None: - with app.test_request_context("/login", method="POST"): - form = LoginForm( - formdata=MultiDict({"username": "admin", "password": "secret"}), - meta={"csrf": False}, - ) - assert form.validate() - - -def test_received_filter_form_accepts_valid_get_params(app: Flask) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict( - { - "status": "declined", - "q": "example.org", - "date_from": "2026-11-08", - "date_to": "2026-11-14", - } - ), - meta={"csrf": False}, - ) - assert form.validate() - assert form.status.data == "declined" - assert form.q.data == "example.org" - assert form.date_from.data.isoformat() == "2026-11-08" - assert form.date_to.data.isoformat() == "2026-11-14" - assert "csrf_token" not in form._fields - - -@pytest.mark.parametrize( - "data", - [ - {"status": "unknown"}, - {"date_from": "not-a-date"}, - {"date_from": "2026-11-8"}, - {"date_to": "2026-02-30"}, - {"date_from": "2026-11-15", "date_to": "2026-11-08"}, - ], -) -def test_received_filter_form_rejects_invalid_get_params( - app: Flask, data: dict[str, str] -) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict(data), meta={"csrf": False} - ) - assert not form.validate() - assert form.errors - - -def test_received_filter_form_validates_sort_and_page_size(app: Flask) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict({"sort": "updated", "order": "asc", "per_page": "37"}), - meta={"csrf": False}, - ) - assert form.validate() - assert form.sort.data == "updated" - assert form.order.data == "asc" - assert form.per_page.data == 37 - - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm(formdata=MultiDict(), meta={"csrf": False}) - assert form.validate() - assert (form.sort.data, form.order.data, form.per_page.data) == ( - "received", - "desc", - 25, - ) - - -@pytest.mark.parametrize( - "data", - [ - {"sort": "invalid"}, - {"order": "invalid"}, - {"per_page": "0"}, - {"per_page": "101"}, - {"per_page": "invalid"}, - ], -) -def test_received_filter_form_rejects_invalid_sort_and_page_size( - app: Flask, data: dict[str, str] -) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict(data), meta={"csrf": False} - ) - assert not form.validate() - assert form.errors - - -def test_admin_action_form_accepts_submission(app: Flask) -> None: - with app.test_request_context(method="POST"): - form = ConfirmActionForm(meta={"csrf": False}) - assert form.validate() - - def test_endpoint_does_not_do_dns_in_web_request( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -355,73 +226,3 @@ def test_endpoint_form_rejects_matching_source_and_target(app: Flask) -> None: assert not form.validate() assert "Field must not be equal to target." in form.errors["source"] - - -def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) - with app.app_context(): - AllowedTargetHostname()(form, form.url) - - -def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedTargetHostname()(form, form.url) - - -def test_allowed_hostname_uses_custom_message(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with ( - app.app_context(), - pytest.raises(ValidationError, match="Hostname is not allowed"), - ): - AllowedTargetHostname("Hostname is not allowed")(form, form.url) - - -@pytest.mark.parametrize( - "address", - ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], -) -def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: - form = URLForm(MultiDict({"url": f"https://[{address}]/"})) - with pytest.raises(ValidationError, match="URL must use a public address"): - RejectNonPublicIPLiteral()(form, form.url) - - -def test_public_ip_literal_accepts_public_ipv6_address() -> None: - form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) - RejectNonPublicIPLiteral()(form, form.url) - - -def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: - form = URLForm(MultiDict({"url": "http://[::1"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedTargetHostname()(form, form.url) - - -def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: - validator = MaxURLBytes() - form = URLForm(MultiDict({"url": "é" * 3})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 6 - validator(form, form.url) - app.config["MAX_WEBMENTION_URL_BYTES"] = 5 - with pytest.raises(ValidationError, match="5 bytes"): - validator(form, form.url) - - -def test_url_max_bytes_custom_message(app: Flask) -> None: - form = URLForm(MultiDict({"url": "too-long"})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 2 - with pytest.raises(ValidationError, match="URL too long"): - MaxURLBytes(message="URL too long")(form, form.url) - - -def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: - form = URLForm(MultiDict({"url": "\ud800"})) - with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): - MaxURLBytes()(form, form.url) diff --git a/tests/forms/test_received.py b/tests/forms/test_received.py new file mode 100644 index 0000000..4ef9ed3 --- /dev/null +++ b/tests/forms/test_received.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.received import ReceivedWebmentionFilterForm + + +def test_received_filter_form_accepts_valid_get_params(app: Flask) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict( + { + "status": "declined", + "q": "example.org", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + } + ), + meta={"csrf": False}, + ) + assert form.validate() + assert form.status.data == "declined" + assert form.q.data == "example.org" + assert form.date_from.data.isoformat() == "2026-11-08" + assert form.date_to.data.isoformat() == "2026-11-14" + assert "csrf_token" not in form._fields + + +@pytest.mark.parametrize( + "data", + [ + {"status": "unknown"}, + {"date_from": "not-a-date"}, + {"date_from": "2026-11-8"}, + {"date_to": "2026-02-30"}, + {"date_from": "2026-11-15", "date_to": "2026-11-08"}, + ], +) +def test_received_filter_form_rejects_invalid_get_params( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict(data), meta={"csrf": False} + ) + assert not form.validate() + assert form.errors + + +def test_received_filter_form_validates_sort_and_page_size(app: Flask) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict({"sort": "updated", "order": "asc", "per_page": "37"}), + meta={"csrf": False}, + ) + assert form.validate() + assert form.sort.data == "updated" + assert form.order.data == "asc" + assert form.per_page.data == 37 + + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm(formdata=MultiDict(), meta={"csrf": False}) + assert form.validate() + assert (form.sort.data, form.order.data, form.per_page.data) == ( + "received", + "desc", + 25, + ) + + +@pytest.mark.parametrize( + "data", + [ + {"sort": "invalid"}, + {"order": "invalid"}, + {"per_page": "0"}, + {"per_page": "101"}, + {"per_page": "invalid"}, + ], +) +def test_received_filter_form_rejects_invalid_sort_and_page_size( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict(data), meta={"csrf": False} + ) + assert not form.validate() + assert form.errors diff --git a/tests/forms/test_user.py b/tests/forms/test_user.py new file mode 100644 index 0000000..4b35a9a --- /dev/null +++ b/tests/forms/test_user.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.user import LoginForm + + +@pytest.mark.parametrize( + ("form_data", "invalid_field"), + [ + pytest.param({"password": "secret"}, "username", id="username-required"), + pytest.param({"username": "admin"}, "password", id="password-required"), + ], +) +def test_login_form_requires_credentials( + app: Flask, form_data: dict[str, str], invalid_field: str +) -> None: + with app.test_request_context("/login", method="POST"): + form = LoginForm(formdata=MultiDict(form_data), meta={"csrf": False}) + + assert not form.validate() + assert "This field is required." in form.errors[invalid_field] + + +def test_login_form_accepts_credentials(app: Flask) -> None: + with app.test_request_context("/login", method="POST"): + form = LoginForm( + formdata=MultiDict({"username": "admin", "password": "secret"}), + meta={"csrf": False}, + ) + assert form.validate() diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py new file mode 100644 index 0000000..4234552 --- /dev/null +++ b/tests/forms/test_validators.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict +from wtforms import Form, StringField, ValidationError + +from webmentions_ssg.forms.validators import ( + AllowedTargetHostname, + MaxURLBytes, + RejectNonPublicIPLiteral, +) + + +class URLForm(Form): + url = StringField("URL") + + +def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) + with app.app_context(): + AllowedTargetHostname()(form, form.url) + + +def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedTargetHostname()(form, form.url) + + +def test_allowed_hostname_uses_custom_message(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with ( + app.app_context(), + pytest.raises(ValidationError, match="Hostname is not allowed"), + ): + AllowedTargetHostname("Hostname is not allowed")(form, form.url) + + +@pytest.mark.parametrize( + "address", + ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], +) +def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: + form = URLForm(MultiDict({"url": f"https://[{address}]/"})) + with pytest.raises(ValidationError, match="URL must use a public address"): + RejectNonPublicIPLiteral()(form, form.url) + + +def test_public_ip_literal_accepts_public_ipv6_address() -> None: + form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) + RejectNonPublicIPLiteral()(form, form.url) + + +def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: + form = URLForm(MultiDict({"url": "http://[::1"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedTargetHostname()(form, form.url) + + +def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: + validator = MaxURLBytes() + form = URLForm(MultiDict({"url": "é" * 3})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 6 + validator(form, form.url) + app.config["MAX_WEBMENTION_URL_BYTES"] = 5 + with pytest.raises(ValidationError, match="5 bytes"): + validator(form, form.url) + + +def test_url_max_bytes_custom_message(app: Flask) -> None: + form = URLForm(MultiDict({"url": "too-long"})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 2 + with pytest.raises(ValidationError, match="URL too long"): + MaxURLBytes(message="URL too long")(form, form.url) + + +def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: + form = URLForm(MultiDict({"url": "\ud800"})) + with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): + MaxURLBytes()(form, form.url) diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..25d78d5 --- /dev/null +++ b/tests/models/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/models/test_received.py b/tests/models/test_received.py new file mode 100644 index 0000000..6f74d2d --- /dev/null +++ b/tests/models/test_received.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import db +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.models.utils import source_target_digest + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + (ReceivedWebmentionStatus.RECEIVED, False), + (ReceivedWebmentionStatus.VERIFYING, False), + (ReceivedWebmentionStatus.VERIFIED, True), + (ReceivedWebmentionStatus.FAILED, False), + (ReceivedWebmentionStatus.SOURCE_REMOVED, False), + ], +) +def test_received_webmention_verified( + status: ReceivedWebmentionStatus, expected: bool +) -> None: + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + status=status, + ) + + assert webmention.verified is expected + + +def test_received_webmention_repr() -> None: + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + ) + + assert repr(webmention) == ( + "<ReceivedWebmention 'https://source.example/post' -> " + "'https://dennisfink.me/blog/example/'>" + ) + + +def test_received_status_uses_non_native_enum() -> None: + status_type = ReceivedWebmention.__table__.c.status.type + + assert isinstance(status_type, sa.Enum) + assert status_type.enum_class is ReceivedWebmentionStatus + assert status_type.native_enum is False + assert status_type.enums == [status.value for status in ReceivedWebmentionStatus] + + +def test_received_status_round_trips_as_enum(app: Flask) -> None: + with app.app_context(): + identifier = uuid.uuid7() + mention = ReceivedWebmention( + uuid=identifier, + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + status=ReceivedWebmentionStatus.VERIFIED, + ) + db.session.add(mention) + db.session.commit() + + assert ( + db.session.scalar( + sa.text("SELECT status FROM received_webmentions WHERE uuid = :uuid"), + {"uuid": identifier.hex}, + ) + == "verified" + ) + + db.session.expunge_all() + loaded = db.session.get(ReceivedWebmention, identifier) + assert loaded is not None + assert loaded.status is ReceivedWebmentionStatus.VERIFIED + + +def test_received_verification_reset_is_reusable_for_atomic_updates() -> None: + mention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/post", + status=ReceivedWebmentionStatus.FAILED, + failure_reason="Old error", + recovery_attempts=3, + ) + assert mention.reopen_verification() is None + assert mention.status == ReceivedWebmentionStatus.RECEIVED + assert mention.failure_reason is None + assert mention.recovery_attempts == 0 + assert mention.updated_at is not None + + +def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: + long_source = "https://source.example/" + "a" * 4000 + long_target = "https://dennisfink.me/" + "b" * 4000 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + db.session.commit() + match = db.session.scalar( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.source_target_digest + == source_target_digest(long_source, long_target) + ) + ) + assert match is not None + assert match.source == long_source + assert match.target == long_target + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/models/test_sources.py b/tests/models/test_sources.py new file mode 100644 index 0000000..8d01d79 --- /dev/null +++ b/tests/models/test_sources.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime +from http import HTTPStatus +from typing import get_type_hints + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Mapped + +from webmentions_ssg import db +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) +from webmentions_ssg.models.utils import target_digest + + +def test_uuid_annotations_and_relationships_resolve() -> None: + """Verify that UUID columns and forward relationships map correctly.""" + for model in (ReceivedWebmention, SourceDocument, SentWebmention): + assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID] + assert sa.inspect(model).columns["uuid"].primary_key + + assert ( + sa.inspect(SourceDocument).relationships["sent_webmentions"].mapper.class_ + is SentWebmention + ) + assert ( + sa.inspect(SentWebmention).relationships["source"].mapper.class_ + is SourceDocument + ) + + +@pytest.mark.parametrize( + ("desired_revision", "processed_revision", "expected"), + [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)], +) +def test_sent_webmention_pending( + desired_revision: int, processed_revision: int | None, expected: bool +) -> None: + webmention = SentWebmention( + target="https://example.com/post", + desired_revision=desired_revision, + processed_revision=processed_revision, + ) + assert webmention.awaiting_processing is expected + + +def test_sent_webmention_pending_sql_matches_python(app: Flask) -> None: + cases = [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)] + + with app.app_context(): + now = datetime.now(UTC) + source = SourceDocument( + path="pending/index.html", + url="https://site.example/pending/", + content_hash="0" * 32, + last_seen_at=now, + revised_at=now, + ) + for i, (desired, processed, _) in enumerate(cases): + source.sent_webmentions.append( + SentWebmention( + target=f"https://example.com/post/{i}", + desired_revision=desired, + processed_revision=processed, + ) + ) + db.session.add(source) + db.session.commit() + + pending_ids = set( + db.session.scalars( + sa.select(SentWebmention.uuid).where(SentWebmention.awaiting_processing) + ) + ) + for webmention, (_, _, expected) in zip( + source.sent_webmentions, cases, strict=True + ): + assert webmention.awaiting_processing is expected + assert (webmention.uuid in pending_ids) is expected + + result = db.session.execute( + sa.update(SentWebmention) + .where(SentWebmention.awaiting_processing) + .values(attempt_count=1) + ) + assert result.rowcount == 3 + + +def test_source_repr() -> None: + source = SourceDocument( + path="example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=1, + last_seen_at=datetime.now(UTC), + revised_at=datetime.now(UTC), + ) + assert repr(source) == "<SourceDocument 'https://dennisfink.me/blog/example/'>" + + +def test_sent_webmention_repr() -> None: + webmention = SentWebmention(target="https://example.com/post", desired_revision=1) + assert repr(webmention) == "<SentWebmention 'https://example.com/post'>" + + +def test_reopen_preserves_delivery_history() -> None: + sent_at = datetime.now(UTC) + mention = SentWebmention( + target="https://example.com/post", + linked=True, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + attempt_count=3, + next_retry_at=sent_at, + last_sent_at=sent_at, + ) + assert mention.reset_for_revision(3, False) is None + assert mention.desired_revision == 3 + assert mention.linked is False + assert mention.attempt_count == 0 + assert mention.next_retry_at is None + assert mention.status == SentWebmentionStatus.SENT + assert mention.sent_revision == 2 + + +def test_resend_requeues_current_revision_without_losing_send_history() -> None: + sent_at = datetime.now(UTC) + mention = SentWebmention( + target="https://example.com/post", + linked=True, + desired_revision=3, + processed_revision=3, + sent_revision=2, + status=SentWebmentionStatus.FAILED, + failure_reason="Previous failure", + webmention_endpoint="https://example.com/webmention", + endpoint_response_status=HTTPStatus.SERVICE_UNAVAILABLE, + status_url="https://example.com/status", + attempt_count=3, + next_retry_at=sent_at, + last_attempted_at=sent_at, + last_sent_at=sent_at, + ) + assert not mention.awaiting_processing + + assert mention.clear_result_for_resend() is None + assert mention.desired_revision == 3 + assert mention.linked is True + assert mention.awaiting_processing + assert mention.processed_revision is None + assert mention.status is None + assert mention.failure_reason is None + assert mention.webmention_endpoint is None + assert mention.endpoint_response_status is None + assert mention.status_url is None + assert mention.attempt_count == 0 + assert mention.next_retry_at is None + assert mention.sent_revision == 2 + assert mention.last_attempted_at == sent_at + assert mention.last_sent_at == sent_at + + +def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( + app: Flask, +) -> None: + now = datetime.now(UTC) + long_target = "https://destination.example/" + "a" * 4000 + with app.app_context(): + source = SourceDocument( + path="long/index.html", + url="https://example.com/long", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + db.session.add(source) + db.session.commit() + assert source.sent_webmentions[0].target_digest == target_digest(long_target) + + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/models/test_user.py b/tests/models/test_user.py new file mode 100644 index 0000000..a3f95ae --- /dev/null +++ b/tests/models/test_user.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest + +from webmentions_ssg.models.user import User + + +def test_user_repr() -> None: + user = User(username="admin", password="secret") + assert repr(user) == "<User admin>" + + +def test_user_password_is_write_only() -> None: + user = User(username="admin", password="secret") + with pytest.raises(AttributeError, match="Password is write-only"): + _ = user.password + + +def test_user_password_is_hashed_and_can_be_checked() -> None: + user = User(username="admin", password="secret") + assert user.password_hash != "secret" + assert user.check_password("secret") + assert not user.check_password("wrong") + + +def test_user_session_id_changes_when_password_hash_changes() -> None: + user = User(username="admin", password="a strong passphrase", id=42) + original_hash = user.password_hash + original_id = user.get_id() + + assert original_id.startswith("42:") + assert original_hash not in original_id + assert len(original_id.split(":", 1)[1]) == 64 + + # Even a reset to the same password rehashes with a new salt, revoking + # any old session token. + user.password = "a strong passphrase" + assert user.password_hash != original_hash + assert user.get_id() != original_id diff --git a/tests/models/test_utils.py b/tests/models/test_utils.py new file mode 100644 index 0000000..8722a02 --- /dev/null +++ b/tests/models/test_utils.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from collections.abc import Callable +from datetime import UTC, datetime + +import pytest + +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import SentWebmention, SourceDocument +from webmentions_ssg.models.utils import ( + source_target_digest, + target_digest, + uuid7_to_datetime, +) + + +def test_uuid7_to_datetime() -> None: + identifier = uuid.uuid7() + timestamp = uuid7_to_datetime(identifier) + + assert timestamp.tzinfo is UTC + assert timestamp.timestamp() == pytest.approx(identifier.time / 1000) + + +@pytest.mark.parametrize( + "factory", + [ + pytest.param( + lambda identifier: ReceivedWebmention( + uuid=identifier, + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + ), + id="received-webmention", + ), + pytest.param( + lambda identifier: SourceDocument( + uuid=identifier, + path="example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=1, + last_seen_at=datetime.now(UTC), + revised_at=datetime.now(UTC), + ), + id="source", + ), + pytest.param( + lambda identifier: SentWebmention( + uuid=identifier, target="https://example.com/post", desired_revision=1 + ), + id="sent-webmention", + ), + ], +) +def test_created_at_comes_from_uuid7_timestamp( + factory: Callable[ + [uuid.UUID], ReceivedWebmention | SourceDocument | SentWebmention + ], +) -> None: + identifier = uuid.uuid7() + model = factory(identifier) + assert model.created_at == uuid7_to_datetime(identifier) + + +def test_received_pair_fingerprint_is_unambiguous() -> None: + assert source_target_digest("a", "bc") != source_target_digest("ab", "c") + assert source_target_digest("a", "b") != source_target_digest("b", "a") + + +def test_models_hash_url_fields_when_created_or_changed() -> None: + received = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/a", + target="https://site.example/b", + ) + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + received.source = "https://source.example/c" + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + received.target = "https://site.example/d" + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + + sent = SentWebmention(target="https://site.example/a", desired_revision=1) + assert sent.target_digest == target_digest(sent.target) + sent.target = "https://site.example/b" + assert sent.target_digest == target_digest(sent.target) diff --git a/tests/tasks/test_notifications.py b/tests/tasks/test_notifications.py index ca89a7a..dadec3e 100644 --- a/tests/tasks/test_notifications.py +++ b/tests/tasks/test_notifications.py @@ -13,7 +13,7 @@ from huey.exceptions import TaskLockedException from webmentions_ssg import db from webmentions_ssg.extensions import huey -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention @pytest.fixture diff --git a/tests/tasks/test_retention.py b/tests/tasks/test_retention.py index 70ab0e0..a3891a9 100644 --- a/tests/tasks/test_retention.py +++ b/tests/tasks/test_retention.py @@ -11,7 +11,7 @@ import sqlalchemy as sa from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention from webmentions_ssg.tasks.retention import purge_old_received_webmentions diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 4cf2a67..dbeeac9 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -17,7 +17,11 @@ from huey.exceptions import RetryTask from pytest import MonkeyPatch from webmentions_ssg import db -from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, SourceDocument +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) from webmentions_ssg.tasks import source_parser from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 9a0430c..0d68933 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -14,7 +14,11 @@ import pytest from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, SourceDocument +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" diff --git a/tests/tasks/test_verification.py b/tests/tasks/test_verification.py index c5550c8..3547464 100644 --- a/tests/tasks/test_verification.py +++ b/tests/tasks/test_verification.py @@ -15,7 +15,8 @@ import sqlalchemy as sa from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import BlockedSource, ReceivedWebmention +from webmentions_ssg.models.moderation import BlockedDomain +from webmentions_ssg.models.received import ReceivedWebmention from webmentions_ssg.net import AddressResolutionError SOURCE_URL = "https://source.example/article" @@ -363,7 +364,7 @@ def test_verify_webmention_refuses_domain_blocked_after_enqueue( ) -> None: identifier = create_webmention(app) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() fetch = Mock(return_value=True) diff --git a/tests/test_auth.py b/tests/test_auth.py index 8219a07..27e8fb0 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -16,7 +16,7 @@ from webmentions_ssg.auth import ( login_manager, validate_new_password, ) -from webmentions_ssg.models import User +from webmentions_ssg.models.user import User @pytest.mark.parametrize("character", ["\x00", "\x1f", "\x7f", "\x80", "\x9f"]) diff --git a/tests/test_cli.py b/tests/test_cli.py index f5535d2..5e71e2a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,7 +19,8 @@ from sqlalchemy.exc import IntegrityError from webmentions_ssg import create_app, db, migrate from webmentions_ssg.config import DevelopmentConfig, TestingConfig -from webmentions_ssg.models import ReceivedWebmention, ReceivedWebmentionStatus, User +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.models.user import User def test_cli_groups_are_registered(app: Flask) -> None: @@ -158,6 +159,8 @@ def test_create_db_builds_schema_and_records_latest_migration( assert "source_target_digest" in received_columns assert {"attempt_count", "next_retry_at"} <= columns assert "recovery_attempts" in received_columns + assert {"blocked_domains", "allowed_domains"} <= table_names + assert not {"blocked_sources", "allowed_sources"} & table_names assert "ix_received_webmentions_status_updated_at" in received_indexes @@ -250,11 +253,27 @@ def test_upgrade_merges_existing_migration_branches( """Existing installations on either branch can upgrade to a single head.""" with database_app.app_context(): migrate_upgrade(revision=existing_head) + if existing_head == "3e8c87a96b2d": + with db.engine.begin() as connection: + connection.execute( + sa.text("INSERT INTO blocked_sources (domain) VALUES (:domain)"), + {"domain": "existing.example"}, + ) migrate_upgrade() assert _database_revision(database_app) == _head_revision(database_app) tables = set(inspect(db.engine).get_table_names()) - assert {"blocked_sources", "allowed_sources", "source_documents"} <= tables + assert {"blocked_domains", "allowed_domains", "source_documents"} <= tables + assert "blocked_sources" not in tables + assert "allowed_sources" not in tables + if existing_head == "3e8c87a96b2d": + with db.engine.connect() as connection: + assert ( + connection.execute( + sa.text("SELECT domain FROM blocked_domains") + ).scalar_one() + == "existing.example" + ) def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None: @@ -304,7 +323,7 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows( :param database_app: Flask application connected to the legacy SQLite schema. """ - from webmentions_ssg.models import source_target_digest + from webmentions_ssg.models.utils import source_target_digest with database_app.app_context(): migrate_upgrade(revision="c0272f25ce17") diff --git a/tests/test_config.py b/tests/test_config.py index 4a5f4cb..81ff498 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,7 @@ from webmentions_ssg.config import ( ProductionConfig, TestingConfig, ) -from webmentions_ssg.forms import AllowedTargetHostname +from webmentions_ssg.forms.validators import AllowedTargetHostname @pytest.mark.parametrize( @@ -42,9 +42,9 @@ def test_session_cookie_defaults( def test_source_allowlist_mode_requires_boolean_configuration(tmp_path: Path) -> None: class BadSourceModeConfig(TestingConfig): LOG_FILE = str(tmp_path / "mode-test.log") - SOURCE_ALLOWLIST_ENABLED = "yes" + ALLOWLIST_ENABLED = "yes" - with pytest.raises(ValueError, match="SOURCE_ALLOWLIST_ENABLED"): + with pytest.raises(ValueError, match="ALLOWLIST_ENABLED"): create_app(BadSourceModeConfig) diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 4e785c7..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,439 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -import uuid -from collections.abc import Callable -from datetime import UTC, datetime -from http import HTTPStatus -from typing import get_type_hints - -import pytest -import sqlalchemy as sa -from flask import Flask -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Mapped - -from webmentions_ssg import db -from webmentions_ssg.models import ( - ReceivedWebmention, - ReceivedWebmentionStatus, - SentWebmention, - SourceDocument, - User, - source_target_digest, - target_digest, - uuid7_to_datetime, -) - - -def test_uuid_annotations_and_relationships_resolve() -> None: - """Verify that UUID columns and forward relationships map correctly.""" - for model in (ReceivedWebmention, SourceDocument, SentWebmention): - assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID] - assert sa.inspect(model).columns["uuid"].primary_key - - assert ( - sa.inspect(SourceDocument).relationships["sent_webmentions"].mapper.class_ - is SentWebmention - ) - assert ( - sa.inspect(SentWebmention).relationships["source"].mapper.class_ - is SourceDocument - ) - - -def test_user_repr() -> None: - user = User(username="admin", password="secret") - assert repr(user) == "<User admin>" - - -def test_user_password_is_write_only() -> None: - user = User(username="admin", password="secret") - with pytest.raises(AttributeError, match="Password is write-only"): - _ = user.password - - -def test_user_password_is_hashed_and_can_be_checked() -> None: - user = User(username="admin", password="secret") - assert user.password_hash != "secret" - assert user.check_password("secret") - assert not user.check_password("wrong") - - -@pytest.mark.parametrize( - ("status", "expected"), - [ - (ReceivedWebmentionStatus.RECEIVED, False), - (ReceivedWebmentionStatus.VERIFYING, False), - (ReceivedWebmentionStatus.VERIFIED, True), - (ReceivedWebmentionStatus.FAILED, False), - (ReceivedWebmentionStatus.SOURCE_REMOVED, False), - ], -) -def test_received_webmention_verified( - status: ReceivedWebmentionStatus, expected: bool -) -> None: - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - status=status, - ) - - assert webmention.verified is expected - - -@pytest.mark.parametrize( - ("desired_revision", "processed_revision", "expected"), - [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)], -) -def test_sent_webmention_pending( - desired_revision: int, processed_revision: int | None, expected: bool -) -> None: - webmention = SentWebmention( - target="https://example.com/post", - desired_revision=desired_revision, - processed_revision=processed_revision, - ) - assert webmention.awaiting_processing is expected - - -def test_sent_webmention_pending_sql_matches_python(app: Flask) -> None: - cases = [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)] - - with app.app_context(): - now = datetime.now(UTC) - source = SourceDocument( - path="pending/index.html", - url="https://site.example/pending/", - content_hash="0" * 32, - last_seen_at=now, - revised_at=now, - ) - for i, (desired, processed, _) in enumerate(cases): - source.sent_webmentions.append( - SentWebmention( - target=f"https://example.com/post/{i}", - desired_revision=desired, - processed_revision=processed, - ) - ) - db.session.add(source) - db.session.commit() - - pending_ids = set( - db.session.scalars( - sa.select(SentWebmention.uuid).where(SentWebmention.awaiting_processing) - ) - ) - for webmention, (_, _, expected) in zip( - source.sent_webmentions, cases, strict=True - ): - assert webmention.awaiting_processing is expected - assert (webmention.uuid in pending_ids) is expected - - result = db.session.execute( - sa.update(SentWebmention) - .where(SentWebmention.awaiting_processing) - .values(attempt_count=1) - ) - assert result.rowcount == 3 - - -def test_uuid7_to_datetime() -> None: - identifier = uuid.uuid7() - timestamp = uuid7_to_datetime(identifier) - - assert timestamp.tzinfo is UTC - assert timestamp.timestamp() == pytest.approx(identifier.time / 1000) - - -@pytest.mark.parametrize( - "factory", - [ - pytest.param( - lambda identifier: ReceivedWebmention( - uuid=identifier, - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - ), - id="received-webmention", - ), - pytest.param( - lambda identifier: SourceDocument( - uuid=identifier, - path="example/index.html", - url="https://dennisfink.me/blog/example/", - content_hash="0" * 32, - revision=1, - last_seen_at=datetime.now(UTC), - revised_at=datetime.now(UTC), - ), - id="source", - ), - pytest.param( - lambda identifier: SentWebmention( - uuid=identifier, target="https://example.com/post", desired_revision=1 - ), - id="sent-webmention", - ), - ], -) -def test_created_at_comes_from_uuid7_timestamp( - factory: Callable[ - [uuid.UUID], ReceivedWebmention | SourceDocument | SentWebmention - ], -) -> None: - identifier = uuid.uuid7() - model = factory(identifier) - assert model.created_at == uuid7_to_datetime(identifier) - - -def test_received_webmention_repr() -> None: - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - ) - - assert repr(webmention) == ( - "<ReceivedWebmention 'https://source.example/post' -> " - "'https://dennisfink.me/blog/example/'>" - ) - - -def test_source_repr() -> None: - source = SourceDocument( - path="example/index.html", - url="https://dennisfink.me/blog/example/", - content_hash="0" * 32, - revision=1, - last_seen_at=datetime.now(UTC), - revised_at=datetime.now(UTC), - ) - assert repr(source) == "<SourceDocument 'https://dennisfink.me/blog/example/'>" - - -def test_sent_webmention_repr() -> None: - webmention = SentWebmention(target="https://example.com/post", desired_revision=1) - assert repr(webmention) == "<SentWebmention 'https://example.com/post'>" - - -def test_user_session_id_changes_when_password_hash_changes() -> None: - user = User(username="admin", password="a strong passphrase", id=42) - original_hash = user.password_hash - original_id = user.get_id() - - assert original_id.startswith("42:") - assert original_hash not in original_id - assert len(original_id.split(":", 1)[1]) == 64 - - # Even a reset to the same password rehashes with a new salt, revoking - # any old session token. - user.password = "a strong passphrase" - assert user.password_hash != original_hash - assert user.get_id() != original_id - - -def test_received_status_uses_non_native_enum() -> None: - status_type = ReceivedWebmention.__table__.c.status.type - - assert isinstance(status_type, sa.Enum) - assert status_type.enum_class is ReceivedWebmentionStatus - assert status_type.native_enum is False - assert status_type.enums == [status.value for status in ReceivedWebmentionStatus] - - -def test_received_status_round_trips_as_enum(app: Flask) -> None: - with app.app_context(): - identifier = uuid.uuid7() - mention = ReceivedWebmention( - uuid=identifier, - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - status=ReceivedWebmentionStatus.VERIFIED, - ) - db.session.add(mention) - db.session.commit() - - assert ( - db.session.scalar( - sa.text("SELECT status FROM received_webmentions WHERE uuid = :uuid"), - {"uuid": identifier.hex}, - ) - == "verified" - ) - - db.session.expunge_all() - loaded = db.session.get(ReceivedWebmention, identifier) - assert loaded is not None - assert loaded.status is ReceivedWebmentionStatus.VERIFIED - - -def test_received_verification_reset_is_reusable_for_atomic_updates() -> None: - from webmentions_ssg.models import ReceivedWebmentionStatus - - mention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/post", - status=ReceivedWebmentionStatus.FAILED, - failure_reason="Old error", - recovery_attempts=3, - ) - assert mention.reopen_verification() is None - assert mention.status == ReceivedWebmentionStatus.RECEIVED - assert mention.failure_reason is None - assert mention.recovery_attempts == 0 - assert mention.updated_at is not None - - -def test_reopen_preserves_delivery_history() -> None: - from webmentions_ssg.models import SentWebmentionStatus - - sent_at = datetime.now(UTC) - mention = SentWebmention( - target="https://example.com/post", - linked=True, - desired_revision=2, - processed_revision=2, - sent_revision=2, - status=SentWebmentionStatus.SENT, - attempt_count=3, - next_retry_at=sent_at, - last_sent_at=sent_at, - ) - assert mention.reset_for_revision(3, False) is None - assert mention.desired_revision == 3 - assert mention.linked is False - assert mention.attempt_count == 0 - assert mention.next_retry_at is None - assert mention.status == SentWebmentionStatus.SENT - assert mention.sent_revision == 2 - - -def test_resend_requeues_current_revision_without_losing_send_history() -> None: - from webmentions_ssg.models import SentWebmentionStatus - - sent_at = datetime.now(UTC) - mention = SentWebmention( - target="https://example.com/post", - linked=True, - desired_revision=3, - processed_revision=3, - sent_revision=2, - status=SentWebmentionStatus.FAILED, - failure_reason="Previous failure", - webmention_endpoint="https://example.com/webmention", - endpoint_response_status=HTTPStatus.SERVICE_UNAVAILABLE, - status_url="https://example.com/status", - attempt_count=3, - next_retry_at=sent_at, - last_attempted_at=sent_at, - last_sent_at=sent_at, - ) - assert not mention.awaiting_processing - - assert mention.clear_result_for_resend() is None - assert mention.desired_revision == 3 - assert mention.linked is True - assert mention.awaiting_processing - assert mention.processed_revision is None - assert mention.status is None - assert mention.failure_reason is None - assert mention.webmention_endpoint is None - assert mention.endpoint_response_status is None - assert mention.status_url is None - assert mention.attempt_count == 0 - assert mention.next_retry_at is None - assert mention.sent_revision == 2 - assert mention.last_attempted_at == sent_at - assert mention.last_sent_at == sent_at - - -def test_received_pair_fingerprint_is_unambiguous() -> None: - assert source_target_digest("a", "bc") != source_target_digest("ab", "c") - assert source_target_digest("a", "b") != source_target_digest("b", "a") - - -def test_models_hash_url_fields_when_created_or_changed() -> None: - received = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/a", - target="https://site.example/b", - ) - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - received.source = "https://source.example/c" - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - received.target = "https://site.example/d" - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - - sent = SentWebmention(target="https://site.example/a", desired_revision=1) - assert sent.target_digest == target_digest(sent.target) - sent.target = "https://site.example/b" - assert sent.target_digest == target_digest(sent.target) - - -def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: - long_source = "https://source.example/" + "a" * 4000 - long_target = "https://dennisfink.me/" + "b" * 4000 - with app.app_context(): - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - db.session.commit() - match = db.session.scalar( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.source_target_digest - == source_target_digest(long_source, long_target) - ) - ) - assert match is not None - assert match.source == long_source - assert match.target == long_target - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() - - -def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( - app: Flask, -) -> None: - now = datetime.now(UTC) - long_target = "https://destination.example/" + "a" * 4000 - with app.app_context(): - source = SourceDocument( - path="long/index.html", - url="https://example.com/long", - content_hash="0" * 32, - revision=1, - last_seen_at=now, - revised_at=now, - ) - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - db.session.add(source) - db.session.commit() - assert source.sent_webmentions[0].target_digest == target_digest(long_target) - - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() diff --git a/tests/test_moderation_policy.py b/tests/test_moderation_policy.py new file mode 100644 index 0000000..48a34ac --- /dev/null +++ b/tests/test_moderation_policy.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain normalization and receiver admission policy.""" + +from flask import Flask + +from webmentions_ssg import db +from webmentions_ssg.models.moderation import AllowedDomain, BlockedDomain +from webmentions_ssg.moderation_policy import domain_from_url, rejection_reason + + +def test_domain_from_url() -> None: + assert ( + domain_from_url("https://BÜCHER.example.:8443/post") == "xn--bcher-kva.example" + ) + assert domain_from_url("https://[2001:db8::1]/post") == "2001:db8::1" + assert domain_from_url("invalid") is None + + +def test_blocked_domain_matches_itself_and_subdomains_but_not_suffixes( + app: Flask, +) -> None: + with app.app_context(): + db.session.add(BlockedDomain(domain="example.org")) + db.session.commit() + for source in ( + "https://example.org/post", + "https://blog.example.org/post", + "https://A.blog.Example.ORG./post", + ): + assert rejection_reason(source) == "Source domain is blocked" + for source in ( + "https://notexample.org/post", + "https://example.org.evil/post", + "https://other.net/post", + ): + assert rejection_reason(source) is None + + +def test_allowlist_is_fail_closed_and_blocklist_takes_precedence(app: Flask) -> None: + app.config["ALLOWLIST_ENABLED"] = True + with app.app_context(): + assert ( + rejection_reason("https://example.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(AllowedDomain(domain="example.org")) + db.session.commit() + assert rejection_reason("https://blog.example.org/post") is None + assert ( + rejection_reason("https://notexample.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(BlockedDomain(domain="blog.example.org")) + db.session.commit() + assert ( + rejection_reason("https://blog.example.org/post") + == "Source domain is blocked" + ) + assert rejection_reason("https://example.org/post") is None diff --git a/tests/test_source_policy.py b/tests/test_source_policy.py deleted file mode 100644 index cddfa3a..0000000 --- a/tests/test_source_policy.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Domain normalization and receiver admission policy.""" - -import pytest -from flask import Flask - -from webmentions_ssg import db -from webmentions_ssg.models import AllowedSource, BlockedSource -from webmentions_ssg.source_policy import ( - source_domain_from_url, - source_rejection_reason, -) - - -def test_source_domain_from_url() -> None: - assert ( - source_domain_from_url("https://BÜCHER.example.:8443/post") - == "xn--bcher-kva.example" - ) - assert source_domain_from_url("https://[2001:db8::1]/post") == "2001:db8::1" - assert source_domain_from_url("invalid") is None - - -def test_blocked_domain_matches_itself_and_subdomains_but_not_suffixes( - app: Flask, -) -> None: - with app.app_context(): - db.session.add(BlockedSource(domain="example.org")) - db.session.commit() - for source in ( - "https://example.org/post", - "https://blog.example.org/post", - "https://A.blog.Example.ORG./post", - ): - assert source_rejection_reason(source) == "Source domain is blocked" - for source in ( - "https://notexample.org/post", - "https://example.org.evil/post", - "https://other.net/post", - ): - assert source_rejection_reason(source) is None - - -def test_allowlist_is_fail_closed_and_blocklist_takes_precedence(app: Flask) -> None: - app.config["SOURCE_ALLOWLIST_ENABLED"] = True - with app.app_context(): - assert ( - source_rejection_reason("https://example.org/post") - == "Source domain is not allowlisted" - ) - db.session.add(AllowedSource(domain="example.org")) - db.session.commit() - assert source_rejection_reason("https://blog.example.org/post") is None - assert ( - source_rejection_reason("https://notexample.org/post") - == "Source domain is not allowlisted" - ) - db.session.add(BlockedSource(domain="blog.example.org")) - db.session.commit() - assert ( - source_rejection_reason("https://blog.example.org/post") - == "Source domain is blocked" - ) - assert source_rejection_reason("https://example.org/post") is None diff --git a/tests/views/helpers.py b/tests/views/helpers.py index 354102b..2bea73e 100644 --- a/tests/views/helpers.py +++ b/tests/views/helpers.py @@ -13,13 +13,13 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ( - ReceivedWebmention, +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import ( SentWebmention, SentWebmentionStatus, SourceDocument, - User, ) +from webmentions_ssg.models.user import User SOURCE_URL = "https://source.example/post" TARGET_URL = "https://dennisfink.me/blog/example/" @@ -41,7 +41,7 @@ def mock_endpoint_form( form.errors = errors or {} monkeypatch.setattr( - views_module.forms, "WebmentionSubmissionForm", Mock(return_value=form) + views_module, "WebmentionSubmissionForm", Mock(return_value=form) ) return form @@ -118,7 +118,7 @@ def mock_login_form( form.username.data = username form.password.data = password - monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form)) + monkeypatch.setattr(views_module, "LoginForm", Mock(return_value=form)) return form @@ -128,9 +128,7 @@ def mock_admin_action_form( form = Mock() form.validate_on_submit.return_value = valid - monkeypatch.setattr( - views_module.forms, "ConfirmActionForm", Mock(return_value=form) - ) + monkeypatch.setattr(views_module, "ConfirmActionForm", Mock(return_value=form)) return form diff --git a/tests/views/test_endpoint.py b/tests/views/test_endpoint.py index 88b665f..23643d9 100644 --- a/tests/views/test_endpoint.py +++ b/tests/views/test_endpoint.py @@ -15,7 +15,7 @@ from flask.testing import FlaskClient from sqlalchemy.exc import IntegrityError from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention from .helpers import ( SOURCE_URL, diff --git a/tests/views/test_source_domains.py b/tests/views/test_moderation.py index c9e72b2..7f683d4 100644 --- a/tests/views/test_source_domains.py +++ b/tests/views/test_moderation.py @@ -15,12 +15,9 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ( - AllowedSource, - BlockedSource, - ReceivedWebmention, - ReceivedWebmentionStatus, -) +from webmentions_ssg.models.moderation import AllowedDomain, BlockedDomain +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.moderation_policy import rejection_reason from webmentions_ssg.tasks.verification import VerificationClaim from .helpers import SOURCE_URL, TARGET_URL, create_received_webmention, log_in @@ -28,9 +25,9 @@ from .helpers import SOURCE_URL, TARGET_URL, create_received_webmention, log_in def test_source_rules_require_login(client: FlaskClient) -> None: for method, path, data in ( - ("get", "/source-domains", None), - ("post", "/source-domains/blocked/add", {"domain": "bad.example"}), - ("post", "/source-domains/blocked/1/delete", None), + ("get", "/moderation", None), + ("post", "/moderation/blocked/add", {"domain": "bad.example"}), + ("post", "/moderation/blocked/1/delete", None), ): response = client.open(path, method=method.upper(), data=data) assert response.status_code == HTTPStatus.FOUND @@ -39,31 +36,29 @@ def test_source_rules_require_login(client: FlaskClient) -> None: def test_add_and_unblock_source_domain(app: Flask, client: FlaskClient) -> None: log_in(app, client) - response = client.post( - "/source-domains/blocked/add", data={"domain": "SPAM.Example."} - ) + response = client.post("/moderation/blocked/add", data={"domain": "SPAM.Example."}) assert response.status_code == HTTPStatus.FOUND - response = client.post( - "/source-domains/blocked/add", data={"domain": "spam.example"} - ) + assert response.headers["Location"].endswith("/moderation") + response = client.post("/moderation/blocked/add", data={"domain": "spam.example"}) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - rows = db.session.scalars(sa.select(BlockedSource)).all() + rows = db.session.scalars(sa.select(BlockedDomain)).all() assert len(rows) == 1 assert rows[0].domain == "spam.example" entry_id = rows[0].id - response = client.get("/source-domains") + response = client.get("/moderation") assert response.status_code == HTTPStatus.OK assert b"spam.example" in response.data assert b"Unblock" in response.data assert b"Source rules" in response.data - response = client.post(f"/source-domains/blocked/{entry_id}/delete") + response = client.post(f"/moderation/blocked/{entry_id}/delete") assert response.status_code == HTTPStatus.FOUND + assert response.headers["Location"].endswith("/moderation") with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_allowlist_can_be_managed_while_mode_is_off( @@ -71,14 +66,14 @@ def test_allowlist_can_be_managed_while_mode_is_off( ) -> None: log_in(app, client) response = client.post( - "/source-domains/allowed/add", data={"domain": "friends.example"} + "/moderation/allowed/add", data={"domain": "friends.example"} ) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - row = db.session.scalar(sa.select(AllowedSource)) + row = db.session.scalar(sa.select(AllowedDomain)) assert row is not None assert row.domain == "friends.example" - assert b"disabled" in client.get("/source-domains").data + assert b"disabled" in client.get("/moderation").data @pytest.mark.parametrize( @@ -88,17 +83,17 @@ def test_invalid_domains_are_rejected( client: FlaskClient, app: Flask, domain: str ) -> None: log_in(app, client) - response = client.post("/source-domains/blocked/add", data={"domain": domain}) + response = client.post("/moderation/blocked/add", data={"domain": domain}) assert response.status_code == HTTPStatus.BAD_REQUEST def test_invalid_list_kind_and_missing_entry(app: Flask, client: FlaskClient) -> None: log_in(app, client) assert ( - client.post("/source-domains/nope/add", data={"domain": "test.org"}).status_code + client.post("/moderation/nope/add", data={"domain": "test.org"}).status_code == 404 ) - assert client.post("/source-domains/blocked/123/delete").status_code == 404 + assert client.post("/moderation/blocked/123/delete").status_code == 404 def test_received_row_can_block_domain(app: Flask, client: FlaskClient) -> None: @@ -109,7 +104,7 @@ def test_received_row_can_block_domain(app: Flask, client: FlaskClient) -> None: response = client.post(f"/received/{mention_id}/block-source") assert response.status_code == HTTPStatus.FOUND with app.app_context(): - rule = db.session.scalar(sa.select(BlockedSource)) + rule = db.session.scalar(sa.select(BlockedDomain)) assert rule is not None and rule.domain == "spam.example" @@ -126,7 +121,7 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( second = create_received_webmention(app, source="https://second.example/article") unselected = create_received_webmention(app, source="https://other.example/article") with app.app_context(): - db.session.add(BlockedSource(domain="second.example")) + db.session.add(BlockedDomain(domain="second.example")) db.session.commit() response = client.post( @@ -142,19 +137,17 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( "sort": ["source"], } with app.app_context(): - assert set(db.session.scalars(sa.select(BlockedSource.domain))) == { + assert set(db.session.scalars(sa.select(BlockedDomain.domain))) == { "spam.example", "second.example", } for identifier in (first, same_domain, second, unselected): assert db.session.get(ReceivedWebmention, identifier) is not None - from webmentions_ssg.source_policy import source_rejection_reason - assert ( - source_rejection_reason("https://child.spam.example/post") + rejection_reason("https://child.spam.example/post") == "Source domain is blocked" ) - assert source_rejection_reason("https://other.example/post") is None + assert rejection_reason("https://other.example/post") is None # Repeating the action is idempotent. response = client.post( @@ -163,7 +156,7 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( ) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 2 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 2 def test_bulk_block_rejects_invalid_source_without_partial_changes( @@ -179,7 +172,7 @@ def test_bulk_block_rejects_invalid_source_without_partial_changes( ) assert response.status_code == HTTPStatus.BAD_REQUEST with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_bulk_block_action_enforces_csrf(app: Flask, client: FlaskClient) -> None: @@ -192,7 +185,7 @@ def test_bulk_block_action_enforces_csrf(app: Flask, client: FlaskClient) -> Non ) assert response.status_code == HTTPStatus.BAD_REQUEST with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_endpoint_rejects_blocked_without_queue_or_insert( @@ -203,7 +196,7 @@ def test_endpoint_rejects_blocked_without_queue_or_insert( enqueue = Mock() monkeypatch.setattr(endpoint, "verify_webmention", enqueue) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() response = client.post( @@ -221,14 +214,14 @@ def test_allowlist_mode_rejects_unlisted_and_allows_listed( ) -> None: from webmentions_ssg.views import endpoint - app.config["SOURCE_ALLOWLIST_ENABLED"] = True + app.config["ALLOWLIST_ENABLED"] = True enqueue = Mock() monkeypatch.setattr(endpoint, "verify_webmention", enqueue) data = {"source": SOURCE_URL, "target": TARGET_URL} assert client.post("/endpoint", data=data).status_code == HTTPStatus.FORBIDDEN with app.app_context(): - db.session.add(AllowedSource(domain="source.example")) + db.session.add(AllowedDomain(domain="source.example")) db.session.commit() assert client.post("/endpoint", data=data).status_code == HTTPStatus.CREATED @@ -249,7 +242,7 @@ def test_worker_checks_rules_before_fetch( fetch = Mock() monkeypatch.setattr(verification, "source_mentions_target", fetch) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() assert verification.determine_verification_outcome(claim) == ( ReceivedWebmentionStatus.FAILED, @@ -272,7 +265,7 @@ def test_queued_worker_skips_fetch_after_source_is_blocked( identifier = mention.uuid # A user blocks the domain after the submission has already been queued. - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() fetch = Mock() @@ -291,7 +284,7 @@ def test_admin_actions_enforce_csrf(app: Flask, client: FlaskClient) -> None: app.config["WTF_CSRF_ENABLED"] = True assert ( client.post( - "/source-domains/blocked/add", data={"domain": "spam.example"} + "/moderation/blocked/add", data={"domain": "spam.example"} ).status_code == HTTPStatus.BAD_REQUEST ) diff --git a/tests/views/test_admin.py b/tests/views/test_received.py index 175bcfd..ec00e05 100644 --- a/tests/views/test_admin.py +++ b/tests/views/test_received.py @@ -17,15 +17,12 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention, SentWebmention, User +from webmentions_ssg.models.received import ReceivedWebmention from .helpers import ( SOURCE_URL, TARGET_URL, - create_outgoing, create_received_webmention, - create_resendable_sent_webmention, - create_sent_source, log_in, mock_admin_action_form, ) @@ -34,66 +31,11 @@ from .helpers import ( @pytest.fixture def views_module(app: Flask) -> ModuleType: _ = app - from webmentions_ssg.views import admin as module + from webmentions_ssg.views import received as module return module -@pytest.mark.parametrize("path", ["/sources", f"/sources/{uuid.uuid7()}"]) -def test_sent_views_require_login(client: FlaskClient, path: str) -> None: - response = client.get(path) - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get("/sources") - - assert response.status_code == HTTPStatus.OK - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - assert f"/sources/{identifier}" in html - - -def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get(f"/sources/{identifier}") - - assert response.status_code == HTTPStatus.OK - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - - first_target = "https://example.com/a" - second_target = "https://example.com/b" - - assert first_target in html - assert second_target in html - assert html.index(first_target) < html.index(second_target) - - assert "Webmention endpoint returned HTTP 400" in html - assert "https://example.com/webmention" in html - - -def test_sent_source_returns_404_for_unknown_source( - app: Flask, client: FlaskClient -) -> None: - log_in(app, client) - response = client.get(f"/sources/{uuid.uuid7()}") - assert response.status_code == HTTPStatus.NOT_FOUND - - def test_received_requires_login(client: FlaskClient) -> None: response = client.get("/received") @@ -471,7 +413,13 @@ def test_received_filters_are_combined(app: Flask, client: FlaskClient) -> None: ) response = client.get( - "/received?status=declined&q=match.example&date_from=2026-11-08&date_to=2026-11-14" + "/received", + query_string={ + "status": "declined", + "q": "match.example", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + }, ) html = response.get_data(as_text=True) @@ -1113,168 +1061,3 @@ def test_received_bulk_delete_requires_real_csrf_token( assert valid.status_code == HTTPStatus.FOUND with app.app_context(): assert db.session.get(ReceivedWebmention, identifier) is None - - -def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: - response = client.post("/sources/rescan") - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -@pytest.mark.parametrize("valid", [False, True]) -def test_rescan_sent_sources( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - valid: bool, -) -> None: - log_in(app, client) - form = mock_admin_action_form(views_module, monkeypatch, valid=valid) - manual_scan_sources = Mock() - monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources) - - response = client.post("/sources/rescan") - - assert response.status_code == ( - HTTPStatus.FOUND if valid else HTTPStatus.BAD_REQUEST - ) - form.validate_on_submit.assert_called_once_with() - - if valid: - assert response.headers["Location"].endswith("/sources") - manual_scan_sources.assert_called_once_with() - else: - manual_scan_sources.assert_not_called() - - -def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: - response = client.post(f"/sources/{uuid.uuid7()}/{uuid.uuid7()}/resend") - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -def test_resend_sent_webmention_rejects_invalid_form( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=False) - - response = client.post( - f"/sources/{source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.BAD_REQUEST - - -def test_resend_sent_webmention_returns_404_for_unknown_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post(f"/sources/{source_identifier}/{uuid.uuid7()}/resend") - - assert response.status_code == HTTPStatus.NOT_FOUND - - -def test_resend_sent_webmention_rejects_wrong_source( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - _, webmention_identifier = create_resendable_sent_webmention(app) - wrong_source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post( - f"/sources/{wrong_source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.NOT_FOUND - - -def test_resend_sent_webmention_reopens_current_revision( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - send_webmention = Mock() - monkeypatch.setattr(views_module, "send_webmention", send_webmention) - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - webmention.attempt_count = 6 - webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) - db.session.commit() - - response = client.post( - f"/sources/{source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.FOUND - assert response.headers["Location"].endswith(f"/sources/{source_identifier}") - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - assert webmention.desired_revision == 3 - assert webmention.processed_revision is None - assert webmention.attempt_count == 0 - assert webmention.next_retry_at is None - assert webmention.sent_revision == 2 - assert webmention.status is None - assert webmention.failure_reason is None - assert webmention.webmention_endpoint is None - assert webmention.endpoint_response_status is None - assert webmention.status_url is None - - send_webmention.assert_called_once_with(webmention_identifier) - - -@pytest.mark.parametrize( - ("status_url", "safe"), - [ - ("https://example.com/status", True), - ("http://example.com/status", True), - ("javascript:alert(1)", False), - ("data:text/html,evil", False), - ("ftp://example.com/status", False), - ], -) -def test_admin_only_links_safe_existing_status_urls( - app: Flask, client: FlaskClient, status_url: str, safe: bool -) -> None: - source_identifier, _ = create_outgoing(app, status_url=status_url) - with app.app_context(): - user = User(username="admin", password="test") - db.session.add(user) - db.session.commit() - user_id = user.get_id() - - with client.session_transaction() as session: - session["_user_id"] = str(user_id) - session["_fresh"] = True - - response = client.get(f"/sources/{source_identifier}") - assert response.status_code == HTTPStatus.OK - html = response.get_data(as_text=True) - assert (f'href="{status_url}"' in html) is safe - assert status_url in html diff --git a/tests/views/test_session.py b/tests/views/test_session.py index a2d3268..7edee71 100644 --- a/tests/views/test_session.py +++ b/tests/views/test_session.py @@ -10,7 +10,7 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import User +from webmentions_ssg.models.user import User from .helpers import log_in, mock_login_form diff --git a/tests/views/test_sources.py b/tests/views/test_sources.py new file mode 100644 index 0000000..012347e --- /dev/null +++ b/tests/views/test_sources.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime, timedelta +from http import HTTPStatus +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import db +from webmentions_ssg.models.sources import SentWebmention +from webmentions_ssg.models.user import User + +from .helpers import ( + create_outgoing, + create_resendable_sent_webmention, + create_sent_source, + log_in, + mock_admin_action_form, +) + + +@pytest.fixture +def sources_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.views import sources as module + + return module + + +@pytest.mark.parametrize("path", ["/sources", f"/sources/{uuid.uuid7()}"]) +def test_sent_views_require_login(client: FlaskClient, path: str) -> None: + response = client.get(path) + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get("/sources") + + assert response.status_code == HTTPStatus.OK + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + assert f"/sources/{identifier}" in html + + +def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get(f"/sources/{identifier}") + + assert response.status_code == HTTPStatus.OK + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + + first_target = "https://example.com/a" + second_target = "https://example.com/b" + + assert first_target in html + assert second_target in html + assert html.index(first_target) < html.index(second_target) + + assert "Webmention endpoint returned HTTP 400" in html + assert "https://example.com/webmention" in html + + +def test_sent_source_returns_404_for_unknown_source( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.get(f"/sources/{uuid.uuid7()}") + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: + response = client.post("/sources/rescan") + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +@pytest.mark.parametrize("valid", [False, True]) +def test_rescan_sent_sources( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + valid: bool, +) -> None: + log_in(app, client) + form = mock_admin_action_form(sources_module, monkeypatch, valid=valid) + manual_scan_sources = Mock() + monkeypatch.setattr(sources_module, "manual_scan_sources", manual_scan_sources) + + response = client.post("/sources/rescan") + + assert response.status_code == ( + HTTPStatus.FOUND if valid else HTTPStatus.BAD_REQUEST + ) + form.validate_on_submit.assert_called_once_with() + + if valid: + assert response.headers["Location"].endswith("/sources") + manual_scan_sources.assert_called_once_with() + else: + manual_scan_sources.assert_not_called() + + +def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: + response = client.post(f"/sources/{uuid.uuid7()}/{uuid.uuid7()}/resend") + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +def test_resend_sent_webmention_rejects_invalid_form( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=False) + + response = client.post( + f"/sources/{source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_resend_sent_webmention_returns_404_for_unknown_webmention( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier = create_sent_source(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + + response = client.post(f"/sources/{source_identifier}/{uuid.uuid7()}/resend") + + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_resend_sent_webmention_rejects_wrong_source( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + _, webmention_identifier = create_resendable_sent_webmention(app) + wrong_source_identifier = create_sent_source(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + + response = client.post( + f"/sources/{wrong_source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_resend_sent_webmention_reopens_current_revision( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + send_webmention = Mock() + monkeypatch.setattr(sources_module, "send_webmention", send_webmention) + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + webmention.attempt_count = 6 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) + db.session.commit() + + response = client.post( + f"/sources/{source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.FOUND + assert response.headers["Location"].endswith(f"/sources/{source_identifier}") + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + assert webmention.desired_revision == 3 + assert webmention.processed_revision is None + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None + assert webmention.sent_revision == 2 + assert webmention.status is None + assert webmention.failure_reason is None + assert webmention.webmention_endpoint is None + assert webmention.endpoint_response_status is None + assert webmention.status_url is None + + send_webmention.assert_called_once_with(webmention_identifier) + + +@pytest.mark.parametrize( + ("status_url", "safe"), + [ + ("https://example.com/status", True), + ("http://example.com/status", True), + ("javascript:alert(1)", False), + ("data:text/html,evil", False), + ("ftp://example.com/status", False), + ], +) +def test_admin_only_links_safe_existing_status_urls( + app: Flask, client: FlaskClient, status_url: str, safe: bool +) -> None: + source_identifier, _ = create_outgoing(app, status_url=status_url) + with app.app_context(): + user = User(username="admin", password="test") + db.session.add(user) + db.session.commit() + user_id = user.get_id() + + with client.session_transaction() as session: + session["_user_id"] = str(user_id) + session["_fresh"] = True + + response = client.get(f"/sources/{source_identifier}") + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert (f'href="{status_url}"' in html) is safe + assert status_url in html diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index ce4960d..ac1f4e7 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -16,7 +16,7 @@ from flask_login import LoginManager from werkzeug.security import generate_password_hash from .extensions import db -from .models import User +from .models.user import User from .validation import is_control_character login_manager = LoginManager() diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index b26b35b..707402d 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -15,7 +15,7 @@ from sqlalchemy.exc import IntegrityError from .auth import validate_new_password from .extensions import db, mail -from .models import User +from .models.user import User USERS_CLI = AppGroup("users", help="Manage users.") MAIL_CLI = AppGroup("mail", help="Handle mail") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 576f376..28ab43d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -54,9 +54,9 @@ class DefaultConfig: # Receiver # Empty by default: no target hostnames are accepted until configured. ALLOWED_TARGET_HOSTNAMES = frozenset() - # When enabled, only source domains listed in AllowedSource are accepted. - # Entries in BlockedSource always take precedence. - SOURCE_ALLOWLIST_ENABLED = False + # When enabled, only source domains listed in AllowedDomain are accepted. + # Entries in BlockedDomain always take precedence. + ALLOWLIST_ENABLED = False # Cap incoming HTTP request bodies (also applies to other Flask routes). MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. @@ -213,8 +213,8 @@ def validated_config_overrides( except (TypeError, ValueError) as exc: raise type(exc)(f"{key}: {exc}") from exc - if type(config["SOURCE_ALLOWLIST_ENABLED"]) is not bool: - raise ValueError("SOURCE_ALLOWLIST_ENABLED must be a boolean.") + if type(config["ALLOWLIST_ENABLED"]) is not bool: + raise ValueError("ALLOWLIST_ENABLED must be a boolean.") proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] if type(proxy_hops) is not int or proxy_hops < 0: diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py deleted file mode 100644 index 3b61a8a..0000000 --- a/webmentions_ssg/forms.py +++ /dev/null @@ -1,268 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Webmention forms and URL validation.""" - -import ipaddress - -import httpx -from flask import current_app -from flask_wtf import FlaskForm -from wtforms import ( - DateField, - Field, - HiddenField, - IntegerField, - PasswordField, - SearchField, - SelectField, - StringField, - SubmitField, - ValidationError, -) -from wtforms.form import BaseForm -from wtforms.validators import ( - AnyOf, - InputRequired, - NumberRange, - Optional, - StopValidation, -) - -from .hostname import normalize_hostname -from .net import is_public_address -from .validation import is_control_character - -RECEIVED_STATUS_CHOICES = { - "": "All statuses", - "pending": "Pending", - "verifying": "Verifying", - "verified": "Verified", - "declined": "Declined", -} - - -class MaxURLBytes: - """Check the UTF-8 byte length of a submitted URL using the current config.""" - - def __init__( - self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None - ) -> None: - self.config_key = config_key - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - if field.data is None: - return - - max_bytes = current_app.config[self.config_key] - - try: - length = len(field.data.encode("utf-8")) - except UnicodeEncodeError as exc: - raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc - - if length > max_bytes: - raise ValidationError( - self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") - ) - - -class StrictHTTPURL: - """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. - - httpx automatically percent-encodes some whitespace. Reject it instead of - silently changing the submitted URL (which may be used in exact URL matches). - """ - - def __call__(self, form: BaseForm, field: Field) -> None: - value = field.data - if not isinstance(value, str): - raise StopValidation(field.gettext("Invalid URL.")) - - if any( - character.isspace() or is_control_character(character) - for character in value - ): - raise StopValidation(field.gettext("Invalid URL.")) - - try: - parsed = httpx.URL(value) - except (httpx.InvalidURL, ValueError, UnicodeError) as exc: - raise StopValidation(field.gettext("Invalid URL.")) from exc - - if not parsed.host: - raise StopValidation(field.gettext("Invalid URL.")) - - if parsed.scheme not in ("http", "https"): - raise StopValidation( - field.gettext(f"{field.name} must begin with http or https") - ) - - -class AllowedTargetHostname: - """Validate that a URL uses an allowed hostname. - - :param message: Optional validation error message. - """ - - def __init__(self, message: str | None = None) -> None: - """Initialize the validator. - - :param message: Optional validation error message. - """ - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - """Validate that a URL uses an allowed hostname. - - :param form: Form containing the field. - :param field: Field containing the URL to validate. - :raises ValidationError: If the URL hostname is not configured as allowed. - """ - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError: - hostname = None - - if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: - return - raise ValidationError(self.message or field.gettext("Invalid input.")) - - -class RejectNonPublicIPLiteral: - """Reject non-public IP literals without performing DNS in the web worker.""" - - def __call__(self, form: BaseForm, field: Field) -> None: - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError as exc: - raise ValidationError(field.gettext("Invalid input.")) from exc - - try: - address = ipaddress.ip_address(hostname) - except ValueError: - return # Named hosts are resolved only during the background fetch. - - if not is_public_address(address): - raise ValidationError(field.gettext("URL must use a public address.")) - - -class LoginForm(FlaskForm): - """Form for authenticating a user.""" - - username = StringField("Username", validators=[InputRequired()]) - password = PasswordField("Password", validators=[InputRequired()]) - submit = SubmitField("Sign In") - - -class WebmentionSubmissionForm(FlaskForm): - """Form for receiving a Webmention. - - The source must be a public HTTP or HTTPS URL distinct from the target. - The target must be an HTTP or HTTPS URL using an allowed hostname. - """ - - source = StringField( - "source", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - RejectNonPublicIPLiteral(), - ], - ) - target = StringField( - "target", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - AllowedTargetHostname(), - ], - ) - - def validate_source(self, field: Field) -> None: - """Reject a source URL that equals the target URL. - - :param field: Source URL field being validated. - :raises ValidationError: If the source URL equals the target URL. - """ - if field.data == self.target.data: - raise ValidationError(field.gettext("Field must not be equal to target.")) - - -class CanonicalDate: - """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" - - def __call__(self, form: BaseForm, field: DateField) -> None: - if ( - field.data is not None - and field.raw_data - and field.raw_data[0] != field.data.isoformat() - ): - raise ValidationError(field.gettext("Invalid date.")) - - -def default_received_page_size() -> int: - """Use 25 rows unless the configured maximum is lower.""" - return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) - - -class ReceivedWebmentionFilterForm(FlaskForm): - """GET filters for the received Webmention list.""" - - status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") - q = SearchField("Source or target") - date_from = DateField( - "Updated from (UTC)", validators=[Optional(), CanonicalDate()] - ) - date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) - sort = HiddenField( - default="received", - validators=[AnyOf(("received", "updated", "status", "source", "target"))], - ) - order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) - per_page = IntegerField( - "Rows per page", - default=default_received_page_size, - validators=[NumberRange(min=1)], - ) - - def validate_per_page(self, field: IntegerField) -> None: - """Enforce the deployment's maximum received-list page size.""" - if ( - field.data is not None - and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] - ): - raise ValidationError("Rows per page exceeds the configured maximum.") - - def validate_date_to(self, field: DateField) -> None: - """Require the end date to be on or after the start date.""" - if ( - self.date_from.data is not None - and field.data is not None - and field.data < self.date_from.data - ): - raise ValidationError("End date must be on or after start date.") - - -class ConfirmActionForm(FlaskForm): - """Form for confirming an administrative action.""" - - submit = SubmitField("Confirm") - - -class SourceDomainForm(FlaskForm): - """Add a domain to the source blocklist or allowlist.""" - - domain = StringField("Source domain", validators=[InputRequired()]) - submit = SubmitField("Add domain") - - def validate_domain(self, field: Field) -> None: - """Require a bare hostname and persist its canonical form.""" - try: - field.data = normalize_hostname(field.data) - except ValueError as exc: - raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py new file mode 100644 index 0000000..5f61c0e --- /dev/null +++ b/webmentions_ssg/forms/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for webmention submission, authentication, filtering and moderation.""" diff --git a/webmentions_ssg/forms/admin.py b/webmentions_ssg/forms/admin.py new file mode 100644 index 0000000..97d523d --- /dev/null +++ b/webmentions_ssg/forms/admin.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Administrative action confirmation forms.""" + +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class ConfirmActionForm(FlaskForm): + """Form for confirming an administrative action.""" + + submit = SubmitField("Confirm") diff --git a/webmentions_ssg/forms/endpoint.py b/webmentions_ssg/forms/endpoint.py new file mode 100644 index 0000000..e063e73 --- /dev/null +++ b/webmentions_ssg/forms/endpoint.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Webmention submission form.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, ValidationError +from wtforms.validators import InputRequired + +from .validators import ( + AllowedTargetHostname, + MaxURLBytes, + RejectNonPublicIPLiteral, + StrictHTTPURL, +) + + +class WebmentionSubmissionForm(FlaskForm): + """Form for receiving a Webmention. + + The source must be a public HTTP or HTTPS URL distinct from the target. + The target must be an HTTP or HTTPS URL using an allowed hostname. + """ + + source = StringField( + "source", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + RejectNonPublicIPLiteral(), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + AllowedTargetHostname(), + ], + ) + + def validate_source(self, field: Field) -> None: + """Reject a source URL that equals the target URL. + + :param field: Source URL field being validated. + :raises ValidationError: If the source URL equals the target URL. + """ + if field.data == self.target.data: + raise ValidationError(field.gettext("Field must not be equal to target.")) diff --git a/webmentions_ssg/forms/moderation.py b/webmentions_ssg/forms/moderation.py new file mode 100644 index 0000000..5987d14 --- /dev/null +++ b/webmentions_ssg/forms/moderation.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for managing domain allowlists and blocklists.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, SubmitField, ValidationError +from wtforms.validators import InputRequired + +from ..hostname import normalize_hostname + + +class ModerateDomainForm(FlaskForm): + """Add a source domain to the blocklist or allowlist.""" + + domain = StringField("Source domain", validators=[InputRequired()]) + submit = SubmitField("Add domain") + + def validate_domain(self, field: Field) -> None: + """Require a bare hostname and persist its canonical form.""" + try: + field.data = normalize_hostname(field.data) + except ValueError as exc: + raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/received.py b/webmentions_ssg/forms/received.py new file mode 100644 index 0000000..c4a6ed0 --- /dev/null +++ b/webmentions_ssg/forms/received.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Received Webmention filtering form.""" + +from flask import current_app +from flask_wtf import FlaskForm +from wtforms import ( + DateField, + HiddenField, + IntegerField, + SearchField, + SelectField, + ValidationError, +) +from wtforms.validators import AnyOf, NumberRange, Optional + +from .validators import CanonicalDate + +RECEIVED_STATUS_CHOICES = { + "": "All statuses", + "pending": "Pending", + "verifying": "Verifying", + "verified": "Verified", + "declined": "Declined", +} + + +def default_received_page_size() -> int: + """Use 25 rows unless the configured maximum is lower.""" + return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) + + +class ReceivedWebmentionFilterForm(FlaskForm): + """GET filters for the received Webmention list.""" + + status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") + q = SearchField("Source or target") + date_from = DateField( + "Updated from (UTC)", validators=[Optional(), CanonicalDate()] + ) + date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) + sort = HiddenField( + default="received", + validators=[AnyOf(("received", "updated", "status", "source", "target"))], + ) + order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) + per_page = IntegerField( + "Rows per page", + default=default_received_page_size, + validators=[NumberRange(min=1)], + ) + + def validate_per_page(self, field: IntegerField) -> None: + """Enforce the deployment's maximum received-list page size.""" + if ( + field.data is not None + and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] + ): + raise ValidationError("Rows per page exceeds the configured maximum.") + + def validate_date_to(self, field: DateField) -> None: + """Require the end date to be on or after the start date.""" + if ( + self.date_from.data is not None + and field.data is not None + and field.data < self.date_from.data + ): + raise ValidationError("End date must be on or after start date.") diff --git a/webmentions_ssg/forms/user.py b/webmentions_ssg/forms/user.py new file mode 100644 index 0000000..3bb6421 --- /dev/null +++ b/webmentions_ssg/forms/user.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""User authentication forms.""" + +from flask_wtf import FlaskForm +from wtforms import PasswordField, StringField, SubmitField +from wtforms.validators import InputRequired + + +class LoginForm(FlaskForm): + """Form for authenticating a user.""" + + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py new file mode 100644 index 0000000..e65e9cb --- /dev/null +++ b/webmentions_ssg/forms/validators.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Custom Webmention URL and date field validators.""" + +import ipaddress + +import httpx +from flask import current_app +from wtforms import DateField, Field, ValidationError +from wtforms.form import BaseForm +from wtforms.validators import StopValidation + +from ..hostname import normalize_hostname +from ..net import is_public_address +from ..validation import is_control_character + + +class MaxURLBytes: + """Check the UTF-8 byte length of a submitted URL using the current config.""" + + def __init__( + self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None + ) -> None: + self.config_key = config_key + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + if field.data is None: + return + + max_bytes = current_app.config[self.config_key] + + try: + length = len(field.data.encode("utf-8")) + except UnicodeEncodeError as exc: + raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc + + if length > max_bytes: + raise ValidationError( + self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") + ) + + +class StrictHTTPURL: + """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or is_control_character(character) + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + if parsed.scheme not in ("http", "https"): + raise StopValidation( + field.gettext(f"{field.name} must begin with http or https") + ) + + +class AllowedTargetHostname: + """Validate that a URL uses an allowed hostname. + + :param message: Optional validation error message. + """ + + def __init__(self, message: str | None = None) -> None: + """Initialize the validator. + + :param message: Optional validation error message. + """ + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + """Validate that a URL uses an allowed hostname. + + :param form: Form containing the field. + :param field: Field containing the URL to validate. + :raises ValidationError: If the URL hostname is not configured as allowed. + """ + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError: + hostname = None + + if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: + return + raise ValidationError(self.message or field.gettext("Invalid input.")) + + +class RejectNonPublicIPLiteral: + """Reject non-public IP literals without performing DNS in the web worker.""" + + def __call__(self, form: BaseForm, field: Field) -> None: + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError as exc: + raise ValidationError(field.gettext("Invalid input.")) from exc + + try: + address = ipaddress.ip_address(hostname) + except ValueError: + return # Named hosts are resolved only during the background fetch. + + if not is_public_address(address): + raise ValidationError(field.gettext("URL must use a public address.")) + + +class CanonicalDate: + """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" + + def __call__(self, form: BaseForm, field: DateField) -> None: + if ( + field.data is not None + and field.raw_data + and field.raw_data[0] != field.data.isoformat() + ): + raise ValidationError(field.gettext("Invalid date.")) diff --git a/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py new file mode 100644 index 0000000..e66dc70 --- /dev/null +++ b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rename source block and allow lists to domain lists. + +Revision ID: a4f91c2e08d7 +Revises: b8179a2e46c3 +""" + +from alembic import op + +revision = "a4f91c2e08d7" +down_revision = "b8179a2e46c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Preserve existing domain rules when renaming their tables.""" + op.rename_table("blocked_sources", "blocked_domains") + op.rename_table("allowed_sources", "allowed_domains") + + +def downgrade() -> None: + """Restore the former table names without deleting domain rules.""" + op.rename_table("allowed_domains", "allowed_sources") + op.rename_table("blocked_domains", "blocked_sources") diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 8518b15..2917cb8 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,8 +11,8 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.models import source_target_digest as received_url_digest -from webmentions_ssg.models import target_digest as sent_target_digest +from webmentions_ssg.models.utils import source_target_digest as received_url_digest +from webmentions_ssg.models.utils import target_digest as sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py deleted file mode 100644 index 2304788..0000000 --- a/webmentions_ssg/models.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Define database models for users and incoming and outgoing Webmentions.""" - -import json -from datetime import UTC, datetime -from enum import StrEnum -from hashlib import sha256 -from typing import Never -from uuid import UUID, uuid7 - -import sqlalchemy as sa -from flask_login import UserMixin -from sqlalchemy import ( - Boolean, - DateTime, - Enum, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - Uuid, -) -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Mapped, mapped_column, relationship, validates -from werkzeug.security import check_password_hash, generate_password_hash - -from .extensions import Base - - -class ReceivedWebmentionStatus(StrEnum): - """Persisted lifecycle state of an incoming Webmention.""" - - RECEIVED = "received" - VERIFYING = "verifying" - VERIFIED = "verified" - FAILED = "failed" - SOURCE_REMOVED = "source_removed" - - -class SentWebmentionStatus(StrEnum): - """Represent the outcome of a Webmention send attempt.""" - - SENT = "sent" - NO_ENDPOINT = "no_endpoint" - FAILED = "failed" - - -class User(UserMixin, Base): - """Represent an authenticated application user.""" - - __tablename__ = "users" - - id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column(Text(), index=True, unique=True) - password_hash: Mapped[str] = mapped_column(Text(), nullable=False) - - def __repr__(self) -> str: - """Represent the user by username.""" - return f"<User {self.username}>" - - @property - def password(self) -> Never: - """Prevent access to the user's plain-text password. - - :raises AttributeError: Always, because the password is write-only. - """ - raise AttributeError("Password is write-only") - - @password.setter - def password(self, password: str) -> None: - """Hash and store a new password. - - :param password: Plain-text password to hash. - """ - self.password_hash = generate_password_hash(password) - - def get_id(self) -> str: - """Bind the login session to the current stored password hash. - - Only a one-way fingerprint is placed in Flask's signed cookie, never - the password hash itself. Rehashing the password invalidates all - previously issued session identifiers without a schema change. - - :returns: Session identifier combining the user ID and password-hash - fingerprint. - """ - fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() - return f"{self.id}:{fingerprint}" - - def check_password(self, password: str) -> bool: - """Check a plain-text password against the stored password hash. - - :param password: Plain-text password to check. - :returns: Whether the password matches the stored hash. - """ - return check_password_hash(self.password_hash, password) - - -class BlockedSource(Base): - """A source domain denied before verification or outbound fetching.""" - - __tablename__ = "blocked_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class AllowedSource(Base): - """A source domain accepted when allowlist mode is enabled.""" - - __tablename__ = "allowed_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class ReceivedWebmention(Base): - """Represent a Webmention received for verification.""" - - __tablename__ = "received_webmentions" - - uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) - - source: Mapped[str] = mapped_column(Text(), nullable=False) - target: Mapped[str] = mapped_column(Text(), nullable=False) - source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("source", "target") - def _refresh_url_digest(self, key: str, value: str) -> str: - source = value if key == "source" else self.__dict__.get("source") - target = value if key == "target" else self.__dict__.get("target") - if source is not None and target is not None: - self.source_target_digest = source_target_digest(source, target) - return value - - status: Mapped[ReceivedWebmentionStatus] = mapped_column( - Enum( - ReceivedWebmentionStatus, - name="received_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=False, - default=ReceivedWebmentionStatus.RECEIVED, - ) - # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh - # explicit submission resets this counter. - recovery_attempts: Mapped[int] = mapped_column( - Integer, nullable=False, default=0, server_default="0" - ) - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - default=lambda: datetime.now(UTC), - onupdate=lambda: datetime.now(UTC), - nullable=False, - ) - - notification_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - __table_args__ = ( - UniqueConstraint( - "source_target_digest", name="uq_webmention_source_target_digest" - ), - Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), - ) - - def __repr__(self) -> str: - """Represent the received Webmention by source and target.""" - return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" - - @staticmethod - def reopen_verification_values( - *, at: datetime, recovery_attempts: int = 0 - ) -> dict[str, object]: - """Return the columns to reset when verification is explicitly reopened. - - :param at: Timestamp to record for the newly opened verification. - :param recovery_attempts: Number of stale-lease recovery attempts to retain. - :returns: Column values for resetting the verification state. - """ - return { - "status": ReceivedWebmentionStatus.RECEIVED, - "failure_reason": None, - "recovery_attempts": recovery_attempts, - "updated_at": at, - } - - def reopen_verification(self) -> None: - """Reopen an existing row for manual verification. - - Clear the previous outcome and restore the received state with a fresh - timestamp. - """ - for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): - setattr(self, key, value) - - @property - def verified(self) -> bool: - """Return whether the Webmention has been successfully verified.""" - return self.status == ReceivedWebmentionStatus.VERIFIED - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SourceDocument(Base): - """Represent a generated source document containing outgoing Webmentions.""" - - __tablename__ = "source_documents" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - content_hash: Mapped[str] = mapped_column(String(32), nullable=False) - revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) - - last_seen_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - revised_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - deleted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - sent_webmentions: Mapped[list[SentWebmention]] = relationship( - back_populates="source", cascade="all, delete-orphan" - ) - - def __repr__(self) -> str: - """Represent the source by its public URL.""" - return f"<SourceDocument {self.url!r}>" - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the source UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SentWebmention(Base): - """Represent an outgoing Webmention associated with a source.""" - - __tablename__ = "sent_webmentions" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - source_uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), - ForeignKey("source_documents.uuid", ondelete="CASCADE"), - nullable=False, - index=True, - ) - - target: Mapped[str] = mapped_column(Text(), nullable=False) - target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("target") - def _update_target_digest(self, key: str, value: str) -> str: - self.target_digest = target_digest(value) - return value - - linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) - - desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) - processed_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - sent_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - - status: Mapped[SentWebmentionStatus | None] = mapped_column( - Enum( - SentWebmentionStatus, - name="sent_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=True, - default=None, - ) - - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - webmention_endpoint: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - endpoint_response_status: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) - - # Retry state belongs to the desired revision, not to the target forever. - attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) - next_retry_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - last_attempted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - last_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") - - __table_args__ = ( - UniqueConstraint( - "source_uuid", - "target_digest", - name="uq_sent_webmention_source_uuid_target_digest", - ), - ) - - def __repr__(self) -> str: - """Represent the sent Webmention by its target URL.""" - return f"<SentWebmention {self.target!r}>" - - def reset_for_revision(self, revision: int, linked: bool) -> None: - """Reopen delivery for a revision while preserving send history. - - :param revision: Source revision to make the desired delivery revision. - :param linked: Whether the target still appears in the current source. - """ - self.desired_revision = revision - self.linked = linked - self.attempt_count = 0 - self.next_retry_at = None - - def clear_result_for_resend(self) -> None: - """Retry the current revision, clearing its previous delivery result. - - Reset attempt and response state without changing the desired source revision. - """ - self.reset_for_revision(self.desired_revision, self.linked) - self.processed_revision = None - self.status = None - self.failure_reason = None - self.webmention_endpoint = None - self.endpoint_response_status = None - self.status_url = None - - @hybrid_property - def awaiting_processing(self) -> bool: - """Return whether the Webmention has an unprocessed source revision.""" - return ( - self.processed_revision is None - or self.processed_revision < self.desired_revision - ) - - @awaiting_processing.inplace.expression - @classmethod - def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: - """Express pending-revision status in database queries. - - :returns: SQLAlchemy expression selecting unprocessed source revisions. - """ - return sa.or_( - cls.processed_revision.is_(None), - cls.processed_revision < cls.desired_revision, - ) - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -def uuid7_to_datetime(identifier: UUID) -> datetime: - """Convert a UUID version 7 timestamp to a datetime. - - :param identifier: UUID whose embedded timestamp should be converted. - :returns: Timestamp as a timezone-aware UTC datetime. - """ - return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) - - -def source_target_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair. - - :param source: Webmention source URL. - :param target: Webmention target URL. - :returns: SHA-256 hexadecimal digest of the ordered URL pair. - """ - encoded = json.dumps( - [source, target], ensure_ascii=False, separators=(",", ":") - ).encode("utf-8") - return sha256(encoded).hexdigest() - - -def target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID). - - :param target: Outgoing Webmention target URL. - :returns: SHA-256 hexadecimal digest of the target URL. - """ - return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/models/__init__.py b/webmentions_ssg/models/__init__.py new file mode 100644 index 0000000..360fc71 --- /dev/null +++ b/webmentions_ssg/models/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Database models grouped by feature.""" diff --git a/webmentions_ssg/models/moderation.py b/webmentions_ssg/models/moderation.py new file mode 100644 index 0000000..4200c39 --- /dev/null +++ b/webmentions_ssg/models/moderation.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain allowlist and blocklist models.""" + +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column + +from ..extensions import Base + + +class BlockedDomain(Base): + """Source domain rejected before submission or verification.""" + + __tablename__ = "blocked_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + +class AllowedDomain(Base): + """A domain accepted when allowlist mode is enabled.""" + + __tablename__ = "allowed_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) diff --git a/webmentions_ssg/models/received.py b/webmentions_ssg/models/received.py new file mode 100644 index 0000000..a9047ad --- /dev/null +++ b/webmentions_ssg/models/received.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Incoming Webmention state and persistence.""" + +from datetime import UTC, datetime +from enum import StrEnum +from uuid import UUID + +from sqlalchemy import ( + DateTime, + Enum, + Index, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.orm import Mapped, mapped_column, validates + +from ..extensions import Base +from .utils import source_target_digest, uuid7_to_datetime + + +class ReceivedWebmentionStatus(StrEnum): + """Persisted lifecycle state of an incoming Webmention.""" + + RECEIVED = "received" + VERIFYING = "verifying" + VERIFIED = "verified" + FAILED = "failed" + SOURCE_REMOVED = "source_removed" + + +class ReceivedWebmention(Base): + """Represent a Webmention received for verification.""" + + __tablename__ = "received_webmentions" + + uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) + + source: Mapped[str] = mapped_column(Text(), nullable=False) + target: Mapped[str] = mapped_column(Text(), nullable=False) + source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("source", "target") + def _refresh_url_digest(self, key: str, value: str) -> str: + source = value if key == "source" else self.__dict__.get("source") + target = value if key == "target" else self.__dict__.get("target") + if source is not None and target is not None: + self.source_target_digest = source_target_digest(source, target) + return value + + status: Mapped[ReceivedWebmentionStatus] = mapped_column( + Enum( + ReceivedWebmentionStatus, + name="received_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=False, + default=ReceivedWebmentionStatus.RECEIVED, + ) + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + nullable=False, + ) + + notification_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + __table_args__ = ( + UniqueConstraint( + "source_target_digest", name="uq_webmention_source_target_digest" + ), + Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), + ) + + def __repr__(self) -> str: + """Represent the received Webmention by source and target.""" + return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" + + @staticmethod + def reopen_verification_values( + *, at: datetime, recovery_attempts: int = 0 + ) -> dict[str, object]: + """Return the columns to reset when verification is explicitly reopened. + + :param at: Timestamp to record for the newly opened verification. + :param recovery_attempts: Number of stale-lease recovery attempts to retain. + :returns: Column values for resetting the verification state. + """ + return { + "status": ReceivedWebmentionStatus.RECEIVED, + "failure_reason": None, + "recovery_attempts": recovery_attempts, + "updated_at": at, + } + + def reopen_verification(self) -> None: + """Reopen an existing row for manual verification. + + Clear the previous outcome and restore the received state with a fresh + timestamp. + """ + for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): + setattr(self, key, value) + + @property + def verified(self) -> bool: + """Return whether the Webmention has been successfully verified.""" + return self.status == ReceivedWebmentionStatus.VERIFIED + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/sources.py b/webmentions_ssg/models/sources.py new file mode 100644 index 0000000..f0fd618 --- /dev/null +++ b/webmentions_ssg/models/sources.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source documents and outgoing Webmention state.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID, uuid7 + +import sqlalchemy as sa +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates + +from ..extensions import Base +from .utils import target_digest, uuid7_to_datetime + + +class SentWebmentionStatus(StrEnum): + """Represent the outcome of a Webmention send attempt.""" + + SENT = "sent" + NO_ENDPOINT = "no_endpoint" + FAILED = "failed" + + +class SourceDocument(Base): + """Represent a generated source document containing outgoing Webmentions.""" + + __tablename__ = "source_documents" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + content_hash: Mapped[str] = mapped_column(String(32), nullable=False) + revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) + + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + revised_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + sent_webmentions: Mapped[list[SentWebmention]] = relationship( + back_populates="source", cascade="all, delete-orphan" + ) + + def __repr__(self) -> str: + """Represent the source by its public URL.""" + return f"<SourceDocument {self.url!r}>" + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the source UUID.""" + return uuid7_to_datetime(self.uuid) + + +class SentWebmention(Base): + """Represent an outgoing Webmention associated with a source.""" + + __tablename__ = "sent_webmentions" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + source_uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("source_documents.uuid", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + target: Mapped[str] = mapped_column(Text(), nullable=False) + target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("target") + def _update_target_digest(self, key: str, value: str) -> str: + self.target_digest = target_digest(value) + return value + + linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + + desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) + processed_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + sent_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + + status: Mapped[SentWebmentionStatus | None] = mapped_column( + Enum( + SentWebmentionStatus, + name="sent_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=True, + default=None, + ) + + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + webmention_endpoint: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + endpoint_response_status: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + last_attempted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") + + __table_args__ = ( + UniqueConstraint( + "source_uuid", + "target_digest", + name="uq_sent_webmention_source_uuid_target_digest", + ), + ) + + def __repr__(self) -> str: + """Represent the sent Webmention by its target URL.""" + return f"<SentWebmention {self.target!r}>" + + def reset_for_revision(self, revision: int, linked: bool) -> None: + """Reopen delivery for a revision while preserving send history. + + :param revision: Source revision to make the desired delivery revision. + :param linked: Whether the target still appears in the current source. + """ + self.desired_revision = revision + self.linked = linked + self.attempt_count = 0 + self.next_retry_at = None + + def clear_result_for_resend(self) -> None: + """Retry the current revision, clearing its previous delivery result. + + Reset attempt and response state without changing the desired source revision. + """ + self.reset_for_revision(self.desired_revision, self.linked) + self.processed_revision = None + self.status = None + self.failure_reason = None + self.webmention_endpoint = None + self.endpoint_response_status = None + self.status_url = None + + @hybrid_property + def awaiting_processing(self) -> bool: + """Return whether the Webmention has an unprocessed source revision.""" + return ( + self.processed_revision is None + or self.processed_revision < self.desired_revision + ) + + @awaiting_processing.inplace.expression + @classmethod + def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: + """Express pending-revision status in database queries. + + :returns: SQLAlchemy expression selecting unprocessed source revisions. + """ + return sa.or_( + cls.processed_revision.is_(None), + cls.processed_revision < cls.desired_revision, + ) + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/user.py b/webmentions_ssg/models/user.py new file mode 100644 index 0000000..b09c3d1 --- /dev/null +++ b/webmentions_ssg/models/user.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Application user and session identity model.""" + +from hashlib import sha256 +from typing import Never + +from flask_login import UserMixin +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column +from werkzeug.security import check_password_hash, generate_password_hash + +from ..extensions import Base + + +class User(UserMixin, Base): + """Represent an authenticated application user.""" + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(Text(), index=True, unique=True) + password_hash: Mapped[str] = mapped_column(Text(), nullable=False) + + def __repr__(self) -> str: + """Represent the user by username.""" + return f"<User {self.username}>" + + @property + def password(self) -> Never: + """Prevent access to the user's plain-text password. + + :raises AttributeError: Always, because the password is write-only. + """ + raise AttributeError("Password is write-only") + + @password.setter + def password(self, password: str) -> None: + """Hash and store a new password. + + :param password: Plain-text password to hash. + """ + self.password_hash = generate_password_hash(password) + + def get_id(self) -> str: + """Bind the login session to the current stored password hash. + + Only a one-way fingerprint is placed in Flask's signed cookie, never + the password hash itself. Rehashing the password invalidates all + previously issued session identifiers without a schema change. + + :returns: Session identifier combining the user ID and password-hash + fingerprint. + """ + fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() + return f"{self.id}:{fingerprint}" + + def check_password(self, password: str) -> bool: + """Check a plain-text password against the stored password hash. + + :param password: Plain-text password to check. + :returns: Whether the password matches the stored hash. + """ + return check_password_hash(self.password_hash, password) diff --git a/webmentions_ssg/models/utils.py b/webmentions_ssg/models/utils.py new file mode 100644 index 0000000..e961088 --- /dev/null +++ b/webmentions_ssg/models/utils.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""URL fingerprints and UUID timestamp utilities.""" + +import json +from datetime import UTC, datetime +from hashlib import sha256 +from uuid import UUID + + +def uuid7_to_datetime(identifier: UUID) -> datetime: + """Convert a UUID version 7 timestamp to a datetime. + + :param identifier: UUID whose embedded timestamp should be converted. + :returns: Timestamp as a timezone-aware UTC datetime. + """ + return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) + + +def source_target_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair. + + :param source: Webmention source URL. + :param target: Webmention target URL. + :returns: SHA-256 hexadecimal digest of the ordered URL pair. + """ + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID). + + :param target: Outgoing Webmention target URL. + :returns: SHA-256 hexadecimal digest of the target URL. + """ + return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/source_policy.py b/webmentions_ssg/moderation_policy.py index 2c62527..51e7995 100644 --- a/webmentions_ssg/source_policy.py +++ b/webmentions_ssg/moderation_policy.py @@ -11,11 +11,11 @@ from flask import current_app from .extensions import db from .hostname import normalize_hostname -from .models import AllowedSource, BlockedSource +from .models.moderation import AllowedDomain, BlockedDomain -def source_domain_from_url(source: str) -> str | None: - """Extract and canonicalize the source hostname without a network request.""" +def domain_from_url(source: str) -> str | None: + """Extract and canonicalize the hostname without a network request.""" try: return normalize_hostname(source, from_url=True) except ValueError: @@ -23,7 +23,7 @@ def source_domain_from_url(source: str) -> str | None: def has_matching_domain( - model: type[AllowedSource | BlockedSource], domain: str + model: type[AllowedDomain | BlockedDomain], domain: str ) -> bool: """Match the hostname itself and parent domains, never partial labels.""" candidates = [domain] @@ -42,20 +42,20 @@ def has_matching_domain( ) -def source_rejection_reason(source: str) -> str | None: - """Return the reason a source cannot be received, or None when accepted. +def rejection_reason(source: str) -> str | None: + """Return the reason a source cannot be received, or ``None`` if accepted. The blocklist always wins. In allowlist mode an empty allowlist rejects every source. The database, not a stale per-worker cache, is consulted. """ - if (domain := source_domain_from_url(source)) is None: + if (domain := domain_from_url(source)) is None: return "Invalid source hostname" - if has_matching_domain(BlockedSource, domain): + if has_matching_domain(BlockedDomain, domain): return "Source domain is blocked" - if current_app.config["SOURCE_ALLOWLIST_ENABLED"] and not has_matching_domain( - AllowedSource, domain + if current_app.config["ALLOWLIST_ENABLED"] and not has_matching_domain( + AllowedDomain, domain ): return "Source domain is not allowlisted" diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index c503e1f..6a0e6b9 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,7 +11,7 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def email_verified_digest() -> None: diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index b882493..b4b6a83 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 6869477..1f98702 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,7 +14,7 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, SourceDocument +from ..models.sources import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 6b14ace..fc70f27 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -15,7 +15,7 @@ from huey.exceptions import TaskLockedException from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import SentWebmention, SentWebmentionStatus +from ..models.sources import SentWebmention, SentWebmentionStatus from ..net import ( AddressResolutionError, UnsafeAddressError, diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index d55608c..ca45777 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -15,7 +15,8 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus +from ..moderation_policy import rejection_reason from ..net import ( AddressResolutionError, ResponseTooLargeError, @@ -25,7 +26,6 @@ from ..net import ( media_type_of, read_limited_body, ) -from ..source_policy import source_rejection_reason from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target @@ -201,7 +201,7 @@ def determine_verification_outcome( :raises httpx.RequestError: If a transient HTTP request error should be retried. """ # Recheck at worker time: a source may have been blocked since submission. - if reason := source_rejection_reason(claim.source): + if reason := rejection_reason(claim.source): return ReceivedWebmentionStatus.FAILED, reason try: diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index d9c8e87..3395dd5 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -45,8 +45,8 @@ SPDX-License-Identifier: BSD-3-Clause <div class="navbar-nav flex-row gap-3"> <a class="nav-link{% if request.endpoint == 'webmentions.received_list' %} active{% endif %}" href='{{ url_for("webmentions.received_list") }}'>Received</a> - <a class="nav-link{% if request.endpoint and request.endpoint.startswith('webmentions.source_domain') %} active{% endif %}" - href="{{ url_for('webmentions.source_domain_list') }}">Source rules</a> + <a class="nav-link{% if request.endpoint and request.endpoint.startswith('webmentions.moderation') %} active{% endif %}" + href="{{ url_for('webmentions.moderation') }}">Source rules</a> <a class="nav-link{% if request.endpoint == 'webmentions.source_list' %} active{% endif %}" href='{{ url_for("webmentions.source_list") }}'>Sent</a> <form action="{{ url_for('webmentions.logout') }}" diff --git a/webmentions_ssg/templates/source_domains.html b/webmentions_ssg/templates/moderation.html index 2db942f..e45fe3f 100644 --- a/webmentions_ssg/templates/source_domains.html +++ b/webmentions_ssg/templates/moderation.html @@ -14,13 +14,13 @@ SPDX-License-Identifier: BSD-3-Clause </p> <p> Allowlist mode is <strong>{{ "enabled" if allowlist_enabled else "disabled" }}</strong>. - Set <code>SOURCE_ALLOWLIST_ENABLED = True</code> in the server configuration to + Set <code>ALLOWLIST_ENABLED = True</code> in the server configuration to accept only listed source domains. An empty enabled allowlist rejects all sources. </p> {% for kind, entries in (("blocked", blocked), ("allowed", allowed)) %} <section class="mb-4" aria-labelledby="{{ kind }}-heading"> <h2 id="{{ kind }}-heading">{{ "Blocked" if kind == "blocked" else "Allowed" }} domains</h2> - <form action="{{ url_for('webmentions.source_domain_add', kind=kind) }}" + <form action="{{ url_for('webmentions.moderation_domain_add', kind=kind) }}" method="post" class="row g-2 align-items-end mb-3"> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> @@ -50,10 +50,11 @@ SPDX-License-Identifier: BSD-3-Clause <tr> <td class="text-break">{{ entry.domain }}</td> <td> - <form action="{{ url_for('webmentions.source_domain_delete', kind=kind, entry_id=entry.id) }}" + <form action="{{ url_for('webmentions.moderation_domain_delete', kind=kind, entry_id=entry.id) }}" method="post"> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> - <button type="submit" class="btn btn-sm btn-outline-danger">{{ "Unblock" if kind == "blocked" else "Remove" }}</button> + <button type="submit" + class="btn btn-sm btn-outline-danger">{{ "Unblock" if kind == "blocked" else "Remove" }}</button> </form> </td> </tr> diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py index 7f091f0..cdf024a 100644 --- a/webmentions_ssg/views/__init__.py +++ b/webmentions_ssg/views/__init__.py @@ -4,7 +4,7 @@ """Collect and expose the routes for the blueprint.""" -from . import admin, endpoint, session +from . import endpoint, moderation, received, session, sources from .blueprint import webmentions_bp -__all__ = ["admin", "endpoint", "session", "webmentions_bp"] +__all__ = ["endpoint", "moderation", "received", "session", "sources", "webmentions_bp"] diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index 6d84d92..322e0f0 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -14,11 +14,12 @@ import sqlalchemy.exc as sa_exc from flask import Response, current_app, render_template, url_for from flask.typing import ResponseReturnValue -from .. import forms from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter -from ..models import ReceivedWebmention, source_target_digest -from ..source_policy import source_rejection_reason +from ..forms.endpoint import WebmentionSubmissionForm +from ..models.received import ReceivedWebmention +from ..models.utils import source_target_digest +from ..moderation_policy import rejection_reason from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -145,7 +146,7 @@ def receive_webmention() -> ResponseReturnValue: :returns: HTTP 201 response with the Webmention status URL in the ``Location`` header, or validation errors with HTTP 400. """ - form = forms.WebmentionSubmissionForm(meta={"csrf": False}) + form = WebmentionSubmissionForm(meta={"csrf": False}) if not form.validate_on_submit(): return form.errors, HTTPStatus.BAD_REQUEST @@ -153,7 +154,7 @@ def receive_webmention() -> ResponseReturnValue: source = cast(str, form.source.data) target = cast(str, form.target.data) - if reason := source_rejection_reason(source): + if reason := rejection_reason(source): return {"error": reason}, HTTPStatus.FORBIDDEN result = record_submission(source, target) diff --git a/webmentions_ssg/views/moderation.py b/webmentions_ssg/views/moderation.py new file mode 100644 index 0000000..c076ce9 --- /dev/null +++ b/webmentions_ssg/views/moderation.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated domain moderation views and operations.""" + +from http import HTTPStatus +from typing import cast + +import sqlalchemy as sa +from flask import abort, current_app, flash, redirect, render_template, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.exc import IntegrityError + +from ..extensions import db +from ..forms.admin import ConfirmActionForm +from ..forms.moderation import ModerateDomainForm +from ..models.moderation import AllowedDomain, BlockedDomain +from .blueprint import webmentions_bp + +MODERATION_DOMAIN_MODELS = {"blocked": BlockedDomain, "allowed": AllowedDomain} + + +def block_domains(domains: set[str]) -> int: + """Persist distinct source domains, tolerating concurrent blocklist inserts. + + Savepoints isolate unique-constraint races so one preexisting entry cannot + prevent other selected domains from being blocked. + """ + existing = set( + db.session.scalars( + sa.select(BlockedDomain.domain).where(BlockedDomain.domain.in_(domains)) + ) + ) + added = 0 + for domain in sorted(domains - existing): + try: + with db.session.begin_nested(): + db.session.add(BlockedDomain(domain=domain)) + db.session.flush() + except IntegrityError: + # A concurrent admin action has already blocked this hostname. + continue + added += 1 + db.session.commit() + return added + + +@webmentions_bp.get("/moderation") +@login_required +def moderation() -> ResponseReturnValue: + """Manage blocklist and allowlist entries used by the receiver.""" + return render_template( + "moderation.html", + title="Source domain rules", + blocked=db.session.scalars( + sa.select(BlockedDomain).order_by(BlockedDomain.domain) + ).all(), + allowed=db.session.scalars( + sa.select(AllowedDomain).order_by(AllowedDomain.domain) + ).all(), + allowlist_enabled=current_app.config["ALLOWLIST_ENABLED"], + ) + + +@webmentions_bp.post("/moderation/<kind>/add") +@login_required +def moderation_domain_add(kind: str) -> ResponseReturnValue: + """Validate and add a normalized hostname to the selected domain list.""" + if (model := MODERATION_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + form = ModerateDomainForm() + if not form.validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + domain = cast(str, form.domain.data) + if db.session.scalar(sa.select(model.id).where(model.domain == domain)) is None: + db.session.add(model(domain=domain)) + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + + flash(f"Source domain {domain} added to {kind} list.", "success") + return redirect(url_for("webmentions.moderation")) + + +@webmentions_bp.post("/moderation/<kind>/<int:entry_id>/delete") +@login_required +def moderation_domain_delete(kind: str, entry_id: int) -> ResponseReturnValue: + """Remove a rule; unblocked domains become eligible for submissions again.""" + if (model := MODERATION_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + entry = db.get_or_404(model, entry_id) + domain = entry.domain + db.session.delete(entry) + db.session.commit() + flash(f"Source domain {domain} removed from {kind} list.", "success") + return redirect(url_for("webmentions.moderation")) diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/received.py index eb5a5e7..7d633ab 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/received.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Authenticated incoming and outgoing Webmention management.""" +"""Authenticated views for received Webmentions.""" import uuid from datetime import UTC, date, datetime, time @@ -13,25 +13,20 @@ import sqlalchemy as sa from flask import abort, current_app, flash, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from flask_login import login_required -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import selectinload -from .. import forms from ..db_helpers import rows_affected from ..extensions import db -from ..models import ( - AllowedSource, - BlockedSource, - ReceivedWebmention, - ReceivedWebmentionStatus, - SentWebmention, - SourceDocument, +from ..forms.admin import ConfirmActionForm +from ..forms.received import ( + RECEIVED_STATUS_CHOICES, + ReceivedWebmentionFilterForm, + default_received_page_size, ) -from ..source_policy import source_domain_from_url -from ..tasks.scanner import manual_scan_sources -from ..tasks.sender import send_webmention +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus +from ..moderation_policy import domain_from_url from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp +from .moderation import block_domains RECEIVED_STATUS_FILTERS = { "pending": (ReceivedWebmentionStatus.RECEIVED,), @@ -42,7 +37,6 @@ RECEIVED_STATUS_FILTERS = { ReceivedWebmentionStatus.SOURCE_REMOVED, ), } -SOURCE_DOMAIN_MODELS = {"blocked": BlockedSource, "allowed": AllowedSource} class ReceivedFilterArgs(TypedDict, total=False): @@ -57,20 +51,16 @@ class ReceivedFilterArgs(TypedDict, total=False): per_page: str -def received_filter_form() -> forms.ReceivedWebmentionFilterForm: +def received_filter_form() -> ReceivedWebmentionFilterForm: """Validate query-string filters without requiring CSRF on the GET form.""" - form = forms.ReceivedWebmentionFilterForm( - formdata=request.args, meta={"csrf": False} - ) + form = ReceivedWebmentionFilterForm(formdata=request.args, meta={"csrf": False}) if not form.validate(): abort(HTTPStatus.BAD_REQUEST) form.q.data = (form.q.data or "").strip() return form -def received_filter_args( - form: forms.ReceivedWebmentionFilterForm, -) -> ReceivedFilterArgs: +def received_filter_args(form: ReceivedWebmentionFilterForm) -> ReceivedFilterArgs: """Serialize non-default form fields registered in ``ReceivedFilterArgs``.""" values: dict[str, str] = {} for name in ReceivedFilterArgs.__annotations__: @@ -147,7 +137,7 @@ def received_list() -> ResponseReturnValue: if sort_key != "received": query = query.order_by(ReceivedWebmention.uuid.desc()) - page_size = int(filters.get("per_page", forms.default_received_page_size())) + page_size = int(filters.get("per_page", default_received_page_size())) webmentions = db.paginate( query, per_page=page_size, @@ -171,11 +161,11 @@ def received_list() -> ResponseReturnValue: filters=filters, sort_key=sort_key, sort_order=sort_order, - status_choices=forms.RECEIVED_STATUS_CHOICES, + status_choices=RECEIVED_STATUS_CHOICES, page_size=page_size, sort_urls=sort_urls, max_received_page_size=current_app.config["RECEIVED_MAX_PAGE_SIZE"], - action_form=forms.ConfirmActionForm(), + action_form=ConfirmActionForm(), ) @@ -187,7 +177,7 @@ def delete_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnValu :param webmention_uuid: UUID of the Webmention to delete. :returns: Redirect response to the received Webmentions view. """ - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -217,7 +207,7 @@ def reverify_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnVa :param webmention_uuid: UUID of the Webmention to reverify. :returns: Redirect response to the received Webmentions view. """ - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -275,7 +265,7 @@ def selected_received_webmentions() -> list[ReceivedWebmention]: @login_required def reverify_selected_received_webmentions() -> ResponseReturnValue: """Reopen and queue each selected received Webmention for verification.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -303,7 +293,7 @@ def reverify_selected_received_webmentions() -> ResponseReturnValue: @login_required def delete_selected_received_webmentions() -> ResponseReturnValue: """Permanently delete only the selected received Webmentions.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -327,7 +317,7 @@ def delete_selected_received_webmentions() -> ResponseReturnValue: @login_required def delete_all_declined_received_webmentions() -> ResponseReturnValue: """Delete every declined received Webmention, regardless of age or filters.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -349,46 +339,21 @@ def delete_all_declined_received_webmentions() -> ResponseReturnValue: ) -def block_source_domains(domains: set[str]) -> int: - """Persist distinct source domains, tolerating concurrent blocklist inserts. - - Savepoints isolate unique-constraint races so one preexisting entry cannot - prevent other selected domains from being blocked. - """ - existing = set( - db.session.scalars( - sa.select(BlockedSource.domain).where(BlockedSource.domain.in_(domains)) - ) - ) - added = 0 - for domain in sorted(domains - existing): - try: - with db.session.begin_nested(): - db.session.add(BlockedSource(domain=domain)) - db.session.flush() - except IntegrityError: - # A concurrent admin action has already blocked this hostname. - continue - added += 1 - db.session.commit() - return added - - @webmentions_bp.post("/received/bulk/block-sources") @login_required def block_selected_received_sources() -> ResponseReturnValue: """Block the unique source domains of selected received Webmentions.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) webmentions = selected_received_webmentions() - domains = {source_domain_from_url(webmention.source) for webmention in webmentions} + domains = {domain_from_url(webmention.source) for webmention in webmentions} if None in domains: # Fail the entire selection instead of silently skipping a source. abort(HTTPStatus.BAD_REQUEST) - added = block_source_domains(cast(set[str], domains)) + added = block_domains(cast(set[str], domains)) flash( f"{added} source domains newly blocked from {len(webmentions)} selected Webmentions.", "success", @@ -406,15 +371,15 @@ def block_selected_received_sources() -> ResponseReturnValue: @login_required def block_received_source(webmention_uuid: uuid.UUID) -> ResponseReturnValue: """Add a received Webmention's source hostname to the blocklist.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) webmention = db.get_or_404(ReceivedWebmention, webmention_uuid) - if (domain := source_domain_from_url(webmention.source)) is None: + if (domain := domain_from_url(webmention.source)) is None: abort(HTTPStatus.BAD_REQUEST) - block_source_domains({domain}) + block_domains({domain}) flash(f"Source domain {domain} blocked.", "success") return redirect( @@ -424,162 +389,3 @@ def block_received_source(webmention_uuid: uuid.UUID) -> ResponseReturnValue: **filters, ) ) - - -@webmentions_bp.get("/source-domains") -@login_required -def source_domain_list() -> ResponseReturnValue: - """Manage blocklist and allowlist entries used by the receiver.""" - return render_template( - "source_domains.html", - title="Source domain rules", - blocked=db.session.scalars( - sa.select(BlockedSource).order_by(BlockedSource.domain) - ).all(), - allowed=db.session.scalars( - sa.select(AllowedSource).order_by(AllowedSource.domain) - ).all(), - allowlist_enabled=current_app.config["SOURCE_ALLOWLIST_ENABLED"], - ) - - -@webmentions_bp.post("/source-domains/<kind>/add") -@login_required -def source_domain_add(kind: str) -> ResponseReturnValue: - """Validate and add one normalized hostname to the selected list.""" - if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: - abort(HTTPStatus.NOT_FOUND) - - form = forms.SourceDomainForm() - if not form.validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - domain = cast(str, form.domain.data) - if db.session.scalar(sa.select(model.id).where(model.domain == domain)) is None: - db.session.add(model(domain=domain)) - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - - flash(f"Source domain {domain} added to {kind} list.", "success") - return redirect(url_for("webmentions.source_domain_list")) - - -@webmentions_bp.post("/source-domains/<kind>/<int:entry_id>/delete") -@login_required -def source_domain_delete(kind: str, entry_id: int) -> ResponseReturnValue: - """Remove one entry; a blocked domain becomes eligible for new submissions.""" - if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: - abort(HTTPStatus.NOT_FOUND) - - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - entry = db.get_or_404(model, entry_id) - domain = entry.domain - db.session.delete(entry) - db.session.commit() - flash(f"Source domain {domain} removed from {kind} list.", "success") - return redirect(url_for("webmentions.source_domain_list")) - - -@webmentions_bp.route("/sources") -@login_required -def source_list() -> ResponseReturnValue: - """Display sources with sent Webmentions. - - :returns: Rendered page containing the paginated source list. - """ - sources = db.paginate( - sa.select(SourceDocument) - .options(selectinload(SourceDocument.sent_webmentions)) - .order_by(SourceDocument.last_seen_at.desc()), - per_page=25, - ) - return render_template( - "sent.html", - title="Sent Webmentions", - sources=sources, - action_form=forms.ConfirmActionForm(), - ) - - -@webmentions_bp.post("/sources/rescan") -@login_required -def rescan_sources() -> ResponseReturnValue: - """Queue a manual scan of sent Webmention sources. - - :returns: Redirect response to the sent Webmentions view. - """ - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - manual_scan_sources() - flash("Source rescan queued.", "success") - - return redirect(url_for("webmentions.source_list")) - - -@webmentions_bp.route("/sources/<uuid:source_uuid>") -@login_required -def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: - """Display the sent Webmentions associated with a source. - - :param source_uuid: UUID of the source to display. - :returns: Rendered source details page. - """ - source = db.get_or_404(SourceDocument, source_uuid) - - webmentions = db.paginate( - sa.select(SentWebmention) - .where(SentWebmention.source_uuid == source.uuid) - .order_by(SentWebmention.target, SentWebmention.uuid), - per_page=25, - ) - - return render_template( - "sent_source.html", - title="Sent Webmentions", - source=source, - webmentions=webmentions, - action_form=forms.ConfirmActionForm(), - ) - - -@webmentions_bp.post("/sources/<uuid:source_uuid>/<uuid:webmention_uuid>/resend") -@login_required -def resend_webmention( - source_uuid: uuid.UUID, webmention_uuid: uuid.UUID -) -> ResponseReturnValue: - """Queue an outgoing Webmention to be sent again. - - The current desired source revision is reopened for processing so the - sender task does not treat the Webmention as already completed. Historical - successful-send fields are preserved, while result fields from the previous - attempt are cleared. - - :param source_uuid: UUID of the source owning the Webmention. - :param webmention_uuid: UUID of the sent Webmention to resend. - :returns: Redirect response to the source details view. - """ - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - webmention = db.get_or_404(SentWebmention, webmention_uuid) - - if webmention.source_uuid != source_uuid: - abort(HTTPStatus.NOT_FOUND) - - # Requeue the existing desired revision instead of inventing a new source - # revision. send_webmention() otherwise exits when this revision has already - # been processed. - webmention.clear_result_for_resend() - - db.session.commit() - - send_webmention(webmention_uuid) - - flash(f"Webmention {webmention.uuid} queued for resend.", "success") - - return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py index d9cec7f..78d1ed6 100644 --- a/webmentions_ssg/views/session.py +++ b/webmentions_ssg/views/session.py @@ -10,10 +10,10 @@ from flask.typing import ResponseReturnValue from flask_login import current_user, login_required, login_user, logout_user from werkzeug.security import check_password_hash -from .. import forms from ..auth import dummy_password_hash, is_safe_next_url, login_username_rate_limit_key from ..extensions import db, limiter -from ..models import User +from ..forms.user import LoginForm +from ..models.user import User from .blueprint import webmentions_bp @@ -50,7 +50,7 @@ def login() -> ResponseReturnValue: if current_user.is_authenticated: return redirect(url_for("webmentions.index")) - form = forms.LoginForm() + form = LoginForm() if form.validate_on_submit(): user = db.session.scalar( diff --git a/webmentions_ssg/views/sources.py b/webmentions_ssg/views/sources.py new file mode 100644 index 0000000..b68d2fd --- /dev/null +++ b/webmentions_ssg/views/sources.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated source document and sent Webmention views.""" + +import uuid +from http import HTTPStatus + +import sqlalchemy as sa +from flask import abort, flash, redirect, render_template, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.orm import selectinload + +from ..extensions import db +from ..forms.admin import ConfirmActionForm +from ..models.sources import SentWebmention, SourceDocument +from ..tasks.scanner import manual_scan_sources +from ..tasks.sender import send_webmention +from .blueprint import webmentions_bp + + +@webmentions_bp.route("/sources") +@login_required +def source_list() -> ResponseReturnValue: + """Display sources with sent Webmentions. + + :returns: Rendered page containing the paginated source list. + """ + sources = db.paginate( + sa.select(SourceDocument) + .options(selectinload(SourceDocument.sent_webmentions)) + .order_by(SourceDocument.last_seen_at.desc()), + per_page=25, + ) + return render_template( + "sent.html", + title="Sent Webmentions", + sources=sources, + action_form=ConfirmActionForm(), + ) + + +@webmentions_bp.post("/sources/rescan") +@login_required +def rescan_sources() -> ResponseReturnValue: + """Queue a manual scan of sent Webmention sources. + + :returns: Redirect response to the sent Webmentions view. + """ + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + manual_scan_sources() + flash("Source rescan queued.", "success") + + return redirect(url_for("webmentions.source_list")) + + +@webmentions_bp.route("/sources/<uuid:source_uuid>") +@login_required +def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: + """Display the sent Webmentions associated with a source. + + :param source_uuid: UUID of the source to display. + :returns: Rendered source details page. + """ + source = db.get_or_404(SourceDocument, source_uuid) + + webmentions = db.paginate( + sa.select(SentWebmention) + .where(SentWebmention.source_uuid == source.uuid) + .order_by(SentWebmention.target, SentWebmention.uuid), + per_page=25, + ) + + return render_template( + "sent_source.html", + title="Sent Webmentions", + source=source, + webmentions=webmentions, + action_form=ConfirmActionForm(), + ) + + +@webmentions_bp.post("/sources/<uuid:source_uuid>/<uuid:webmention_uuid>/resend") +@login_required +def resend_webmention( + source_uuid: uuid.UUID, webmention_uuid: uuid.UUID +) -> ResponseReturnValue: + """Queue an outgoing Webmention to be sent again. + + The current desired source revision is reopened for processing so the + sender task does not treat the Webmention as already completed. Historical + successful-send fields are preserved, while result fields from the previous + attempt are cleared. + + :param source_uuid: UUID of the source owning the Webmention. + :param webmention_uuid: UUID of the sent Webmention to resend. + :returns: Redirect response to the source details view. + """ + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + webmention = db.get_or_404(SentWebmention, webmention_uuid) + + if webmention.source_uuid != source_uuid: + abort(HTTPStatus.NOT_FOUND) + + # Requeue the existing desired revision instead of inventing a new source + # revision. send_webmention() otherwise exits when this revision has already + # been processed. + webmention.clear_result_for_resend() + + db.session.commit() + + send_webmention(webmention_uuid) + + flash(f"Webmention {webmention.uuid} queued for resend.", "success") + + return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) |
