diff options
Diffstat (limited to '')
63 files changed, 2235 insertions, 1949 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 882c86d..9be2cf0 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -95,12 +95,12 @@ MAX_REDIRECTS = 20 # also used for outgoing requests # Webmention. Include each hostname you serve; wildcards are not supported. ALLOWED_TARGET_HOSTNAMES = {"example.com", "www.example.com"} -# Optional source allowlist mode. By default, every valid public source is +# Optional allowlist mode. By default, every valid public source is # accepted unless its hostname appears in the admin-managed blocklist. # When enabled, a source must match an allowed domain; blocked entries always -# override allowed entries. Manage both lists in the authenticated "Source rules" +# override allowed entries. Manage both lists in the authenticated "Moderation" # page. Entries include subdomains; an empty allowlist rejects all sources. -SOURCE_ALLOWLIST_ENABLED = False +ALLOWLIST_ENABLED = False # Maximum size of an incoming HTTP request body, including form encoding. This # global Flask limit also applies to login and administration routes. diff --git a/tests/forms/__init__.py b/tests/forms/__init__.py new file mode 100644 index 0000000..25d78d5 --- /dev/null +++ b/tests/forms/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/forms/test_admin.py b/tests/forms/test_admin.py new file mode 100644 index 0000000..cb011ee --- /dev/null +++ b/tests/forms/test_admin.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from flask import Flask + +from webmentions_ssg.forms.admin import ConfirmActionForm + + +def test_admin_action_form_accepts_submission(app: Flask) -> None: + with app.test_request_context(method="POST"): + form = ConfirmActionForm(meta={"csrf": False}) + assert form.validate() diff --git a/tests/test_forms.py b/tests/forms/test_endpoint.py index c6a8b25..11f233d 100644 --- a/tests/test_forms.py +++ b/tests/forms/test_endpoint.py @@ -5,26 +5,13 @@ import pytest from flask import Flask from werkzeug.datastructures import MultiDict -from wtforms import Form, StringField, ValidationError -from webmentions_ssg.forms import ( - AllowedTargetHostname, - ConfirmActionForm, - LoginForm, - MaxURLBytes, - ReceivedWebmentionFilterForm, - RejectNonPublicIPLiteral, - WebmentionSubmissionForm, -) +from webmentions_ssg.forms.endpoint import WebmentionSubmissionForm VALID_SOURCE = "https://source.example/post" VALID_TARGET = "https://dennisfink.me/blog/example/" -class URLForm(Form): - url = StringField("URL") - - @pytest.mark.parametrize( ("form_data", "invalid_field", "expected_error"), [ @@ -118,122 +105,6 @@ def test_endpoint_form_accepts_valid_data(app: Flask) -> None: assert form.errors == {} -@pytest.mark.parametrize( - ("form_data", "invalid_field"), - [ - pytest.param({"password": "secret"}, "username", id="username-required"), - pytest.param({"username": "admin"}, "password", id="password-required"), - ], -) -def test_login_form_requires_credentials( - app: Flask, form_data: dict[str, str], invalid_field: str -) -> None: - with app.test_request_context("/login", method="POST"): - form = LoginForm(formdata=MultiDict(form_data), meta={"csrf": False}) - - assert not form.validate() - assert "This field is required." in form.errors[invalid_field] - - -def test_login_form_accepts_credentials(app: Flask) -> None: - with app.test_request_context("/login", method="POST"): - form = LoginForm( - formdata=MultiDict({"username": "admin", "password": "secret"}), - meta={"csrf": False}, - ) - assert form.validate() - - -def test_received_filter_form_accepts_valid_get_params(app: Flask) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict( - { - "status": "declined", - "q": "example.org", - "date_from": "2026-11-08", - "date_to": "2026-11-14", - } - ), - meta={"csrf": False}, - ) - assert form.validate() - assert form.status.data == "declined" - assert form.q.data == "example.org" - assert form.date_from.data.isoformat() == "2026-11-08" - assert form.date_to.data.isoformat() == "2026-11-14" - assert "csrf_token" not in form._fields - - -@pytest.mark.parametrize( - "data", - [ - {"status": "unknown"}, - {"date_from": "not-a-date"}, - {"date_from": "2026-11-8"}, - {"date_to": "2026-02-30"}, - {"date_from": "2026-11-15", "date_to": "2026-11-08"}, - ], -) -def test_received_filter_form_rejects_invalid_get_params( - app: Flask, data: dict[str, str] -) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict(data), meta={"csrf": False} - ) - assert not form.validate() - assert form.errors - - -def test_received_filter_form_validates_sort_and_page_size(app: Flask) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict({"sort": "updated", "order": "asc", "per_page": "37"}), - meta={"csrf": False}, - ) - assert form.validate() - assert form.sort.data == "updated" - assert form.order.data == "asc" - assert form.per_page.data == 37 - - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm(formdata=MultiDict(), meta={"csrf": False}) - assert form.validate() - assert (form.sort.data, form.order.data, form.per_page.data) == ( - "received", - "desc", - 25, - ) - - -@pytest.mark.parametrize( - "data", - [ - {"sort": "invalid"}, - {"order": "invalid"}, - {"per_page": "0"}, - {"per_page": "101"}, - {"per_page": "invalid"}, - ], -) -def test_received_filter_form_rejects_invalid_sort_and_page_size( - app: Flask, data: dict[str, str] -) -> None: - with app.test_request_context("/received", method="GET"): - form = ReceivedWebmentionFilterForm( - formdata=MultiDict(data), meta={"csrf": False} - ) - assert not form.validate() - assert form.errors - - -def test_admin_action_form_accepts_submission(app: Flask) -> None: - with app.test_request_context(method="POST"): - form = ConfirmActionForm(meta={"csrf": False}) - assert form.validate() - - def test_endpoint_does_not_do_dns_in_web_request( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -355,73 +226,3 @@ def test_endpoint_form_rejects_matching_source_and_target(app: Flask) -> None: assert not form.validate() assert "Field must not be equal to target." in form.errors["source"] - - -def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) - with app.app_context(): - AllowedTargetHostname()(form, form.url) - - -def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedTargetHostname()(form, form.url) - - -def test_allowed_hostname_uses_custom_message(app: Flask) -> None: - app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] - form = URLForm(MultiDict({"url": "https://example.com/post"})) - with ( - app.app_context(), - pytest.raises(ValidationError, match="Hostname is not allowed"), - ): - AllowedTargetHostname("Hostname is not allowed")(form, form.url) - - -@pytest.mark.parametrize( - "address", - ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], -) -def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: - form = URLForm(MultiDict({"url": f"https://[{address}]/"})) - with pytest.raises(ValidationError, match="URL must use a public address"): - RejectNonPublicIPLiteral()(form, form.url) - - -def test_public_ip_literal_accepts_public_ipv6_address() -> None: - form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) - RejectNonPublicIPLiteral()(form, form.url) - - -def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: - form = URLForm(MultiDict({"url": "http://[::1"})) - with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): - AllowedTargetHostname()(form, form.url) - - -def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: - validator = MaxURLBytes() - form = URLForm(MultiDict({"url": "é" * 3})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 6 - validator(form, form.url) - app.config["MAX_WEBMENTION_URL_BYTES"] = 5 - with pytest.raises(ValidationError, match="5 bytes"): - validator(form, form.url) - - -def test_url_max_bytes_custom_message(app: Flask) -> None: - form = URLForm(MultiDict({"url": "too-long"})) - with app.app_context(): - app.config["MAX_WEBMENTION_URL_BYTES"] = 2 - with pytest.raises(ValidationError, match="URL too long"): - MaxURLBytes(message="URL too long")(form, form.url) - - -def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: - form = URLForm(MultiDict({"url": "\ud800"})) - with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): - MaxURLBytes()(form, form.url) diff --git a/tests/forms/test_received.py b/tests/forms/test_received.py new file mode 100644 index 0000000..4ef9ed3 --- /dev/null +++ b/tests/forms/test_received.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.received import ReceivedWebmentionFilterForm + + +def test_received_filter_form_accepts_valid_get_params(app: Flask) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict( + { + "status": "declined", + "q": "example.org", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + } + ), + meta={"csrf": False}, + ) + assert form.validate() + assert form.status.data == "declined" + assert form.q.data == "example.org" + assert form.date_from.data.isoformat() == "2026-11-08" + assert form.date_to.data.isoformat() == "2026-11-14" + assert "csrf_token" not in form._fields + + +@pytest.mark.parametrize( + "data", + [ + {"status": "unknown"}, + {"date_from": "not-a-date"}, + {"date_from": "2026-11-8"}, + {"date_to": "2026-02-30"}, + {"date_from": "2026-11-15", "date_to": "2026-11-08"}, + ], +) +def test_received_filter_form_rejects_invalid_get_params( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict(data), meta={"csrf": False} + ) + assert not form.validate() + assert form.errors + + +def test_received_filter_form_validates_sort_and_page_size(app: Flask) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict({"sort": "updated", "order": "asc", "per_page": "37"}), + meta={"csrf": False}, + ) + assert form.validate() + assert form.sort.data == "updated" + assert form.order.data == "asc" + assert form.per_page.data == 37 + + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm(formdata=MultiDict(), meta={"csrf": False}) + assert form.validate() + assert (form.sort.data, form.order.data, form.per_page.data) == ( + "received", + "desc", + 25, + ) + + +@pytest.mark.parametrize( + "data", + [ + {"sort": "invalid"}, + {"order": "invalid"}, + {"per_page": "0"}, + {"per_page": "101"}, + {"per_page": "invalid"}, + ], +) +def test_received_filter_form_rejects_invalid_sort_and_page_size( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/received", method="GET"): + form = ReceivedWebmentionFilterForm( + formdata=MultiDict(data), meta={"csrf": False} + ) + assert not form.validate() + assert form.errors diff --git a/tests/forms/test_user.py b/tests/forms/test_user.py new file mode 100644 index 0000000..4b35a9a --- /dev/null +++ b/tests/forms/test_user.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.user import LoginForm + + +@pytest.mark.parametrize( + ("form_data", "invalid_field"), + [ + pytest.param({"password": "secret"}, "username", id="username-required"), + pytest.param({"username": "admin"}, "password", id="password-required"), + ], +) +def test_login_form_requires_credentials( + app: Flask, form_data: dict[str, str], invalid_field: str +) -> None: + with app.test_request_context("/login", method="POST"): + form = LoginForm(formdata=MultiDict(form_data), meta={"csrf": False}) + + assert not form.validate() + assert "This field is required." in form.errors[invalid_field] + + +def test_login_form_accepts_credentials(app: Flask) -> None: + with app.test_request_context("/login", method="POST"): + form = LoginForm( + formdata=MultiDict({"username": "admin", "password": "secret"}), + meta={"csrf": False}, + ) + assert form.validate() diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py new file mode 100644 index 0000000..4234552 --- /dev/null +++ b/tests/forms/test_validators.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict +from wtforms import Form, StringField, ValidationError + +from webmentions_ssg.forms.validators import ( + AllowedTargetHostname, + MaxURLBytes, + RejectNonPublicIPLiteral, +) + + +class URLForm(Form): + url = StringField("URL") + + +def test_allowed_hostname_accepts_configured_hostname(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://dennisfink.me/blog/example/"})) + with app.app_context(): + AllowedTargetHostname()(form, form.url) + + +def test_allowed_hostname_rejects_unconfigured_hostname(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedTargetHostname()(form, form.url) + + +def test_allowed_hostname_uses_custom_message(app: Flask) -> None: + app.config["ALLOWED_TARGET_HOSTNAMES"] = ["dennisfink.me"] + form = URLForm(MultiDict({"url": "https://example.com/post"})) + with ( + app.app_context(), + pytest.raises(ValidationError, match="Hostname is not allowed"), + ): + AllowedTargetHostname("Hostname is not allowed")(form, form.url) + + +@pytest.mark.parametrize( + "address", + ["64:ff9b::a9fe:a9fe", "::10.0.0.1", "2002:a9fe:a9fe::1", "::ffff:169.254.169.254"], +) +def test_public_ip_literal_rejects_ipv6_transition_addresses(address: str) -> None: + form = URLForm(MultiDict({"url": f"https://[{address}]/"})) + with pytest.raises(ValidationError, match="URL must use a public address"): + RejectNonPublicIPLiteral()(form, form.url) + + +def test_public_ip_literal_accepts_public_ipv6_address() -> None: + form = URLForm(MultiDict({"url": "https://[2606:2800:220:1:248:1893:25c8:1946]/"})) + RejectNonPublicIPLiteral()(form, form.url) + + +def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: + form = URLForm(MultiDict({"url": "http://[::1"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedTargetHostname()(form, form.url) + + +def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: + validator = MaxURLBytes() + form = URLForm(MultiDict({"url": "é" * 3})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 6 + validator(form, form.url) + app.config["MAX_WEBMENTION_URL_BYTES"] = 5 + with pytest.raises(ValidationError, match="5 bytes"): + validator(form, form.url) + + +def test_url_max_bytes_custom_message(app: Flask) -> None: + form = URLForm(MultiDict({"url": "too-long"})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 2 + with pytest.raises(ValidationError, match="URL too long"): + MaxURLBytes(message="URL too long")(form, form.url) + + +def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: + form = URLForm(MultiDict({"url": "\ud800"})) + with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): + MaxURLBytes()(form, form.url) diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..25d78d5 --- /dev/null +++ b/tests/models/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/models/test_received.py b/tests/models/test_received.py new file mode 100644 index 0000000..6f74d2d --- /dev/null +++ b/tests/models/test_received.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import db +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.models.utils import source_target_digest + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + (ReceivedWebmentionStatus.RECEIVED, False), + (ReceivedWebmentionStatus.VERIFYING, False), + (ReceivedWebmentionStatus.VERIFIED, True), + (ReceivedWebmentionStatus.FAILED, False), + (ReceivedWebmentionStatus.SOURCE_REMOVED, False), + ], +) +def test_received_webmention_verified( + status: ReceivedWebmentionStatus, expected: bool +) -> None: + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + status=status, + ) + + assert webmention.verified is expected + + +def test_received_webmention_repr() -> None: + webmention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + ) + + assert repr(webmention) == ( + "<ReceivedWebmention 'https://source.example/post' -> " + "'https://dennisfink.me/blog/example/'>" + ) + + +def test_received_status_uses_non_native_enum() -> None: + status_type = ReceivedWebmention.__table__.c.status.type + + assert isinstance(status_type, sa.Enum) + assert status_type.enum_class is ReceivedWebmentionStatus + assert status_type.native_enum is False + assert status_type.enums == [status.value for status in ReceivedWebmentionStatus] + + +def test_received_status_round_trips_as_enum(app: Flask) -> None: + with app.app_context(): + identifier = uuid.uuid7() + mention = ReceivedWebmention( + uuid=identifier, + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + status=ReceivedWebmentionStatus.VERIFIED, + ) + db.session.add(mention) + db.session.commit() + + assert ( + db.session.scalar( + sa.text("SELECT status FROM received_webmentions WHERE uuid = :uuid"), + {"uuid": identifier.hex}, + ) + == "verified" + ) + + db.session.expunge_all() + loaded = db.session.get(ReceivedWebmention, identifier) + assert loaded is not None + assert loaded.status is ReceivedWebmentionStatus.VERIFIED + + +def test_received_verification_reset_is_reusable_for_atomic_updates() -> None: + mention = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/post", + target="https://dennisfink.me/post", + status=ReceivedWebmentionStatus.FAILED, + failure_reason="Old error", + recovery_attempts=3, + ) + assert mention.reopen_verification() is None + assert mention.status == ReceivedWebmentionStatus.RECEIVED + assert mention.failure_reason is None + assert mention.recovery_attempts == 0 + assert mention.updated_at is not None + + +def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: + long_source = "https://source.example/" + "a" * 4000 + long_target = "https://dennisfink.me/" + "b" * 4000 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + db.session.commit() + match = db.session.scalar( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.source_target_digest + == source_target_digest(long_source, long_target) + ) + ) + assert match is not None + assert match.source == long_source + assert match.target == long_target + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/models/test_sources.py b/tests/models/test_sources.py new file mode 100644 index 0000000..8d01d79 --- /dev/null +++ b/tests/models/test_sources.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime +from http import HTTPStatus +from typing import get_type_hints + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Mapped + +from webmentions_ssg import db +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) +from webmentions_ssg.models.utils import target_digest + + +def test_uuid_annotations_and_relationships_resolve() -> None: + """Verify that UUID columns and forward relationships map correctly.""" + for model in (ReceivedWebmention, SourceDocument, SentWebmention): + assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID] + assert sa.inspect(model).columns["uuid"].primary_key + + assert ( + sa.inspect(SourceDocument).relationships["sent_webmentions"].mapper.class_ + is SentWebmention + ) + assert ( + sa.inspect(SentWebmention).relationships["source"].mapper.class_ + is SourceDocument + ) + + +@pytest.mark.parametrize( + ("desired_revision", "processed_revision", "expected"), + [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)], +) +def test_sent_webmention_pending( + desired_revision: int, processed_revision: int | None, expected: bool +) -> None: + webmention = SentWebmention( + target="https://example.com/post", + desired_revision=desired_revision, + processed_revision=processed_revision, + ) + assert webmention.awaiting_processing is expected + + +def test_sent_webmention_pending_sql_matches_python(app: Flask) -> None: + cases = [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)] + + with app.app_context(): + now = datetime.now(UTC) + source = SourceDocument( + path="pending/index.html", + url="https://site.example/pending/", + content_hash="0" * 32, + last_seen_at=now, + revised_at=now, + ) + for i, (desired, processed, _) in enumerate(cases): + source.sent_webmentions.append( + SentWebmention( + target=f"https://example.com/post/{i}", + desired_revision=desired, + processed_revision=processed, + ) + ) + db.session.add(source) + db.session.commit() + + pending_ids = set( + db.session.scalars( + sa.select(SentWebmention.uuid).where(SentWebmention.awaiting_processing) + ) + ) + for webmention, (_, _, expected) in zip( + source.sent_webmentions, cases, strict=True + ): + assert webmention.awaiting_processing is expected + assert (webmention.uuid in pending_ids) is expected + + result = db.session.execute( + sa.update(SentWebmention) + .where(SentWebmention.awaiting_processing) + .values(attempt_count=1) + ) + assert result.rowcount == 3 + + +def test_source_repr() -> None: + source = SourceDocument( + path="example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=1, + last_seen_at=datetime.now(UTC), + revised_at=datetime.now(UTC), + ) + assert repr(source) == "<SourceDocument 'https://dennisfink.me/blog/example/'>" + + +def test_sent_webmention_repr() -> None: + webmention = SentWebmention(target="https://example.com/post", desired_revision=1) + assert repr(webmention) == "<SentWebmention 'https://example.com/post'>" + + +def test_reopen_preserves_delivery_history() -> None: + sent_at = datetime.now(UTC) + mention = SentWebmention( + target="https://example.com/post", + linked=True, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + attempt_count=3, + next_retry_at=sent_at, + last_sent_at=sent_at, + ) + assert mention.reset_for_revision(3, False) is None + assert mention.desired_revision == 3 + assert mention.linked is False + assert mention.attempt_count == 0 + assert mention.next_retry_at is None + assert mention.status == SentWebmentionStatus.SENT + assert mention.sent_revision == 2 + + +def test_resend_requeues_current_revision_without_losing_send_history() -> None: + sent_at = datetime.now(UTC) + mention = SentWebmention( + target="https://example.com/post", + linked=True, + desired_revision=3, + processed_revision=3, + sent_revision=2, + status=SentWebmentionStatus.FAILED, + failure_reason="Previous failure", + webmention_endpoint="https://example.com/webmention", + endpoint_response_status=HTTPStatus.SERVICE_UNAVAILABLE, + status_url="https://example.com/status", + attempt_count=3, + next_retry_at=sent_at, + last_attempted_at=sent_at, + last_sent_at=sent_at, + ) + assert not mention.awaiting_processing + + assert mention.clear_result_for_resend() is None + assert mention.desired_revision == 3 + assert mention.linked is True + assert mention.awaiting_processing + assert mention.processed_revision is None + assert mention.status is None + assert mention.failure_reason is None + assert mention.webmention_endpoint is None + assert mention.endpoint_response_status is None + assert mention.status_url is None + assert mention.attempt_count == 0 + assert mention.next_retry_at is None + assert mention.sent_revision == 2 + assert mention.last_attempted_at == sent_at + assert mention.last_sent_at == sent_at + + +def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( + app: Flask, +) -> None: + now = datetime.now(UTC) + long_target = "https://destination.example/" + "a" * 4000 + with app.app_context(): + source = SourceDocument( + path="long/index.html", + url="https://example.com/long", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + db.session.add(source) + db.session.commit() + assert source.sent_webmentions[0].target_digest == target_digest(long_target) + + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/models/test_user.py b/tests/models/test_user.py new file mode 100644 index 0000000..a3f95ae --- /dev/null +++ b/tests/models/test_user.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest + +from webmentions_ssg.models.user import User + + +def test_user_repr() -> None: + user = User(username="admin", password="secret") + assert repr(user) == "<User admin>" + + +def test_user_password_is_write_only() -> None: + user = User(username="admin", password="secret") + with pytest.raises(AttributeError, match="Password is write-only"): + _ = user.password + + +def test_user_password_is_hashed_and_can_be_checked() -> None: + user = User(username="admin", password="secret") + assert user.password_hash != "secret" + assert user.check_password("secret") + assert not user.check_password("wrong") + + +def test_user_session_id_changes_when_password_hash_changes() -> None: + user = User(username="admin", password="a strong passphrase", id=42) + original_hash = user.password_hash + original_id = user.get_id() + + assert original_id.startswith("42:") + assert original_hash not in original_id + assert len(original_id.split(":", 1)[1]) == 64 + + # Even a reset to the same password rehashes with a new salt, revoking + # any old session token. + user.password = "a strong passphrase" + assert user.password_hash != original_hash + assert user.get_id() != original_id diff --git a/tests/models/test_utils.py b/tests/models/test_utils.py new file mode 100644 index 0000000..8722a02 --- /dev/null +++ b/tests/models/test_utils.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from collections.abc import Callable +from datetime import UTC, datetime + +import pytest + +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import SentWebmention, SourceDocument +from webmentions_ssg.models.utils import ( + source_target_digest, + target_digest, + uuid7_to_datetime, +) + + +def test_uuid7_to_datetime() -> None: + identifier = uuid.uuid7() + timestamp = uuid7_to_datetime(identifier) + + assert timestamp.tzinfo is UTC + assert timestamp.timestamp() == pytest.approx(identifier.time / 1000) + + +@pytest.mark.parametrize( + "factory", + [ + pytest.param( + lambda identifier: ReceivedWebmention( + uuid=identifier, + source="https://source.example/post", + target="https://dennisfink.me/blog/example/", + ), + id="received-webmention", + ), + pytest.param( + lambda identifier: SourceDocument( + uuid=identifier, + path="example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=1, + last_seen_at=datetime.now(UTC), + revised_at=datetime.now(UTC), + ), + id="source", + ), + pytest.param( + lambda identifier: SentWebmention( + uuid=identifier, target="https://example.com/post", desired_revision=1 + ), + id="sent-webmention", + ), + ], +) +def test_created_at_comes_from_uuid7_timestamp( + factory: Callable[ + [uuid.UUID], ReceivedWebmention | SourceDocument | SentWebmention + ], +) -> None: + identifier = uuid.uuid7() + model = factory(identifier) + assert model.created_at == uuid7_to_datetime(identifier) + + +def test_received_pair_fingerprint_is_unambiguous() -> None: + assert source_target_digest("a", "bc") != source_target_digest("ab", "c") + assert source_target_digest("a", "b") != source_target_digest("b", "a") + + +def test_models_hash_url_fields_when_created_or_changed() -> None: + received = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/a", + target="https://site.example/b", + ) + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + received.source = "https://source.example/c" + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + received.target = "https://site.example/d" + assert received.source_target_digest == source_target_digest( + received.source, received.target + ) + + sent = SentWebmention(target="https://site.example/a", desired_revision=1) + assert sent.target_digest == target_digest(sent.target) + sent.target = "https://site.example/b" + assert sent.target_digest == target_digest(sent.target) diff --git a/tests/tasks/test_notifications.py b/tests/tasks/test_notifications.py index ca89a7a..dadec3e 100644 --- a/tests/tasks/test_notifications.py +++ b/tests/tasks/test_notifications.py @@ -13,7 +13,7 @@ from huey.exceptions import TaskLockedException from webmentions_ssg import db from webmentions_ssg.extensions import huey -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention @pytest.fixture diff --git a/tests/tasks/test_retention.py b/tests/tasks/test_retention.py index 70ab0e0..a3891a9 100644 --- a/tests/tasks/test_retention.py +++ b/tests/tasks/test_retention.py @@ -11,7 +11,7 @@ import sqlalchemy as sa from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention from webmentions_ssg.tasks.retention import purge_old_received_webmentions diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 4cf2a67..dbeeac9 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -17,7 +17,11 @@ from huey.exceptions import RetryTask from pytest import MonkeyPatch from webmentions_ssg import db -from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, SourceDocument +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) from webmentions_ssg.tasks import source_parser from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 9a0430c..0d68933 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -14,7 +14,11 @@ import pytest from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, SourceDocument +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" diff --git a/tests/tasks/test_verification.py b/tests/tasks/test_verification.py index c5550c8..3547464 100644 --- a/tests/tasks/test_verification.py +++ b/tests/tasks/test_verification.py @@ -15,7 +15,8 @@ import sqlalchemy as sa from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import BlockedSource, ReceivedWebmention +from webmentions_ssg.models.moderation import BlockedDomain +from webmentions_ssg.models.received import ReceivedWebmention from webmentions_ssg.net import AddressResolutionError SOURCE_URL = "https://source.example/article" @@ -363,7 +364,7 @@ def test_verify_webmention_refuses_domain_blocked_after_enqueue( ) -> None: identifier = create_webmention(app) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() fetch = Mock(return_value=True) diff --git a/tests/test_auth.py b/tests/test_auth.py index 8219a07..27e8fb0 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -16,7 +16,7 @@ from webmentions_ssg.auth import ( login_manager, validate_new_password, ) -from webmentions_ssg.models import User +from webmentions_ssg.models.user import User @pytest.mark.parametrize("character", ["\x00", "\x1f", "\x7f", "\x80", "\x9f"]) diff --git a/tests/test_cli.py b/tests/test_cli.py index f5535d2..5e71e2a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,7 +19,8 @@ from sqlalchemy.exc import IntegrityError from webmentions_ssg import create_app, db, migrate from webmentions_ssg.config import DevelopmentConfig, TestingConfig -from webmentions_ssg.models import ReceivedWebmention, ReceivedWebmentionStatus, User +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.models.user import User def test_cli_groups_are_registered(app: Flask) -> None: @@ -158,6 +159,8 @@ def test_create_db_builds_schema_and_records_latest_migration( assert "source_target_digest" in received_columns assert {"attempt_count", "next_retry_at"} <= columns assert "recovery_attempts" in received_columns + assert {"blocked_domains", "allowed_domains"} <= table_names + assert not {"blocked_sources", "allowed_sources"} & table_names assert "ix_received_webmentions_status_updated_at" in received_indexes @@ -250,11 +253,27 @@ def test_upgrade_merges_existing_migration_branches( """Existing installations on either branch can upgrade to a single head.""" with database_app.app_context(): migrate_upgrade(revision=existing_head) + if existing_head == "3e8c87a96b2d": + with db.engine.begin() as connection: + connection.execute( + sa.text("INSERT INTO blocked_sources (domain) VALUES (:domain)"), + {"domain": "existing.example"}, + ) migrate_upgrade() assert _database_revision(database_app) == _head_revision(database_app) tables = set(inspect(db.engine).get_table_names()) - assert {"blocked_sources", "allowed_sources", "source_documents"} <= tables + assert {"blocked_domains", "allowed_domains", "source_documents"} <= tables + assert "blocked_sources" not in tables + assert "allowed_sources" not in tables + if existing_head == "3e8c87a96b2d": + with db.engine.connect() as connection: + assert ( + connection.execute( + sa.text("SELECT domain FROM blocked_domains") + ).scalar_one() + == "existing.example" + ) def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None: @@ -304,7 +323,7 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows( :param database_app: Flask application connected to the legacy SQLite schema. """ - from webmentions_ssg.models import source_target_digest + from webmentions_ssg.models.utils import source_target_digest with database_app.app_context(): migrate_upgrade(revision="c0272f25ce17") diff --git a/tests/test_config.py b/tests/test_config.py index 4a5f4cb..81ff498 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,7 @@ from webmentions_ssg.config import ( ProductionConfig, TestingConfig, ) -from webmentions_ssg.forms import AllowedTargetHostname +from webmentions_ssg.forms.validators import AllowedTargetHostname @pytest.mark.parametrize( @@ -42,9 +42,9 @@ def test_session_cookie_defaults( def test_source_allowlist_mode_requires_boolean_configuration(tmp_path: Path) -> None: class BadSourceModeConfig(TestingConfig): LOG_FILE = str(tmp_path / "mode-test.log") - SOURCE_ALLOWLIST_ENABLED = "yes" + ALLOWLIST_ENABLED = "yes" - with pytest.raises(ValueError, match="SOURCE_ALLOWLIST_ENABLED"): + with pytest.raises(ValueError, match="ALLOWLIST_ENABLED"): create_app(BadSourceModeConfig) diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 4e785c7..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,439 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -import uuid -from collections.abc import Callable -from datetime import UTC, datetime -from http import HTTPStatus -from typing import get_type_hints - -import pytest -import sqlalchemy as sa -from flask import Flask -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Mapped - -from webmentions_ssg import db -from webmentions_ssg.models import ( - ReceivedWebmention, - ReceivedWebmentionStatus, - SentWebmention, - SourceDocument, - User, - source_target_digest, - target_digest, - uuid7_to_datetime, -) - - -def test_uuid_annotations_and_relationships_resolve() -> None: - """Verify that UUID columns and forward relationships map correctly.""" - for model in (ReceivedWebmention, SourceDocument, SentWebmention): - assert get_type_hints(model)["uuid"] == Mapped[uuid.UUID] - assert sa.inspect(model).columns["uuid"].primary_key - - assert ( - sa.inspect(SourceDocument).relationships["sent_webmentions"].mapper.class_ - is SentWebmention - ) - assert ( - sa.inspect(SentWebmention).relationships["source"].mapper.class_ - is SourceDocument - ) - - -def test_user_repr() -> None: - user = User(username="admin", password="secret") - assert repr(user) == "<User admin>" - - -def test_user_password_is_write_only() -> None: - user = User(username="admin", password="secret") - with pytest.raises(AttributeError, match="Password is write-only"): - _ = user.password - - -def test_user_password_is_hashed_and_can_be_checked() -> None: - user = User(username="admin", password="secret") - assert user.password_hash != "secret" - assert user.check_password("secret") - assert not user.check_password("wrong") - - -@pytest.mark.parametrize( - ("status", "expected"), - [ - (ReceivedWebmentionStatus.RECEIVED, False), - (ReceivedWebmentionStatus.VERIFYING, False), - (ReceivedWebmentionStatus.VERIFIED, True), - (ReceivedWebmentionStatus.FAILED, False), - (ReceivedWebmentionStatus.SOURCE_REMOVED, False), - ], -) -def test_received_webmention_verified( - status: ReceivedWebmentionStatus, expected: bool -) -> None: - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - status=status, - ) - - assert webmention.verified is expected - - -@pytest.mark.parametrize( - ("desired_revision", "processed_revision", "expected"), - [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)], -) -def test_sent_webmention_pending( - desired_revision: int, processed_revision: int | None, expected: bool -) -> None: - webmention = SentWebmention( - target="https://example.com/post", - desired_revision=desired_revision, - processed_revision=processed_revision, - ) - assert webmention.awaiting_processing is expected - - -def test_sent_webmention_pending_sql_matches_python(app: Flask) -> None: - cases = [(1, None, True), (1, 0, True), (2, 1, True), (1, 1, False), (1, 2, False)] - - with app.app_context(): - now = datetime.now(UTC) - source = SourceDocument( - path="pending/index.html", - url="https://site.example/pending/", - content_hash="0" * 32, - last_seen_at=now, - revised_at=now, - ) - for i, (desired, processed, _) in enumerate(cases): - source.sent_webmentions.append( - SentWebmention( - target=f"https://example.com/post/{i}", - desired_revision=desired, - processed_revision=processed, - ) - ) - db.session.add(source) - db.session.commit() - - pending_ids = set( - db.session.scalars( - sa.select(SentWebmention.uuid).where(SentWebmention.awaiting_processing) - ) - ) - for webmention, (_, _, expected) in zip( - source.sent_webmentions, cases, strict=True - ): - assert webmention.awaiting_processing is expected - assert (webmention.uuid in pending_ids) is expected - - result = db.session.execute( - sa.update(SentWebmention) - .where(SentWebmention.awaiting_processing) - .values(attempt_count=1) - ) - assert result.rowcount == 3 - - -def test_uuid7_to_datetime() -> None: - identifier = uuid.uuid7() - timestamp = uuid7_to_datetime(identifier) - - assert timestamp.tzinfo is UTC - assert timestamp.timestamp() == pytest.approx(identifier.time / 1000) - - -@pytest.mark.parametrize( - "factory", - [ - pytest.param( - lambda identifier: ReceivedWebmention( - uuid=identifier, - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - ), - id="received-webmention", - ), - pytest.param( - lambda identifier: SourceDocument( - uuid=identifier, - path="example/index.html", - url="https://dennisfink.me/blog/example/", - content_hash="0" * 32, - revision=1, - last_seen_at=datetime.now(UTC), - revised_at=datetime.now(UTC), - ), - id="source", - ), - pytest.param( - lambda identifier: SentWebmention( - uuid=identifier, target="https://example.com/post", desired_revision=1 - ), - id="sent-webmention", - ), - ], -) -def test_created_at_comes_from_uuid7_timestamp( - factory: Callable[ - [uuid.UUID], ReceivedWebmention | SourceDocument | SentWebmention - ], -) -> None: - identifier = uuid.uuid7() - model = factory(identifier) - assert model.created_at == uuid7_to_datetime(identifier) - - -def test_received_webmention_repr() -> None: - webmention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - ) - - assert repr(webmention) == ( - "<ReceivedWebmention 'https://source.example/post' -> " - "'https://dennisfink.me/blog/example/'>" - ) - - -def test_source_repr() -> None: - source = SourceDocument( - path="example/index.html", - url="https://dennisfink.me/blog/example/", - content_hash="0" * 32, - revision=1, - last_seen_at=datetime.now(UTC), - revised_at=datetime.now(UTC), - ) - assert repr(source) == "<SourceDocument 'https://dennisfink.me/blog/example/'>" - - -def test_sent_webmention_repr() -> None: - webmention = SentWebmention(target="https://example.com/post", desired_revision=1) - assert repr(webmention) == "<SentWebmention 'https://example.com/post'>" - - -def test_user_session_id_changes_when_password_hash_changes() -> None: - user = User(username="admin", password="a strong passphrase", id=42) - original_hash = user.password_hash - original_id = user.get_id() - - assert original_id.startswith("42:") - assert original_hash not in original_id - assert len(original_id.split(":", 1)[1]) == 64 - - # Even a reset to the same password rehashes with a new salt, revoking - # any old session token. - user.password = "a strong passphrase" - assert user.password_hash != original_hash - assert user.get_id() != original_id - - -def test_received_status_uses_non_native_enum() -> None: - status_type = ReceivedWebmention.__table__.c.status.type - - assert isinstance(status_type, sa.Enum) - assert status_type.enum_class is ReceivedWebmentionStatus - assert status_type.native_enum is False - assert status_type.enums == [status.value for status in ReceivedWebmentionStatus] - - -def test_received_status_round_trips_as_enum(app: Flask) -> None: - with app.app_context(): - identifier = uuid.uuid7() - mention = ReceivedWebmention( - uuid=identifier, - source="https://source.example/post", - target="https://dennisfink.me/blog/example/", - status=ReceivedWebmentionStatus.VERIFIED, - ) - db.session.add(mention) - db.session.commit() - - assert ( - db.session.scalar( - sa.text("SELECT status FROM received_webmentions WHERE uuid = :uuid"), - {"uuid": identifier.hex}, - ) - == "verified" - ) - - db.session.expunge_all() - loaded = db.session.get(ReceivedWebmention, identifier) - assert loaded is not None - assert loaded.status is ReceivedWebmentionStatus.VERIFIED - - -def test_received_verification_reset_is_reusable_for_atomic_updates() -> None: - from webmentions_ssg.models import ReceivedWebmentionStatus - - mention = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/post", - target="https://dennisfink.me/post", - status=ReceivedWebmentionStatus.FAILED, - failure_reason="Old error", - recovery_attempts=3, - ) - assert mention.reopen_verification() is None - assert mention.status == ReceivedWebmentionStatus.RECEIVED - assert mention.failure_reason is None - assert mention.recovery_attempts == 0 - assert mention.updated_at is not None - - -def test_reopen_preserves_delivery_history() -> None: - from webmentions_ssg.models import SentWebmentionStatus - - sent_at = datetime.now(UTC) - mention = SentWebmention( - target="https://example.com/post", - linked=True, - desired_revision=2, - processed_revision=2, - sent_revision=2, - status=SentWebmentionStatus.SENT, - attempt_count=3, - next_retry_at=sent_at, - last_sent_at=sent_at, - ) - assert mention.reset_for_revision(3, False) is None - assert mention.desired_revision == 3 - assert mention.linked is False - assert mention.attempt_count == 0 - assert mention.next_retry_at is None - assert mention.status == SentWebmentionStatus.SENT - assert mention.sent_revision == 2 - - -def test_resend_requeues_current_revision_without_losing_send_history() -> None: - from webmentions_ssg.models import SentWebmentionStatus - - sent_at = datetime.now(UTC) - mention = SentWebmention( - target="https://example.com/post", - linked=True, - desired_revision=3, - processed_revision=3, - sent_revision=2, - status=SentWebmentionStatus.FAILED, - failure_reason="Previous failure", - webmention_endpoint="https://example.com/webmention", - endpoint_response_status=HTTPStatus.SERVICE_UNAVAILABLE, - status_url="https://example.com/status", - attempt_count=3, - next_retry_at=sent_at, - last_attempted_at=sent_at, - last_sent_at=sent_at, - ) - assert not mention.awaiting_processing - - assert mention.clear_result_for_resend() is None - assert mention.desired_revision == 3 - assert mention.linked is True - assert mention.awaiting_processing - assert mention.processed_revision is None - assert mention.status is None - assert mention.failure_reason is None - assert mention.webmention_endpoint is None - assert mention.endpoint_response_status is None - assert mention.status_url is None - assert mention.attempt_count == 0 - assert mention.next_retry_at is None - assert mention.sent_revision == 2 - assert mention.last_attempted_at == sent_at - assert mention.last_sent_at == sent_at - - -def test_received_pair_fingerprint_is_unambiguous() -> None: - assert source_target_digest("a", "bc") != source_target_digest("ab", "c") - assert source_target_digest("a", "b") != source_target_digest("b", "a") - - -def test_models_hash_url_fields_when_created_or_changed() -> None: - received = ReceivedWebmention( - uuid=uuid.uuid7(), - source="https://source.example/a", - target="https://site.example/b", - ) - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - received.source = "https://source.example/c" - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - received.target = "https://site.example/d" - assert received.source_target_digest == source_target_digest( - received.source, received.target - ) - - sent = SentWebmention(target="https://site.example/a", desired_revision=1) - assert sent.target_digest == target_digest(sent.target) - sent.target = "https://site.example/b" - assert sent.target_digest == target_digest(sent.target) - - -def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: - long_source = "https://source.example/" + "a" * 4000 - long_target = "https://dennisfink.me/" + "b" * 4000 - with app.app_context(): - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - db.session.commit() - match = db.session.scalar( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.source_target_digest - == source_target_digest(long_source, long_target) - ) - ) - assert match is not None - assert match.source == long_source - assert match.target == long_target - db.session.add( - ReceivedWebmention( - uuid=uuid.uuid7(), source=long_source, target=long_target - ) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() - - -def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( - app: Flask, -) -> None: - now = datetime.now(UTC) - long_target = "https://destination.example/" + "a" * 4000 - with app.app_context(): - source = SourceDocument( - path="long/index.html", - url="https://example.com/long", - content_hash="0" * 32, - revision=1, - last_seen_at=now, - revised_at=now, - ) - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - db.session.add(source) - db.session.commit() - assert source.sent_webmentions[0].target_digest == target_digest(long_target) - - source.sent_webmentions.append( - SentWebmention(target=long_target, desired_revision=1) - ) - with pytest.raises(IntegrityError): - db.session.commit() - db.session.rollback() diff --git a/tests/test_moderation_policy.py b/tests/test_moderation_policy.py new file mode 100644 index 0000000..48a34ac --- /dev/null +++ b/tests/test_moderation_policy.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain normalization and receiver admission policy.""" + +from flask import Flask + +from webmentions_ssg import db +from webmentions_ssg.models.moderation import AllowedDomain, BlockedDomain +from webmentions_ssg.moderation_policy import domain_from_url, rejection_reason + + +def test_domain_from_url() -> None: + assert ( + domain_from_url("https://BÜCHER.example.:8443/post") == "xn--bcher-kva.example" + ) + assert domain_from_url("https://[2001:db8::1]/post") == "2001:db8::1" + assert domain_from_url("invalid") is None + + +def test_blocked_domain_matches_itself_and_subdomains_but_not_suffixes( + app: Flask, +) -> None: + with app.app_context(): + db.session.add(BlockedDomain(domain="example.org")) + db.session.commit() + for source in ( + "https://example.org/post", + "https://blog.example.org/post", + "https://A.blog.Example.ORG./post", + ): + assert rejection_reason(source) == "Source domain is blocked" + for source in ( + "https://notexample.org/post", + "https://example.org.evil/post", + "https://other.net/post", + ): + assert rejection_reason(source) is None + + +def test_allowlist_is_fail_closed_and_blocklist_takes_precedence(app: Flask) -> None: + app.config["ALLOWLIST_ENABLED"] = True + with app.app_context(): + assert ( + rejection_reason("https://example.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(AllowedDomain(domain="example.org")) + db.session.commit() + assert rejection_reason("https://blog.example.org/post") is None + assert ( + rejection_reason("https://notexample.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(BlockedDomain(domain="blog.example.org")) + db.session.commit() + assert ( + rejection_reason("https://blog.example.org/post") + == "Source domain is blocked" + ) + assert rejection_reason("https://example.org/post") is None diff --git a/tests/test_source_policy.py b/tests/test_source_policy.py deleted file mode 100644 index cddfa3a..0000000 --- a/tests/test_source_policy.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Domain normalization and receiver admission policy.""" - -import pytest -from flask import Flask - -from webmentions_ssg import db -from webmentions_ssg.models import AllowedSource, BlockedSource -from webmentions_ssg.source_policy import ( - source_domain_from_url, - source_rejection_reason, -) - - -def test_source_domain_from_url() -> None: - assert ( - source_domain_from_url("https://BÜCHER.example.:8443/post") - == "xn--bcher-kva.example" - ) - assert source_domain_from_url("https://[2001:db8::1]/post") == "2001:db8::1" - assert source_domain_from_url("invalid") is None - - -def test_blocked_domain_matches_itself_and_subdomains_but_not_suffixes( - app: Flask, -) -> None: - with app.app_context(): - db.session.add(BlockedSource(domain="example.org")) - db.session.commit() - for source in ( - "https://example.org/post", - "https://blog.example.org/post", - "https://A.blog.Example.ORG./post", - ): - assert source_rejection_reason(source) == "Source domain is blocked" - for source in ( - "https://notexample.org/post", - "https://example.org.evil/post", - "https://other.net/post", - ): - assert source_rejection_reason(source) is None - - -def test_allowlist_is_fail_closed_and_blocklist_takes_precedence(app: Flask) -> None: - app.config["SOURCE_ALLOWLIST_ENABLED"] = True - with app.app_context(): - assert ( - source_rejection_reason("https://example.org/post") - == "Source domain is not allowlisted" - ) - db.session.add(AllowedSource(domain="example.org")) - db.session.commit() - assert source_rejection_reason("https://blog.example.org/post") is None - assert ( - source_rejection_reason("https://notexample.org/post") - == "Source domain is not allowlisted" - ) - db.session.add(BlockedSource(domain="blog.example.org")) - db.session.commit() - assert ( - source_rejection_reason("https://blog.example.org/post") - == "Source domain is blocked" - ) - assert source_rejection_reason("https://example.org/post") is None diff --git a/tests/views/helpers.py b/tests/views/helpers.py index 354102b..2bea73e 100644 --- a/tests/views/helpers.py +++ b/tests/views/helpers.py @@ -13,13 +13,13 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ( - ReceivedWebmention, +from webmentions_ssg.models.received import ReceivedWebmention +from webmentions_ssg.models.sources import ( SentWebmention, SentWebmentionStatus, SourceDocument, - User, ) +from webmentions_ssg.models.user import User SOURCE_URL = "https://source.example/post" TARGET_URL = "https://dennisfink.me/blog/example/" @@ -41,7 +41,7 @@ def mock_endpoint_form( form.errors = errors or {} monkeypatch.setattr( - views_module.forms, "WebmentionSubmissionForm", Mock(return_value=form) + views_module, "WebmentionSubmissionForm", Mock(return_value=form) ) return form @@ -118,7 +118,7 @@ def mock_login_form( form.username.data = username form.password.data = password - monkeypatch.setattr(views_module.forms, "LoginForm", Mock(return_value=form)) + monkeypatch.setattr(views_module, "LoginForm", Mock(return_value=form)) return form @@ -128,9 +128,7 @@ def mock_admin_action_form( form = Mock() form.validate_on_submit.return_value = valid - monkeypatch.setattr( - views_module.forms, "ConfirmActionForm", Mock(return_value=form) - ) + monkeypatch.setattr(views_module, "ConfirmActionForm", Mock(return_value=form)) return form diff --git a/tests/views/test_endpoint.py b/tests/views/test_endpoint.py index 88b665f..23643d9 100644 --- a/tests/views/test_endpoint.py +++ b/tests/views/test_endpoint.py @@ -15,7 +15,7 @@ from flask.testing import FlaskClient from sqlalchemy.exc import IntegrityError from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models.received import ReceivedWebmention from .helpers import ( SOURCE_URL, diff --git a/tests/views/test_source_domains.py b/tests/views/test_moderation.py index c9e72b2..7f683d4 100644 --- a/tests/views/test_source_domains.py +++ b/tests/views/test_moderation.py @@ -15,12 +15,9 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ( - AllowedSource, - BlockedSource, - ReceivedWebmention, - ReceivedWebmentionStatus, -) +from webmentions_ssg.models.moderation import AllowedDomain, BlockedDomain +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.moderation_policy import rejection_reason from webmentions_ssg.tasks.verification import VerificationClaim from .helpers import SOURCE_URL, TARGET_URL, create_received_webmention, log_in @@ -28,9 +25,9 @@ from .helpers import SOURCE_URL, TARGET_URL, create_received_webmention, log_in def test_source_rules_require_login(client: FlaskClient) -> None: for method, path, data in ( - ("get", "/source-domains", None), - ("post", "/source-domains/blocked/add", {"domain": "bad.example"}), - ("post", "/source-domains/blocked/1/delete", None), + ("get", "/moderation", None), + ("post", "/moderation/blocked/add", {"domain": "bad.example"}), + ("post", "/moderation/blocked/1/delete", None), ): response = client.open(path, method=method.upper(), data=data) assert response.status_code == HTTPStatus.FOUND @@ -39,31 +36,29 @@ def test_source_rules_require_login(client: FlaskClient) -> None: def test_add_and_unblock_source_domain(app: Flask, client: FlaskClient) -> None: log_in(app, client) - response = client.post( - "/source-domains/blocked/add", data={"domain": "SPAM.Example."} - ) + response = client.post("/moderation/blocked/add", data={"domain": "SPAM.Example."}) assert response.status_code == HTTPStatus.FOUND - response = client.post( - "/source-domains/blocked/add", data={"domain": "spam.example"} - ) + assert response.headers["Location"].endswith("/moderation") + response = client.post("/moderation/blocked/add", data={"domain": "spam.example"}) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - rows = db.session.scalars(sa.select(BlockedSource)).all() + rows = db.session.scalars(sa.select(BlockedDomain)).all() assert len(rows) == 1 assert rows[0].domain == "spam.example" entry_id = rows[0].id - response = client.get("/source-domains") + response = client.get("/moderation") assert response.status_code == HTTPStatus.OK assert b"spam.example" in response.data assert b"Unblock" in response.data assert b"Source rules" in response.data - response = client.post(f"/source-domains/blocked/{entry_id}/delete") + response = client.post(f"/moderation/blocked/{entry_id}/delete") assert response.status_code == HTTPStatus.FOUND + assert response.headers["Location"].endswith("/moderation") with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_allowlist_can_be_managed_while_mode_is_off( @@ -71,14 +66,14 @@ def test_allowlist_can_be_managed_while_mode_is_off( ) -> None: log_in(app, client) response = client.post( - "/source-domains/allowed/add", data={"domain": "friends.example"} + "/moderation/allowed/add", data={"domain": "friends.example"} ) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - row = db.session.scalar(sa.select(AllowedSource)) + row = db.session.scalar(sa.select(AllowedDomain)) assert row is not None assert row.domain == "friends.example" - assert b"disabled" in client.get("/source-domains").data + assert b"disabled" in client.get("/moderation").data @pytest.mark.parametrize( @@ -88,17 +83,17 @@ def test_invalid_domains_are_rejected( client: FlaskClient, app: Flask, domain: str ) -> None: log_in(app, client) - response = client.post("/source-domains/blocked/add", data={"domain": domain}) + response = client.post("/moderation/blocked/add", data={"domain": domain}) assert response.status_code == HTTPStatus.BAD_REQUEST def test_invalid_list_kind_and_missing_entry(app: Flask, client: FlaskClient) -> None: log_in(app, client) assert ( - client.post("/source-domains/nope/add", data={"domain": "test.org"}).status_code + client.post("/moderation/nope/add", data={"domain": "test.org"}).status_code == 404 ) - assert client.post("/source-domains/blocked/123/delete").status_code == 404 + assert client.post("/moderation/blocked/123/delete").status_code == 404 def test_received_row_can_block_domain(app: Flask, client: FlaskClient) -> None: @@ -109,7 +104,7 @@ def test_received_row_can_block_domain(app: Flask, client: FlaskClient) -> None: response = client.post(f"/received/{mention_id}/block-source") assert response.status_code == HTTPStatus.FOUND with app.app_context(): - rule = db.session.scalar(sa.select(BlockedSource)) + rule = db.session.scalar(sa.select(BlockedDomain)) assert rule is not None and rule.domain == "spam.example" @@ -126,7 +121,7 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( second = create_received_webmention(app, source="https://second.example/article") unselected = create_received_webmention(app, source="https://other.example/article") with app.app_context(): - db.session.add(BlockedSource(domain="second.example")) + db.session.add(BlockedDomain(domain="second.example")) db.session.commit() response = client.post( @@ -142,19 +137,17 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( "sort": ["source"], } with app.app_context(): - assert set(db.session.scalars(sa.select(BlockedSource.domain))) == { + assert set(db.session.scalars(sa.select(BlockedDomain.domain))) == { "spam.example", "second.example", } for identifier in (first, same_domain, second, unselected): assert db.session.get(ReceivedWebmention, identifier) is not None - from webmentions_ssg.source_policy import source_rejection_reason - assert ( - source_rejection_reason("https://child.spam.example/post") + rejection_reason("https://child.spam.example/post") == "Source domain is blocked" ) - assert source_rejection_reason("https://other.example/post") is None + assert rejection_reason("https://other.example/post") is None # Repeating the action is idempotent. response = client.post( @@ -163,7 +156,7 @@ def test_block_selected_source_domains_deduplicates_and_preserves_mentions( ) assert response.status_code == HTTPStatus.FOUND with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 2 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 2 def test_bulk_block_rejects_invalid_source_without_partial_changes( @@ -179,7 +172,7 @@ def test_bulk_block_rejects_invalid_source_without_partial_changes( ) assert response.status_code == HTTPStatus.BAD_REQUEST with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_bulk_block_action_enforces_csrf(app: Flask, client: FlaskClient) -> None: @@ -192,7 +185,7 @@ def test_bulk_block_action_enforces_csrf(app: Flask, client: FlaskClient) -> Non ) assert response.status_code == HTTPStatus.BAD_REQUEST with app.app_context(): - assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + assert db.session.scalar(sa.select(sa.func.count(BlockedDomain.id))) == 0 def test_endpoint_rejects_blocked_without_queue_or_insert( @@ -203,7 +196,7 @@ def test_endpoint_rejects_blocked_without_queue_or_insert( enqueue = Mock() monkeypatch.setattr(endpoint, "verify_webmention", enqueue) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() response = client.post( @@ -221,14 +214,14 @@ def test_allowlist_mode_rejects_unlisted_and_allows_listed( ) -> None: from webmentions_ssg.views import endpoint - app.config["SOURCE_ALLOWLIST_ENABLED"] = True + app.config["ALLOWLIST_ENABLED"] = True enqueue = Mock() monkeypatch.setattr(endpoint, "verify_webmention", enqueue) data = {"source": SOURCE_URL, "target": TARGET_URL} assert client.post("/endpoint", data=data).status_code == HTTPStatus.FORBIDDEN with app.app_context(): - db.session.add(AllowedSource(domain="source.example")) + db.session.add(AllowedDomain(domain="source.example")) db.session.commit() assert client.post("/endpoint", data=data).status_code == HTTPStatus.CREATED @@ -249,7 +242,7 @@ def test_worker_checks_rules_before_fetch( fetch = Mock() monkeypatch.setattr(verification, "source_mentions_target", fetch) with app.app_context(): - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() assert verification.determine_verification_outcome(claim) == ( ReceivedWebmentionStatus.FAILED, @@ -272,7 +265,7 @@ def test_queued_worker_skips_fetch_after_source_is_blocked( identifier = mention.uuid # A user blocks the domain after the submission has already been queued. - db.session.add(BlockedSource(domain="source.example")) + db.session.add(BlockedDomain(domain="source.example")) db.session.commit() fetch = Mock() @@ -291,7 +284,7 @@ def test_admin_actions_enforce_csrf(app: Flask, client: FlaskClient) -> None: app.config["WTF_CSRF_ENABLED"] = True assert ( client.post( - "/source-domains/blocked/add", data={"domain": "spam.example"} + "/moderation/blocked/add", data={"domain": "spam.example"} ).status_code == HTTPStatus.BAD_REQUEST ) diff --git a/tests/views/test_admin.py b/tests/views/test_received.py index 175bcfd..ec00e05 100644 --- a/tests/views/test_admin.py +++ b/tests/views/test_received.py @@ -17,15 +17,12 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention, SentWebmention, User +from webmentions_ssg.models.received import ReceivedWebmention from .helpers import ( SOURCE_URL, TARGET_URL, - create_outgoing, create_received_webmention, - create_resendable_sent_webmention, - create_sent_source, log_in, mock_admin_action_form, ) @@ -34,66 +31,11 @@ from .helpers import ( @pytest.fixture def views_module(app: Flask) -> ModuleType: _ = app - from webmentions_ssg.views import admin as module + from webmentions_ssg.views import received as module return module -@pytest.mark.parametrize("path", ["/sources", f"/sources/{uuid.uuid7()}"]) -def test_sent_views_require_login(client: FlaskClient, path: str) -> None: - response = client.get(path) - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get("/sources") - - assert response.status_code == HTTPStatus.OK - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - assert f"/sources/{identifier}" in html - - -def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: - log_in(app, client) - identifier = create_sent_source(app) - - response = client.get(f"/sources/{identifier}") - - assert response.status_code == HTTPStatus.OK - - html = response.get_data(as_text=True) - - assert "blog/example/index.html" in html - assert "https://dennisfink.me/blog/example/" in html - - first_target = "https://example.com/a" - second_target = "https://example.com/b" - - assert first_target in html - assert second_target in html - assert html.index(first_target) < html.index(second_target) - - assert "Webmention endpoint returned HTTP 400" in html - assert "https://example.com/webmention" in html - - -def test_sent_source_returns_404_for_unknown_source( - app: Flask, client: FlaskClient -) -> None: - log_in(app, client) - response = client.get(f"/sources/{uuid.uuid7()}") - assert response.status_code == HTTPStatus.NOT_FOUND - - def test_received_requires_login(client: FlaskClient) -> None: response = client.get("/received") @@ -471,7 +413,13 @@ def test_received_filters_are_combined(app: Flask, client: FlaskClient) -> None: ) response = client.get( - "/received?status=declined&q=match.example&date_from=2026-11-08&date_to=2026-11-14" + "/received", + query_string={ + "status": "declined", + "q": "match.example", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + }, ) html = response.get_data(as_text=True) @@ -1113,168 +1061,3 @@ def test_received_bulk_delete_requires_real_csrf_token( assert valid.status_code == HTTPStatus.FOUND with app.app_context(): assert db.session.get(ReceivedWebmention, identifier) is None - - -def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: - response = client.post("/sources/rescan") - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -@pytest.mark.parametrize("valid", [False, True]) -def test_rescan_sent_sources( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, - valid: bool, -) -> None: - log_in(app, client) - form = mock_admin_action_form(views_module, monkeypatch, valid=valid) - manual_scan_sources = Mock() - monkeypatch.setattr(views_module, "manual_scan_sources", manual_scan_sources) - - response = client.post("/sources/rescan") - - assert response.status_code == ( - HTTPStatus.FOUND if valid else HTTPStatus.BAD_REQUEST - ) - form.validate_on_submit.assert_called_once_with() - - if valid: - assert response.headers["Location"].endswith("/sources") - manual_scan_sources.assert_called_once_with() - else: - manual_scan_sources.assert_not_called() - - -def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: - response = client.post(f"/sources/{uuid.uuid7()}/{uuid.uuid7()}/resend") - - assert response.status_code == HTTPStatus.FOUND - assert "/login" in response.headers["Location"] - - -def test_resend_sent_webmention_rejects_invalid_form( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=False) - - response = client.post( - f"/sources/{source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.BAD_REQUEST - - -def test_resend_sent_webmention_returns_404_for_unknown_webmention( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post(f"/sources/{source_identifier}/{uuid.uuid7()}/resend") - - assert response.status_code == HTTPStatus.NOT_FOUND - - -def test_resend_sent_webmention_rejects_wrong_source( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - _, webmention_identifier = create_resendable_sent_webmention(app) - wrong_source_identifier = create_sent_source(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - - response = client.post( - f"/sources/{wrong_source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.NOT_FOUND - - -def test_resend_sent_webmention_reopens_current_revision( - app: Flask, - client: FlaskClient, - views_module: ModuleType, - monkeypatch: pytest.MonkeyPatch, -) -> None: - log_in(app, client) - source_identifier, webmention_identifier = create_resendable_sent_webmention(app) - mock_admin_action_form(views_module, monkeypatch, valid=True) - send_webmention = Mock() - monkeypatch.setattr(views_module, "send_webmention", send_webmention) - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - webmention.attempt_count = 6 - webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) - db.session.commit() - - response = client.post( - f"/sources/{source_identifier}/{webmention_identifier}/resend" - ) - - assert response.status_code == HTTPStatus.FOUND - assert response.headers["Location"].endswith(f"/sources/{source_identifier}") - - with app.app_context(): - webmention = db.session.get(SentWebmention, webmention_identifier) - assert webmention is not None - assert webmention.desired_revision == 3 - assert webmention.processed_revision is None - assert webmention.attempt_count == 0 - assert webmention.next_retry_at is None - assert webmention.sent_revision == 2 - assert webmention.status is None - assert webmention.failure_reason is None - assert webmention.webmention_endpoint is None - assert webmention.endpoint_response_status is None - assert webmention.status_url is None - - send_webmention.assert_called_once_with(webmention_identifier) - - -@pytest.mark.parametrize( - ("status_url", "safe"), - [ - ("https://example.com/status", True), - ("http://example.com/status", True), - ("javascript:alert(1)", False), - ("data:text/html,evil", False), - ("ftp://example.com/status", False), - ], -) -def test_admin_only_links_safe_existing_status_urls( - app: Flask, client: FlaskClient, status_url: str, safe: bool -) -> None: - source_identifier, _ = create_outgoing(app, status_url=status_url) - with app.app_context(): - user = User(username="admin", password="test") - db.session.add(user) - db.session.commit() - user_id = user.get_id() - - with client.session_transaction() as session: - session["_user_id"] = str(user_id) - session["_fresh"] = True - - response = client.get(f"/sources/{source_identifier}") - assert response.status_code == HTTPStatus.OK - html = response.get_data(as_text=True) - assert (f'href="{status_url}"' in html) is safe - assert status_url in html diff --git a/tests/views/test_session.py b/tests/views/test_session.py index a2d3268..7edee71 100644 --- a/tests/views/test_session.py +++ b/tests/views/test_session.py @@ -10,7 +10,7 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models import User +from webmentions_ssg.models.user import User from .helpers import log_in, mock_login_form diff --git a/tests/views/test_sources.py b/tests/views/test_sources.py new file mode 100644 index 0000000..012347e --- /dev/null +++ b/tests/views/test_sources.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +import uuid +from datetime import UTC, datetime, timedelta +from http import HTTPStatus +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import db +from webmentions_ssg.models.sources import SentWebmention +from webmentions_ssg.models.user import User + +from .helpers import ( + create_outgoing, + create_resendable_sent_webmention, + create_sent_source, + log_in, + mock_admin_action_form, +) + + +@pytest.fixture +def sources_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.views import sources as module + + return module + + +@pytest.mark.parametrize("path", ["/sources", f"/sources/{uuid.uuid7()}"]) +def test_sent_views_require_login(client: FlaskClient, path: str) -> None: + response = client.get(path) + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get("/sources") + + assert response.status_code == HTTPStatus.OK + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + assert f"/sources/{identifier}" in html + + +def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get(f"/sources/{identifier}") + + assert response.status_code == HTTPStatus.OK + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + + first_target = "https://example.com/a" + second_target = "https://example.com/b" + + assert first_target in html + assert second_target in html + assert html.index(first_target) < html.index(second_target) + + assert "Webmention endpoint returned HTTP 400" in html + assert "https://example.com/webmention" in html + + +def test_sent_source_returns_404_for_unknown_source( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.get(f"/sources/{uuid.uuid7()}") + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_rescan_sent_sources_requires_login(client: FlaskClient) -> None: + response = client.post("/sources/rescan") + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +@pytest.mark.parametrize("valid", [False, True]) +def test_rescan_sent_sources( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + valid: bool, +) -> None: + log_in(app, client) + form = mock_admin_action_form(sources_module, monkeypatch, valid=valid) + manual_scan_sources = Mock() + monkeypatch.setattr(sources_module, "manual_scan_sources", manual_scan_sources) + + response = client.post("/sources/rescan") + + assert response.status_code == ( + HTTPStatus.FOUND if valid else HTTPStatus.BAD_REQUEST + ) + form.validate_on_submit.assert_called_once_with() + + if valid: + assert response.headers["Location"].endswith("/sources") + manual_scan_sources.assert_called_once_with() + else: + manual_scan_sources.assert_not_called() + + +def test_resend_sent_webmention_requires_login(client: FlaskClient) -> None: + response = client.post(f"/sources/{uuid.uuid7()}/{uuid.uuid7()}/resend") + + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +def test_resend_sent_webmention_rejects_invalid_form( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=False) + + response = client.post( + f"/sources/{source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_resend_sent_webmention_returns_404_for_unknown_webmention( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier = create_sent_source(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + + response = client.post(f"/sources/{source_identifier}/{uuid.uuid7()}/resend") + + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_resend_sent_webmention_rejects_wrong_source( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + _, webmention_identifier = create_resendable_sent_webmention(app) + wrong_source_identifier = create_sent_source(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + + response = client.post( + f"/sources/{wrong_source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_resend_sent_webmention_reopens_current_revision( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_identifier, webmention_identifier = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + send_webmention = Mock() + monkeypatch.setattr(sources_module, "send_webmention", send_webmention) + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + webmention.attempt_count = 6 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) + db.session.commit() + + response = client.post( + f"/sources/{source_identifier}/{webmention_identifier}/resend" + ) + + assert response.status_code == HTTPStatus.FOUND + assert response.headers["Location"].endswith(f"/sources/{source_identifier}") + + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + assert webmention.desired_revision == 3 + assert webmention.processed_revision is None + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None + assert webmention.sent_revision == 2 + assert webmention.status is None + assert webmention.failure_reason is None + assert webmention.webmention_endpoint is None + assert webmention.endpoint_response_status is None + assert webmention.status_url is None + + send_webmention.assert_called_once_with(webmention_identifier) + + +@pytest.mark.parametrize( + ("status_url", "safe"), + [ + ("https://example.com/status", True), + ("http://example.com/status", True), + ("javascript:alert(1)", False), + ("data:text/html,evil", False), + ("ftp://example.com/status", False), + ], +) +def test_admin_only_links_safe_existing_status_urls( + app: Flask, client: FlaskClient, status_url: str, safe: bool +) -> None: + source_identifier, _ = create_outgoing(app, status_url=status_url) + with app.app_context(): + user = User(username="admin", password="test") + db.session.add(user) + db.session.commit() + user_id = user.get_id() + + with client.session_transaction() as session: + session["_user_id"] = str(user_id) + session["_fresh"] = True + + response = client.get(f"/sources/{source_identifier}") + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert (f'href="{status_url}"' in html) is safe + assert status_url in html diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index ce4960d..ac1f4e7 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -16,7 +16,7 @@ from flask_login import LoginManager from werkzeug.security import generate_password_hash from .extensions import db -from .models import User +from .models.user import User from .validation import is_control_character login_manager = LoginManager() diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index b26b35b..707402d 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -15,7 +15,7 @@ from sqlalchemy.exc import IntegrityError from .auth import validate_new_password from .extensions import db, mail -from .models import User +from .models.user import User USERS_CLI = AppGroup("users", help="Manage users.") MAIL_CLI = AppGroup("mail", help="Handle mail") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 576f376..28ab43d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -54,9 +54,9 @@ class DefaultConfig: # Receiver # Empty by default: no target hostnames are accepted until configured. ALLOWED_TARGET_HOSTNAMES = frozenset() - # When enabled, only source domains listed in AllowedSource are accepted. - # Entries in BlockedSource always take precedence. - SOURCE_ALLOWLIST_ENABLED = False + # When enabled, only source domains listed in AllowedDomain are accepted. + # Entries in BlockedDomain always take precedence. + ALLOWLIST_ENABLED = False # Cap incoming HTTP request bodies (also applies to other Flask routes). MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. @@ -213,8 +213,8 @@ def validated_config_overrides( except (TypeError, ValueError) as exc: raise type(exc)(f"{key}: {exc}") from exc - if type(config["SOURCE_ALLOWLIST_ENABLED"]) is not bool: - raise ValueError("SOURCE_ALLOWLIST_ENABLED must be a boolean.") + if type(config["ALLOWLIST_ENABLED"]) is not bool: + raise ValueError("ALLOWLIST_ENABLED must be a boolean.") proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] if type(proxy_hops) is not int or proxy_hops < 0: diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py deleted file mode 100644 index 3b61a8a..0000000 --- a/webmentions_ssg/forms.py +++ /dev/null @@ -1,268 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Webmention forms and URL validation.""" - -import ipaddress - -import httpx -from flask import current_app -from flask_wtf import FlaskForm -from wtforms import ( - DateField, - Field, - HiddenField, - IntegerField, - PasswordField, - SearchField, - SelectField, - StringField, - SubmitField, - ValidationError, -) -from wtforms.form import BaseForm -from wtforms.validators import ( - AnyOf, - InputRequired, - NumberRange, - Optional, - StopValidation, -) - -from .hostname import normalize_hostname -from .net import is_public_address -from .validation import is_control_character - -RECEIVED_STATUS_CHOICES = { - "": "All statuses", - "pending": "Pending", - "verifying": "Verifying", - "verified": "Verified", - "declined": "Declined", -} - - -class MaxURLBytes: - """Check the UTF-8 byte length of a submitted URL using the current config.""" - - def __init__( - self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None - ) -> None: - self.config_key = config_key - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - if field.data is None: - return - - max_bytes = current_app.config[self.config_key] - - try: - length = len(field.data.encode("utf-8")) - except UnicodeEncodeError as exc: - raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc - - if length > max_bytes: - raise ValidationError( - self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") - ) - - -class StrictHTTPURL: - """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. - - httpx automatically percent-encodes some whitespace. Reject it instead of - silently changing the submitted URL (which may be used in exact URL matches). - """ - - def __call__(self, form: BaseForm, field: Field) -> None: - value = field.data - if not isinstance(value, str): - raise StopValidation(field.gettext("Invalid URL.")) - - if any( - character.isspace() or is_control_character(character) - for character in value - ): - raise StopValidation(field.gettext("Invalid URL.")) - - try: - parsed = httpx.URL(value) - except (httpx.InvalidURL, ValueError, UnicodeError) as exc: - raise StopValidation(field.gettext("Invalid URL.")) from exc - - if not parsed.host: - raise StopValidation(field.gettext("Invalid URL.")) - - if parsed.scheme not in ("http", "https"): - raise StopValidation( - field.gettext(f"{field.name} must begin with http or https") - ) - - -class AllowedTargetHostname: - """Validate that a URL uses an allowed hostname. - - :param message: Optional validation error message. - """ - - def __init__(self, message: str | None = None) -> None: - """Initialize the validator. - - :param message: Optional validation error message. - """ - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - """Validate that a URL uses an allowed hostname. - - :param form: Form containing the field. - :param field: Field containing the URL to validate. - :raises ValidationError: If the URL hostname is not configured as allowed. - """ - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError: - hostname = None - - if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: - return - raise ValidationError(self.message or field.gettext("Invalid input.")) - - -class RejectNonPublicIPLiteral: - """Reject non-public IP literals without performing DNS in the web worker.""" - - def __call__(self, form: BaseForm, field: Field) -> None: - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError as exc: - raise ValidationError(field.gettext("Invalid input.")) from exc - - try: - address = ipaddress.ip_address(hostname) - except ValueError: - return # Named hosts are resolved only during the background fetch. - - if not is_public_address(address): - raise ValidationError(field.gettext("URL must use a public address.")) - - -class LoginForm(FlaskForm): - """Form for authenticating a user.""" - - username = StringField("Username", validators=[InputRequired()]) - password = PasswordField("Password", validators=[InputRequired()]) - submit = SubmitField("Sign In") - - -class WebmentionSubmissionForm(FlaskForm): - """Form for receiving a Webmention. - - The source must be a public HTTP or HTTPS URL distinct from the target. - The target must be an HTTP or HTTPS URL using an allowed hostname. - """ - - source = StringField( - "source", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - RejectNonPublicIPLiteral(), - ], - ) - target = StringField( - "target", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - AllowedTargetHostname(), - ], - ) - - def validate_source(self, field: Field) -> None: - """Reject a source URL that equals the target URL. - - :param field: Source URL field being validated. - :raises ValidationError: If the source URL equals the target URL. - """ - if field.data == self.target.data: - raise ValidationError(field.gettext("Field must not be equal to target.")) - - -class CanonicalDate: - """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" - - def __call__(self, form: BaseForm, field: DateField) -> None: - if ( - field.data is not None - and field.raw_data - and field.raw_data[0] != field.data.isoformat() - ): - raise ValidationError(field.gettext("Invalid date.")) - - -def default_received_page_size() -> int: - """Use 25 rows unless the configured maximum is lower.""" - return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) - - -class ReceivedWebmentionFilterForm(FlaskForm): - """GET filters for the received Webmention list.""" - - status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") - q = SearchField("Source or target") - date_from = DateField( - "Updated from (UTC)", validators=[Optional(), CanonicalDate()] - ) - date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) - sort = HiddenField( - default="received", - validators=[AnyOf(("received", "updated", "status", "source", "target"))], - ) - order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) - per_page = IntegerField( - "Rows per page", - default=default_received_page_size, - validators=[NumberRange(min=1)], - ) - - def validate_per_page(self, field: IntegerField) -> None: - """Enforce the deployment's maximum received-list page size.""" - if ( - field.data is not None - and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] - ): - raise ValidationError("Rows per page exceeds the configured maximum.") - - def validate_date_to(self, field: DateField) -> None: - """Require the end date to be on or after the start date.""" - if ( - self.date_from.data is not None - and field.data is not None - and field.data < self.date_from.data - ): - raise ValidationError("End date must be on or after start date.") - - -class ConfirmActionForm(FlaskForm): - """Form for confirming an administrative action.""" - - submit = SubmitField("Confirm") - - -class SourceDomainForm(FlaskForm): - """Add a domain to the source blocklist or allowlist.""" - - domain = StringField("Source domain", validators=[InputRequired()]) - submit = SubmitField("Add domain") - - def validate_domain(self, field: Field) -> None: - """Require a bare hostname and persist its canonical form.""" - try: - field.data = normalize_hostname(field.data) - except ValueError as exc: - raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py new file mode 100644 index 0000000..5f61c0e --- /dev/null +++ b/webmentions_ssg/forms/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for webmention submission, authentication, filtering and moderation.""" diff --git a/webmentions_ssg/forms/admin.py b/webmentions_ssg/forms/admin.py new file mode 100644 index 0000000..97d523d --- /dev/null +++ b/webmentions_ssg/forms/admin.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Administrative action confirmation forms.""" + +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class ConfirmActionForm(FlaskForm): + """Form for confirming an administrative action.""" + + submit = SubmitField("Confirm") diff --git a/webmentions_ssg/forms/endpoint.py b/webmentions_ssg/forms/endpoint.py new file mode 100644 index 0000000..e063e73 --- /dev/null +++ b/webmentions_ssg/forms/endpoint.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Webmention submission form.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, ValidationError +from wtforms.validators import InputRequired + +from .validators import ( + AllowedTargetHostname, + MaxURLBytes, + RejectNonPublicIPLiteral, + StrictHTTPURL, +) + + +class WebmentionSubmissionForm(FlaskForm): + """Form for receiving a Webmention. + + The source must be a public HTTP or HTTPS URL distinct from the target. + The target must be an HTTP or HTTPS URL using an allowed hostname. + """ + + source = StringField( + "source", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + RejectNonPublicIPLiteral(), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + AllowedTargetHostname(), + ], + ) + + def validate_source(self, field: Field) -> None: + """Reject a source URL that equals the target URL. + + :param field: Source URL field being validated. + :raises ValidationError: If the source URL equals the target URL. + """ + if field.data == self.target.data: + raise ValidationError(field.gettext("Field must not be equal to target.")) diff --git a/webmentions_ssg/forms/moderation.py b/webmentions_ssg/forms/moderation.py new file mode 100644 index 0000000..5987d14 --- /dev/null +++ b/webmentions_ssg/forms/moderation.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for managing domain allowlists and blocklists.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, SubmitField, ValidationError +from wtforms.validators import InputRequired + +from ..hostname import normalize_hostname + + +class ModerateDomainForm(FlaskForm): + """Add a source domain to the blocklist or allowlist.""" + + domain = StringField("Source domain", validators=[InputRequired()]) + submit = SubmitField("Add domain") + + def validate_domain(self, field: Field) -> None: + """Require a bare hostname and persist its canonical form.""" + try: + field.data = normalize_hostname(field.data) + except ValueError as exc: + raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/received.py b/webmentions_ssg/forms/received.py new file mode 100644 index 0000000..c4a6ed0 --- /dev/null +++ b/webmentions_ssg/forms/received.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Received Webmention filtering form.""" + +from flask import current_app +from flask_wtf import FlaskForm +from wtforms import ( + DateField, + HiddenField, + IntegerField, + SearchField, + SelectField, + ValidationError, +) +from wtforms.validators import AnyOf, NumberRange, Optional + +from .validators import CanonicalDate + +RECEIVED_STATUS_CHOICES = { + "": "All statuses", + "pending": "Pending", + "verifying": "Verifying", + "verified": "Verified", + "declined": "Declined", +} + + +def default_received_page_size() -> int: + """Use 25 rows unless the configured maximum is lower.""" + return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) + + +class ReceivedWebmentionFilterForm(FlaskForm): + """GET filters for the received Webmention list.""" + + status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") + q = SearchField("Source or target") + date_from = DateField( + "Updated from (UTC)", validators=[Optional(), CanonicalDate()] + ) + date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) + sort = HiddenField( + default="received", + validators=[AnyOf(("received", "updated", "status", "source", "target"))], + ) + order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) + per_page = IntegerField( + "Rows per page", + default=default_received_page_size, + validators=[NumberRange(min=1)], + ) + + def validate_per_page(self, field: IntegerField) -> None: + """Enforce the deployment's maximum received-list page size.""" + if ( + field.data is not None + and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] + ): + raise ValidationError("Rows per page exceeds the configured maximum.") + + def validate_date_to(self, field: DateField) -> None: + """Require the end date to be on or after the start date.""" + if ( + self.date_from.data is not None + and field.data is not None + and field.data < self.date_from.data + ): + raise ValidationError("End date must be on or after start date.") diff --git a/webmentions_ssg/forms/user.py b/webmentions_ssg/forms/user.py new file mode 100644 index 0000000..3bb6421 --- /dev/null +++ b/webmentions_ssg/forms/user.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""User authentication forms.""" + +from flask_wtf import FlaskForm +from wtforms import PasswordField, StringField, SubmitField +from wtforms.validators import InputRequired + + +class LoginForm(FlaskForm): + """Form for authenticating a user.""" + + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py new file mode 100644 index 0000000..e65e9cb --- /dev/null +++ b/webmentions_ssg/forms/validators.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Custom Webmention URL and date field validators.""" + +import ipaddress + +import httpx +from flask import current_app +from wtforms import DateField, Field, ValidationError +from wtforms.form import BaseForm +from wtforms.validators import StopValidation + +from ..hostname import normalize_hostname +from ..net import is_public_address +from ..validation import is_control_character + + +class MaxURLBytes: + """Check the UTF-8 byte length of a submitted URL using the current config.""" + + def __init__( + self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None + ) -> None: + self.config_key = config_key + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + if field.data is None: + return + + max_bytes = current_app.config[self.config_key] + + try: + length = len(field.data.encode("utf-8")) + except UnicodeEncodeError as exc: + raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc + + if length > max_bytes: + raise ValidationError( + self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") + ) + + +class StrictHTTPURL: + """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or is_control_character(character) + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + if parsed.scheme not in ("http", "https"): + raise StopValidation( + field.gettext(f"{field.name} must begin with http or https") + ) + + +class AllowedTargetHostname: + """Validate that a URL uses an allowed hostname. + + :param message: Optional validation error message. + """ + + def __init__(self, message: str | None = None) -> None: + """Initialize the validator. + + :param message: Optional validation error message. + """ + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + """Validate that a URL uses an allowed hostname. + + :param form: Form containing the field. + :param field: Field containing the URL to validate. + :raises ValidationError: If the URL hostname is not configured as allowed. + """ + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError: + hostname = None + + if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: + return + raise ValidationError(self.message or field.gettext("Invalid input.")) + + +class RejectNonPublicIPLiteral: + """Reject non-public IP literals without performing DNS in the web worker.""" + + def __call__(self, form: BaseForm, field: Field) -> None: + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError as exc: + raise ValidationError(field.gettext("Invalid input.")) from exc + + try: + address = ipaddress.ip_address(hostname) + except ValueError: + return # Named hosts are resolved only during the background fetch. + + if not is_public_address(address): + raise ValidationError(field.gettext("URL must use a public address.")) + + +class CanonicalDate: + """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" + + def __call__(self, form: BaseForm, field: DateField) -> None: + if ( + field.data is not None + and field.raw_data + and field.raw_data[0] != field.data.isoformat() + ): + raise ValidationError(field.gettext("Invalid date.")) diff --git a/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py new file mode 100644 index 0000000..e66dc70 --- /dev/null +++ b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rename source block and allow lists to domain lists. + +Revision ID: a4f91c2e08d7 +Revises: b8179a2e46c3 +""" + +from alembic import op + +revision = "a4f91c2e08d7" +down_revision = "b8179a2e46c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Preserve existing domain rules when renaming their tables.""" + op.rename_table("blocked_sources", "blocked_domains") + op.rename_table("allowed_sources", "allowed_domains") + + +def downgrade() -> None: + """Restore the former table names without deleting domain rules.""" + op.rename_table("allowed_domains", "allowed_sources") + op.rename_table("blocked_domains", "blocked_sources") diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 8518b15..2917cb8 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,8 +11,8 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.models import source_target_digest as received_url_digest -from webmentions_ssg.models import target_digest as sent_target_digest +from webmentions_ssg.models.utils import source_target_digest as received_url_digest +from webmentions_ssg.models.utils import target_digest as sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py deleted file mode 100644 index 2304788..0000000 --- a/webmentions_ssg/models.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Define database models for users and incoming and outgoing Webmentions.""" - -import json -from datetime import UTC, datetime -from enum import StrEnum -from hashlib import sha256 -from typing import Never -from uuid import UUID, uuid7 - -import sqlalchemy as sa -from flask_login import UserMixin -from sqlalchemy import ( - Boolean, - DateTime, - Enum, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - Uuid, -) -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Mapped, mapped_column, relationship, validates -from werkzeug.security import check_password_hash, generate_password_hash - -from .extensions import Base - - -class ReceivedWebmentionStatus(StrEnum): - """Persisted lifecycle state of an incoming Webmention.""" - - RECEIVED = "received" - VERIFYING = "verifying" - VERIFIED = "verified" - FAILED = "failed" - SOURCE_REMOVED = "source_removed" - - -class SentWebmentionStatus(StrEnum): - """Represent the outcome of a Webmention send attempt.""" - - SENT = "sent" - NO_ENDPOINT = "no_endpoint" - FAILED = "failed" - - -class User(UserMixin, Base): - """Represent an authenticated application user.""" - - __tablename__ = "users" - - id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column(Text(), index=True, unique=True) - password_hash: Mapped[str] = mapped_column(Text(), nullable=False) - - def __repr__(self) -> str: - """Represent the user by username.""" - return f"<User {self.username}>" - - @property - def password(self) -> Never: - """Prevent access to the user's plain-text password. - - :raises AttributeError: Always, because the password is write-only. - """ - raise AttributeError("Password is write-only") - - @password.setter - def password(self, password: str) -> None: - """Hash and store a new password. - - :param password: Plain-text password to hash. - """ - self.password_hash = generate_password_hash(password) - - def get_id(self) -> str: - """Bind the login session to the current stored password hash. - - Only a one-way fingerprint is placed in Flask's signed cookie, never - the password hash itself. Rehashing the password invalidates all - previously issued session identifiers without a schema change. - - :returns: Session identifier combining the user ID and password-hash - fingerprint. - """ - fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() - return f"{self.id}:{fingerprint}" - - def check_password(self, password: str) -> bool: - """Check a plain-text password against the stored password hash. - - :param password: Plain-text password to check. - :returns: Whether the password matches the stored hash. - """ - return check_password_hash(self.password_hash, password) - - -class BlockedSource(Base): - """A source domain denied before verification or outbound fetching.""" - - __tablename__ = "blocked_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class AllowedSource(Base): - """A source domain accepted when allowlist mode is enabled.""" - - __tablename__ = "allowed_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class ReceivedWebmention(Base): - """Represent a Webmention received for verification.""" - - __tablename__ = "received_webmentions" - - uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) - - source: Mapped[str] = mapped_column(Text(), nullable=False) - target: Mapped[str] = mapped_column(Text(), nullable=False) - source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("source", "target") - def _refresh_url_digest(self, key: str, value: str) -> str: - source = value if key == "source" else self.__dict__.get("source") - target = value if key == "target" else self.__dict__.get("target") - if source is not None and target is not None: - self.source_target_digest = source_target_digest(source, target) - return value - - status: Mapped[ReceivedWebmentionStatus] = mapped_column( - Enum( - ReceivedWebmentionStatus, - name="received_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=False, - default=ReceivedWebmentionStatus.RECEIVED, - ) - # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh - # explicit submission resets this counter. - recovery_attempts: Mapped[int] = mapped_column( - Integer, nullable=False, default=0, server_default="0" - ) - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - default=lambda: datetime.now(UTC), - onupdate=lambda: datetime.now(UTC), - nullable=False, - ) - - notification_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - __table_args__ = ( - UniqueConstraint( - "source_target_digest", name="uq_webmention_source_target_digest" - ), - Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), - ) - - def __repr__(self) -> str: - """Represent the received Webmention by source and target.""" - return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" - - @staticmethod - def reopen_verification_values( - *, at: datetime, recovery_attempts: int = 0 - ) -> dict[str, object]: - """Return the columns to reset when verification is explicitly reopened. - - :param at: Timestamp to record for the newly opened verification. - :param recovery_attempts: Number of stale-lease recovery attempts to retain. - :returns: Column values for resetting the verification state. - """ - return { - "status": ReceivedWebmentionStatus.RECEIVED, - "failure_reason": None, - "recovery_attempts": recovery_attempts, - "updated_at": at, - } - - def reopen_verification(self) -> None: - """Reopen an existing row for manual verification. - - Clear the previous outcome and restore the received state with a fresh - timestamp. - """ - for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): - setattr(self, key, value) - - @property - def verified(self) -> bool: - """Return whether the Webmention has been successfully verified.""" - return self.status == ReceivedWebmentionStatus.VERIFIED - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SourceDocument(Base): - """Represent a generated source document containing outgoing Webmentions.""" - - __tablename__ = "source_documents" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - content_hash: Mapped[str] = mapped_column(String(32), nullable=False) - revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) - - last_seen_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - revised_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - deleted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - sent_webmentions: Mapped[list[SentWebmention]] = relationship( - back_populates="source", cascade="all, delete-orphan" - ) - - def __repr__(self) -> str: - """Represent the source by its public URL.""" - return f"<SourceDocument {self.url!r}>" - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the source UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SentWebmention(Base): - """Represent an outgoing Webmention associated with a source.""" - - __tablename__ = "sent_webmentions" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - source_uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), - ForeignKey("source_documents.uuid", ondelete="CASCADE"), - nullable=False, - index=True, - ) - - target: Mapped[str] = mapped_column(Text(), nullable=False) - target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("target") - def _update_target_digest(self, key: str, value: str) -> str: - self.target_digest = target_digest(value) - return value - - linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) - - desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) - processed_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - sent_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - - status: Mapped[SentWebmentionStatus | None] = mapped_column( - Enum( - SentWebmentionStatus, - name="sent_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=True, - default=None, - ) - - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - webmention_endpoint: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - endpoint_response_status: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) - - # Retry state belongs to the desired revision, not to the target forever. - attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) - next_retry_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - last_attempted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - last_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") - - __table_args__ = ( - UniqueConstraint( - "source_uuid", - "target_digest", - name="uq_sent_webmention_source_uuid_target_digest", - ), - ) - - def __repr__(self) -> str: - """Represent the sent Webmention by its target URL.""" - return f"<SentWebmention {self.target!r}>" - - def reset_for_revision(self, revision: int, linked: bool) -> None: - """Reopen delivery for a revision while preserving send history. - - :param revision: Source revision to make the desired delivery revision. - :param linked: Whether the target still appears in the current source. - """ - self.desired_revision = revision - self.linked = linked - self.attempt_count = 0 - self.next_retry_at = None - - def clear_result_for_resend(self) -> None: - """Retry the current revision, clearing its previous delivery result. - - Reset attempt and response state without changing the desired source revision. - """ - self.reset_for_revision(self.desired_revision, self.linked) - self.processed_revision = None - self.status = None - self.failure_reason = None - self.webmention_endpoint = None - self.endpoint_response_status = None - self.status_url = None - - @hybrid_property - def awaiting_processing(self) -> bool: - """Return whether the Webmention has an unprocessed source revision.""" - return ( - self.processed_revision is None - or self.processed_revision < self.desired_revision - ) - - @awaiting_processing.inplace.expression - @classmethod - def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: - """Express pending-revision status in database queries. - - :returns: SQLAlchemy expression selecting unprocessed source revisions. - """ - return sa.or_( - cls.processed_revision.is_(None), - cls.processed_revision < cls.desired_revision, - ) - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -def uuid7_to_datetime(identifier: UUID) -> datetime: - """Convert a UUID version 7 timestamp to a datetime. - - :param identifier: UUID whose embedded timestamp should be converted. - :returns: Timestamp as a timezone-aware UTC datetime. - """ - return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) - - -def source_target_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair. - - :param source: Webmention source URL. - :param target: Webmention target URL. - :returns: SHA-256 hexadecimal digest of the ordered URL pair. - """ - encoded = json.dumps( - [source, target], ensure_ascii=False, separators=(",", ":") - ).encode("utf-8") - return sha256(encoded).hexdigest() - - -def target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID). - - :param target: Outgoing Webmention target URL. - :returns: SHA-256 hexadecimal digest of the target URL. - """ - return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/models/__init__.py b/webmentions_ssg/models/__init__.py new file mode 100644 index 0000000..360fc71 --- /dev/null +++ b/webmentions_ssg/models/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Database models grouped by feature.""" diff --git a/webmentions_ssg/models/moderation.py b/webmentions_ssg/models/moderation.py new file mode 100644 index 0000000..4200c39 --- /dev/null +++ b/webmentions_ssg/models/moderation.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain allowlist and blocklist models.""" + +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column + +from ..extensions import Base + + +class BlockedDomain(Base): + """Source domain rejected before submission or verification.""" + + __tablename__ = "blocked_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + +class AllowedDomain(Base): + """A domain accepted when allowlist mode is enabled.""" + + __tablename__ = "allowed_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) diff --git a/webmentions_ssg/models/received.py b/webmentions_ssg/models/received.py new file mode 100644 index 0000000..a9047ad --- /dev/null +++ b/webmentions_ssg/models/received.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Incoming Webmention state and persistence.""" + +from datetime import UTC, datetime +from enum import StrEnum +from uuid import UUID + +from sqlalchemy import ( + DateTime, + Enum, + Index, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.orm import Mapped, mapped_column, validates + +from ..extensions import Base +from .utils import source_target_digest, uuid7_to_datetime + + +class ReceivedWebmentionStatus(StrEnum): + """Persisted lifecycle state of an incoming Webmention.""" + + RECEIVED = "received" + VERIFYING = "verifying" + VERIFIED = "verified" + FAILED = "failed" + SOURCE_REMOVED = "source_removed" + + +class ReceivedWebmention(Base): + """Represent a Webmention received for verification.""" + + __tablename__ = "received_webmentions" + + uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) + + source: Mapped[str] = mapped_column(Text(), nullable=False) + target: Mapped[str] = mapped_column(Text(), nullable=False) + source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("source", "target") + def _refresh_url_digest(self, key: str, value: str) -> str: + source = value if key == "source" else self.__dict__.get("source") + target = value if key == "target" else self.__dict__.get("target") + if source is not None and target is not None: + self.source_target_digest = source_target_digest(source, target) + return value + + status: Mapped[ReceivedWebmentionStatus] = mapped_column( + Enum( + ReceivedWebmentionStatus, + name="received_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=False, + default=ReceivedWebmentionStatus.RECEIVED, + ) + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + nullable=False, + ) + + notification_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + __table_args__ = ( + UniqueConstraint( + "source_target_digest", name="uq_webmention_source_target_digest" + ), + Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), + ) + + def __repr__(self) -> str: + """Represent the received Webmention by source and target.""" + return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" + + @staticmethod + def reopen_verification_values( + *, at: datetime, recovery_attempts: int = 0 + ) -> dict[str, object]: + """Return the columns to reset when verification is explicitly reopened. + + :param at: Timestamp to record for the newly opened verification. + :param recovery_attempts: Number of stale-lease recovery attempts to retain. + :returns: Column values for resetting the verification state. + """ + return { + "status": ReceivedWebmentionStatus.RECEIVED, + "failure_reason": None, + "recovery_attempts": recovery_attempts, + "updated_at": at, + } + + def reopen_verification(self) -> None: + """Reopen an existing row for manual verification. + + Clear the previous outcome and restore the received state with a fresh + timestamp. + """ + for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): + setattr(self, key, value) + + @property + def verified(self) -> bool: + """Return whether the Webmention has been successfully verified.""" + return self.status == ReceivedWebmentionStatus.VERIFIED + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/sources.py b/webmentions_ssg/models/sources.py new file mode 100644 index 0000000..f0fd618 --- /dev/null +++ b/webmentions_ssg/models/sources.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source documents and outgoing Webmention state.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID, uuid7 + +import sqlalchemy as sa +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates + +from ..extensions import Base +from .utils import target_digest, uuid7_to_datetime + + +class SentWebmentionStatus(StrEnum): + """Represent the outcome of a Webmention send attempt.""" + + SENT = "sent" + NO_ENDPOINT = "no_endpoint" + FAILED = "failed" + + +class SourceDocument(Base): + """Represent a generated source document containing outgoing Webmentions.""" + + __tablename__ = "source_documents" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + content_hash: Mapped[str] = mapped_column(String(32), nullable=False) + revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) + + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + revised_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + sent_webmentions: Mapped[list[SentWebmention]] = relationship( + back_populates="source", cascade="all, delete-orphan" + ) + + def __repr__(self) -> str: + """Represent the source by its public URL.""" + return f"<SourceDocument {self.url!r}>" + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the source UUID.""" + return uuid7_to_datetime(self.uuid) + + +class SentWebmention(Base): + """Represent an outgoing Webmention associated with a source.""" + + __tablename__ = "sent_webmentions" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + source_uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("source_documents.uuid", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + target: Mapped[str] = mapped_column(Text(), nullable=False) + target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("target") + def _update_target_digest(self, key: str, value: str) -> str: + self.target_digest = target_digest(value) + return value + + linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + + desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) + processed_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + sent_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + + status: Mapped[SentWebmentionStatus | None] = mapped_column( + Enum( + SentWebmentionStatus, + name="sent_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=True, + default=None, + ) + + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + webmention_endpoint: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + endpoint_response_status: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + last_attempted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") + + __table_args__ = ( + UniqueConstraint( + "source_uuid", + "target_digest", + name="uq_sent_webmention_source_uuid_target_digest", + ), + ) + + def __repr__(self) -> str: + """Represent the sent Webmention by its target URL.""" + return f"<SentWebmention {self.target!r}>" + + def reset_for_revision(self, revision: int, linked: bool) -> None: + """Reopen delivery for a revision while preserving send history. + + :param revision: Source revision to make the desired delivery revision. + :param linked: Whether the target still appears in the current source. + """ + self.desired_revision = revision + self.linked = linked + self.attempt_count = 0 + self.next_retry_at = None + + def clear_result_for_resend(self) -> None: + """Retry the current revision, clearing its previous delivery result. + + Reset attempt and response state without changing the desired source revision. + """ + self.reset_for_revision(self.desired_revision, self.linked) + self.processed_revision = None + self.status = None + self.failure_reason = None + self.webmention_endpoint = None + self.endpoint_response_status = None + self.status_url = None + + @hybrid_property + def awaiting_processing(self) -> bool: + """Return whether the Webmention has an unprocessed source revision.""" + return ( + self.processed_revision is None + or self.processed_revision < self.desired_revision + ) + + @awaiting_processing.inplace.expression + @classmethod + def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: + """Express pending-revision status in database queries. + + :returns: SQLAlchemy expression selecting unprocessed source revisions. + """ + return sa.or_( + cls.processed_revision.is_(None), + cls.processed_revision < cls.desired_revision, + ) + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/user.py b/webmentions_ssg/models/user.py new file mode 100644 index 0000000..b09c3d1 --- /dev/null +++ b/webmentions_ssg/models/user.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Application user and session identity model.""" + +from hashlib import sha256 +from typing import Never + +from flask_login import UserMixin +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column +from werkzeug.security import check_password_hash, generate_password_hash + +from ..extensions import Base + + +class User(UserMixin, Base): + """Represent an authenticated application user.""" + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(Text(), index=True, unique=True) + password_hash: Mapped[str] = mapped_column(Text(), nullable=False) + + def __repr__(self) -> str: + """Represent the user by username.""" + return f"<User {self.username}>" + + @property + def password(self) -> Never: + """Prevent access to the user's plain-text password. + + :raises AttributeError: Always, because the password is write-only. + """ + raise AttributeError("Password is write-only") + + @password.setter + def password(self, password: str) -> None: + """Hash and store a new password. + + :param password: Plain-text password to hash. + """ + self.password_hash = generate_password_hash(password) + + def get_id(self) -> str: + """Bind the login session to the current stored password hash. + + Only a one-way fingerprint is placed in Flask's signed cookie, never + the password hash itself. Rehashing the password invalidates all + previously issued session identifiers without a schema change. + + :returns: Session identifier combining the user ID and password-hash + fingerprint. + """ + fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() + return f"{self.id}:{fingerprint}" + + def check_password(self, password: str) -> bool: + """Check a plain-text password against the stored password hash. + + :param password: Plain-text password to check. + :returns: Whether the password matches the stored hash. + """ + return check_password_hash(self.password_hash, password) diff --git a/webmentions_ssg/models/utils.py b/webmentions_ssg/models/utils.py new file mode 100644 index 0000000..e961088 --- /dev/null +++ b/webmentions_ssg/models/utils.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""URL fingerprints and UUID timestamp utilities.""" + +import json +from datetime import UTC, datetime +from hashlib import sha256 +from uuid import UUID + + +def uuid7_to_datetime(identifier: UUID) -> datetime: + """Convert a UUID version 7 timestamp to a datetime. + + :param identifier: UUID whose embedded timestamp should be converted. + :returns: Timestamp as a timezone-aware UTC datetime. + """ + return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) + + +def source_target_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair. + + :param source: Webmention source URL. + :param target: Webmention target URL. + :returns: SHA-256 hexadecimal digest of the ordered URL pair. + """ + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID). + + :param target: Outgoing Webmention target URL. + :returns: SHA-256 hexadecimal digest of the target URL. + """ + return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/source_policy.py b/webmentions_ssg/moderation_policy.py index 2c62527..51e7995 100644 --- a/webmentions_ssg/source_policy.py +++ b/webmentions_ssg/moderation_policy.py @@ -11,11 +11,11 @@ from flask import current_app from .extensions import db from .hostname import normalize_hostname -from .models import AllowedSource, BlockedSource +from .models.moderation import AllowedDomain, BlockedDomain -def source_domain_from_url(source: str) -> str | None: - """Extract and canonicalize the source hostname without a network request.""" +def domain_from_url(source: str) -> str | None: + """Extract and canonicalize the hostname without a network request.""" try: return normalize_hostname(source, from_url=True) except ValueError: @@ -23,7 +23,7 @@ def source_domain_from_url(source: str) -> str | None: def has_matching_domain( - model: type[AllowedSource | BlockedSource], domain: str + model: type[AllowedDomain | BlockedDomain], domain: str ) -> bool: """Match the hostname itself and parent domains, never partial labels.""" candidates = [domain] @@ -42,20 +42,20 @@ def has_matching_domain( ) -def source_rejection_reason(source: str) -> str | None: - """Return the reason a source cannot be received, or None when accepted. +def rejection_reason(source: str) -> str | None: + """Return the reason a source cannot be received, or ``None`` if accepted. The blocklist always wins. In allowlist mode an empty allowlist rejects every source. The database, not a stale per-worker cache, is consulted. """ - if (domain := source_domain_from_url(source)) is None: + if (domain := domain_from_url(source)) is None: return "Invalid source hostname" - if has_matching_domain(BlockedSource, domain): + if has_matching_domain(BlockedDomain, domain): return "Source domain is blocked" - if current_app.config["SOURCE_ALLOWLIST_ENABLED"] and not has_matching_domain( - AllowedSource, domain + if current_app.config["ALLOWLIST_ENABLED"] and not has_matching_domain( + AllowedDomain, domain ): return "Source domain is not allowlisted" diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index c503e1f..6a0e6b9 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,7 +11,7 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def email_verified_digest() -> None: diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index b882493..b4b6a83 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 6869477..1f98702 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,7 +14,7 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, SourceDocument +from ..models.sources import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 6b14ace..fc70f27 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -15,7 +15,7 @@ from huey.exceptions import TaskLockedException from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import SentWebmention, SentWebmentionStatus +from ..models.sources import SentWebmention, SentWebmentionStatus from ..net import ( AddressResolutionError, UnsafeAddressError, diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index d55608c..ca45777 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -15,7 +15,8 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus +from ..moderation_policy import rejection_reason from ..net import ( AddressResolutionError, ResponseTooLargeError, @@ -25,7 +26,6 @@ from ..net import ( media_type_of, read_limited_body, ) -from ..source_policy import source_rejection_reason from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target @@ -201,7 +201,7 @@ def determine_verification_outcome( :raises httpx.RequestError: If a transient HTTP request error should be retried. """ # Recheck at worker time: a source may have been blocked since submission. - if reason := source_rejection_reason(claim.source): + if reason := rejection_reason(claim.source): return ReceivedWebmentionStatus.FAILED, reason try: diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index d9c8e87..3395dd5 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -45,8 +45,8 @@ SPDX-License-Identifier: BSD-3-Clause <div class="navbar-nav flex-row gap-3"> <a class="nav-link{% if request.endpoint == 'webmentions.received_list' %} active{% endif %}" href='{{ url_for("webmentions.received_list") }}'>Received</a> - <a class="nav-link{% if request.endpoint and request.endpoint.startswith('webmentions.source_domain') %} active{% endif %}" - href="{{ url_for('webmentions.source_domain_list') }}">Source rules</a> + <a class="nav-link{% if request.endpoint and request.endpoint.startswith('webmentions.moderation') %} active{% endif %}" + href="{{ url_for('webmentions.moderation') }}">Source rules</a> <a class="nav-link{% if request.endpoint == 'webmentions.source_list' %} active{% endif %}" href='{{ url_for("webmentions.source_list") }}'>Sent</a> <form action="{{ url_for('webmentions.logout') }}" diff --git a/webmentions_ssg/templates/source_domains.html b/webmentions_ssg/templates/moderation.html index 2db942f..e45fe3f 100644 --- a/webmentions_ssg/templates/source_domains.html +++ b/webmentions_ssg/templates/moderation.html @@ -14,13 +14,13 @@ SPDX-License-Identifier: BSD-3-Clause </p> <p> Allowlist mode is <strong>{{ "enabled" if allowlist_enabled else "disabled" }}</strong>. - Set <code>SOURCE_ALLOWLIST_ENABLED = True</code> in the server configuration to + Set <code>ALLOWLIST_ENABLED = True</code> in the server configuration to accept only listed source domains. An empty enabled allowlist rejects all sources. </p> {% for kind, entries in (("blocked", blocked), ("allowed", allowed)) %} <section class="mb-4" aria-labelledby="{{ kind }}-heading"> <h2 id="{{ kind }}-heading">{{ "Blocked" if kind == "blocked" else "Allowed" }} domains</h2> - <form action="{{ url_for('webmentions.source_domain_add', kind=kind) }}" + <form action="{{ url_for('webmentions.moderation_domain_add', kind=kind) }}" method="post" class="row g-2 align-items-end mb-3"> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> @@ -50,10 +50,11 @@ SPDX-License-Identifier: BSD-3-Clause <tr> <td class="text-break">{{ entry.domain }}</td> <td> - <form action="{{ url_for('webmentions.source_domain_delete', kind=kind, entry_id=entry.id) }}" + <form action="{{ url_for('webmentions.moderation_domain_delete', kind=kind, entry_id=entry.id) }}" method="post"> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> - <button type="submit" class="btn btn-sm btn-outline-danger">{{ "Unblock" if kind == "blocked" else "Remove" }}</button> + <button type="submit" + class="btn btn-sm btn-outline-danger">{{ "Unblock" if kind == "blocked" else "Remove" }}</button> </form> </td> </tr> diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py index 7f091f0..cdf024a 100644 --- a/webmentions_ssg/views/__init__.py +++ b/webmentions_ssg/views/__init__.py @@ -4,7 +4,7 @@ """Collect and expose the routes for the blueprint.""" -from . import admin, endpoint, session +from . import endpoint, moderation, received, session, sources from .blueprint import webmentions_bp -__all__ = ["admin", "endpoint", "session", "webmentions_bp"] +__all__ = ["endpoint", "moderation", "received", "session", "sources", "webmentions_bp"] diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index 6d84d92..322e0f0 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -14,11 +14,12 @@ import sqlalchemy.exc as sa_exc from flask import Response, current_app, render_template, url_for from flask.typing import ResponseReturnValue -from .. import forms from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter -from ..models import ReceivedWebmention, source_target_digest -from ..source_policy import source_rejection_reason +from ..forms.endpoint import WebmentionSubmissionForm +from ..models.received import ReceivedWebmention +from ..models.utils import source_target_digest +from ..moderation_policy import rejection_reason from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -145,7 +146,7 @@ def receive_webmention() -> ResponseReturnValue: :returns: HTTP 201 response with the Webmention status URL in the ``Location`` header, or validation errors with HTTP 400. """ - form = forms.WebmentionSubmissionForm(meta={"csrf": False}) + form = WebmentionSubmissionForm(meta={"csrf": False}) if not form.validate_on_submit(): return form.errors, HTTPStatus.BAD_REQUEST @@ -153,7 +154,7 @@ def receive_webmention() -> ResponseReturnValue: source = cast(str, form.source.data) target = cast(str, form.target.data) - if reason := source_rejection_reason(source): + if reason := rejection_reason(source): return {"error": reason}, HTTPStatus.FORBIDDEN result = record_submission(source, target) diff --git a/webmentions_ssg/views/moderation.py b/webmentions_ssg/views/moderation.py new file mode 100644 index 0000000..c076ce9 --- /dev/null +++ b/webmentions_ssg/views/moderation.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated domain moderation views and operations.""" + +from http import HTTPStatus +from typing import cast + +import sqlalchemy as sa +from flask import abort, current_app, flash, redirect, render_template, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.exc import IntegrityError + +from ..extensions import db +from ..forms.admin import ConfirmActionForm +from ..forms.moderation import ModerateDomainForm +from ..models.moderation import AllowedDomain, BlockedDomain +from .blueprint import webmentions_bp + +MODERATION_DOMAIN_MODELS = {"blocked": BlockedDomain, "allowed": AllowedDomain} + + +def block_domains(domains: set[str]) -> int: + """Persist distinct source domains, tolerating concurrent blocklist inserts. + + Savepoints isolate unique-constraint races so one preexisting entry cannot + prevent other selected domains from being blocked. + """ + existing = set( + db.session.scalars( + sa.select(BlockedDomain.domain).where(BlockedDomain.domain.in_(domains)) + ) + ) + added = 0 + for domain in sorted(domains - existing): + try: + with db.session.begin_nested(): + db.session.add(BlockedDomain(domain=domain)) + db.session.flush() + except IntegrityError: + # A concurrent admin action has already blocked this hostname. + continue + added += 1 + db.session.commit() + return added + + +@webmentions_bp.get("/moderation") +@login_required +def moderation() -> ResponseReturnValue: + """Manage blocklist and allowlist entries used by the receiver.""" + return render_template( + "moderation.html", + title="Source domain rules", + blocked=db.session.scalars( + sa.select(BlockedDomain).order_by(BlockedDomain.domain) + ).all(), + allowed=db.session.scalars( + sa.select(AllowedDomain).order_by(AllowedDomain.domain) + ).all(), + allowlist_enabled=current_app.config["ALLOWLIST_ENABLED"], + ) + + +@webmentions_bp.post("/moderation/<kind>/add") +@login_required +def moderation_domain_add(kind: str) -> ResponseReturnValue: + """Validate and add a normalized hostname to the selected domain list.""" + if (model := MODERATION_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + form = ModerateDomainForm() + if not form.validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + domain = cast(str, form.domain.data) + if db.session.scalar(sa.select(model.id).where(model.domain == domain)) is None: + db.session.add(model(domain=domain)) + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + + flash(f"Source domain {domain} added to {kind} list.", "success") + return redirect(url_for("webmentions.moderation")) + + +@webmentions_bp.post("/moderation/<kind>/<int:entry_id>/delete") +@login_required +def moderation_domain_delete(kind: str, entry_id: int) -> ResponseReturnValue: + """Remove a rule; unblocked domains become eligible for submissions again.""" + if (model := MODERATION_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + entry = db.get_or_404(model, entry_id) + domain = entry.domain + db.session.delete(entry) + db.session.commit() + flash(f"Source domain {domain} removed from {kind} list.", "success") + return redirect(url_for("webmentions.moderation")) diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/received.py index eb5a5e7..7d633ab 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/received.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Authenticated incoming and outgoing Webmention management.""" +"""Authenticated views for received Webmentions.""" import uuid from datetime import UTC, date, datetime, time @@ -13,25 +13,20 @@ import sqlalchemy as sa from flask import abort, current_app, flash, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from flask_login import login_required -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import selectinload -from .. import forms from ..db_helpers import rows_affected from ..extensions import db -from ..models import ( - AllowedSource, - BlockedSource, - ReceivedWebmention, - ReceivedWebmentionStatus, - SentWebmention, - SourceDocument, +from ..forms.admin import ConfirmActionForm +from ..forms.received import ( + RECEIVED_STATUS_CHOICES, + ReceivedWebmentionFilterForm, + default_received_page_size, ) -from ..source_policy import source_domain_from_url -from ..tasks.scanner import manual_scan_sources -from ..tasks.sender import send_webmention +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus +from ..moderation_policy import domain_from_url from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp +from .moderation import block_domains RECEIVED_STATUS_FILTERS = { "pending": (ReceivedWebmentionStatus.RECEIVED,), @@ -42,7 +37,6 @@ RECEIVED_STATUS_FILTERS = { ReceivedWebmentionStatus.SOURCE_REMOVED, ), } -SOURCE_DOMAIN_MODELS = {"blocked": BlockedSource, "allowed": AllowedSource} class ReceivedFilterArgs(TypedDict, total=False): @@ -57,20 +51,16 @@ class ReceivedFilterArgs(TypedDict, total=False): per_page: str -def received_filter_form() -> forms.ReceivedWebmentionFilterForm: +def received_filter_form() -> ReceivedWebmentionFilterForm: """Validate query-string filters without requiring CSRF on the GET form.""" - form = forms.ReceivedWebmentionFilterForm( - formdata=request.args, meta={"csrf": False} - ) + form = ReceivedWebmentionFilterForm(formdata=request.args, meta={"csrf": False}) if not form.validate(): abort(HTTPStatus.BAD_REQUEST) form.q.data = (form.q.data or "").strip() return form -def received_filter_args( - form: forms.ReceivedWebmentionFilterForm, -) -> ReceivedFilterArgs: +def received_filter_args(form: ReceivedWebmentionFilterForm) -> ReceivedFilterArgs: """Serialize non-default form fields registered in ``ReceivedFilterArgs``.""" values: dict[str, str] = {} for name in ReceivedFilterArgs.__annotations__: @@ -147,7 +137,7 @@ def received_list() -> ResponseReturnValue: if sort_key != "received": query = query.order_by(ReceivedWebmention.uuid.desc()) - page_size = int(filters.get("per_page", forms.default_received_page_size())) + page_size = int(filters.get("per_page", default_received_page_size())) webmentions = db.paginate( query, per_page=page_size, @@ -171,11 +161,11 @@ def received_list() -> ResponseReturnValue: filters=filters, sort_key=sort_key, sort_order=sort_order, - status_choices=forms.RECEIVED_STATUS_CHOICES, + status_choices=RECEIVED_STATUS_CHOICES, page_size=page_size, sort_urls=sort_urls, max_received_page_size=current_app.config["RECEIVED_MAX_PAGE_SIZE"], - action_form=forms.ConfirmActionForm(), + action_form=ConfirmActionForm(), ) @@ -187,7 +177,7 @@ def delete_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnValu :param webmention_uuid: UUID of the Webmention to delete. :returns: Redirect response to the received Webmentions view. """ - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -217,7 +207,7 @@ def reverify_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnVa :param webmention_uuid: UUID of the Webmention to reverify. :returns: Redirect response to the received Webmentions view. """ - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -275,7 +265,7 @@ def selected_received_webmentions() -> list[ReceivedWebmention]: @login_required def reverify_selected_received_webmentions() -> ResponseReturnValue: """Reopen and queue each selected received Webmention for verification.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -303,7 +293,7 @@ def reverify_selected_received_webmentions() -> ResponseReturnValue: @login_required def delete_selected_received_webmentions() -> ResponseReturnValue: """Permanently delete only the selected received Webmentions.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -327,7 +317,7 @@ def delete_selected_received_webmentions() -> ResponseReturnValue: @login_required def delete_all_declined_received_webmentions() -> ResponseReturnValue: """Delete every declined received Webmention, regardless of age or filters.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) @@ -349,46 +339,21 @@ def delete_all_declined_received_webmentions() -> ResponseReturnValue: ) -def block_source_domains(domains: set[str]) -> int: - """Persist distinct source domains, tolerating concurrent blocklist inserts. - - Savepoints isolate unique-constraint races so one preexisting entry cannot - prevent other selected domains from being blocked. - """ - existing = set( - db.session.scalars( - sa.select(BlockedSource.domain).where(BlockedSource.domain.in_(domains)) - ) - ) - added = 0 - for domain in sorted(domains - existing): - try: - with db.session.begin_nested(): - db.session.add(BlockedSource(domain=domain)) - db.session.flush() - except IntegrityError: - # A concurrent admin action has already blocked this hostname. - continue - added += 1 - db.session.commit() - return added - - @webmentions_bp.post("/received/bulk/block-sources") @login_required def block_selected_received_sources() -> ResponseReturnValue: """Block the unique source domains of selected received Webmentions.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) webmentions = selected_received_webmentions() - domains = {source_domain_from_url(webmention.source) for webmention in webmentions} + domains = {domain_from_url(webmention.source) for webmention in webmentions} if None in domains: # Fail the entire selection instead of silently skipping a source. abort(HTTPStatus.BAD_REQUEST) - added = block_source_domains(cast(set[str], domains)) + added = block_domains(cast(set[str], domains)) flash( f"{added} source domains newly blocked from {len(webmentions)} selected Webmentions.", "success", @@ -406,15 +371,15 @@ def block_selected_received_sources() -> ResponseReturnValue: @login_required def block_received_source(webmention_uuid: uuid.UUID) -> ResponseReturnValue: """Add a received Webmention's source hostname to the blocklist.""" - if not forms.ConfirmActionForm().validate_on_submit(): + if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) filters = received_filter_args(received_filter_form()) webmention = db.get_or_404(ReceivedWebmention, webmention_uuid) - if (domain := source_domain_from_url(webmention.source)) is None: + if (domain := domain_from_url(webmention.source)) is None: abort(HTTPStatus.BAD_REQUEST) - block_source_domains({domain}) + block_domains({domain}) flash(f"Source domain {domain} blocked.", "success") return redirect( @@ -424,162 +389,3 @@ def block_received_source(webmention_uuid: uuid.UUID) -> ResponseReturnValue: **filters, ) ) - - -@webmentions_bp.get("/source-domains") -@login_required -def source_domain_list() -> ResponseReturnValue: - """Manage blocklist and allowlist entries used by the receiver.""" - return render_template( - "source_domains.html", - title="Source domain rules", - blocked=db.session.scalars( - sa.select(BlockedSource).order_by(BlockedSource.domain) - ).all(), - allowed=db.session.scalars( - sa.select(AllowedSource).order_by(AllowedSource.domain) - ).all(), - allowlist_enabled=current_app.config["SOURCE_ALLOWLIST_ENABLED"], - ) - - -@webmentions_bp.post("/source-domains/<kind>/add") -@login_required -def source_domain_add(kind: str) -> ResponseReturnValue: - """Validate and add one normalized hostname to the selected list.""" - if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: - abort(HTTPStatus.NOT_FOUND) - - form = forms.SourceDomainForm() - if not form.validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - domain = cast(str, form.domain.data) - if db.session.scalar(sa.select(model.id).where(model.domain == domain)) is None: - db.session.add(model(domain=domain)) - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - - flash(f"Source domain {domain} added to {kind} list.", "success") - return redirect(url_for("webmentions.source_domain_list")) - - -@webmentions_bp.post("/source-domains/<kind>/<int:entry_id>/delete") -@login_required -def source_domain_delete(kind: str, entry_id: int) -> ResponseReturnValue: - """Remove one entry; a blocked domain becomes eligible for new submissions.""" - if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: - abort(HTTPStatus.NOT_FOUND) - - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - entry = db.get_or_404(model, entry_id) - domain = entry.domain - db.session.delete(entry) - db.session.commit() - flash(f"Source domain {domain} removed from {kind} list.", "success") - return redirect(url_for("webmentions.source_domain_list")) - - -@webmentions_bp.route("/sources") -@login_required -def source_list() -> ResponseReturnValue: - """Display sources with sent Webmentions. - - :returns: Rendered page containing the paginated source list. - """ - sources = db.paginate( - sa.select(SourceDocument) - .options(selectinload(SourceDocument.sent_webmentions)) - .order_by(SourceDocument.last_seen_at.desc()), - per_page=25, - ) - return render_template( - "sent.html", - title="Sent Webmentions", - sources=sources, - action_form=forms.ConfirmActionForm(), - ) - - -@webmentions_bp.post("/sources/rescan") -@login_required -def rescan_sources() -> ResponseReturnValue: - """Queue a manual scan of sent Webmention sources. - - :returns: Redirect response to the sent Webmentions view. - """ - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - manual_scan_sources() - flash("Source rescan queued.", "success") - - return redirect(url_for("webmentions.source_list")) - - -@webmentions_bp.route("/sources/<uuid:source_uuid>") -@login_required -def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: - """Display the sent Webmentions associated with a source. - - :param source_uuid: UUID of the source to display. - :returns: Rendered source details page. - """ - source = db.get_or_404(SourceDocument, source_uuid) - - webmentions = db.paginate( - sa.select(SentWebmention) - .where(SentWebmention.source_uuid == source.uuid) - .order_by(SentWebmention.target, SentWebmention.uuid), - per_page=25, - ) - - return render_template( - "sent_source.html", - title="Sent Webmentions", - source=source, - webmentions=webmentions, - action_form=forms.ConfirmActionForm(), - ) - - -@webmentions_bp.post("/sources/<uuid:source_uuid>/<uuid:webmention_uuid>/resend") -@login_required -def resend_webmention( - source_uuid: uuid.UUID, webmention_uuid: uuid.UUID -) -> ResponseReturnValue: - """Queue an outgoing Webmention to be sent again. - - The current desired source revision is reopened for processing so the - sender task does not treat the Webmention as already completed. Historical - successful-send fields are preserved, while result fields from the previous - attempt are cleared. - - :param source_uuid: UUID of the source owning the Webmention. - :param webmention_uuid: UUID of the sent Webmention to resend. - :returns: Redirect response to the source details view. - """ - if not forms.ConfirmActionForm().validate_on_submit(): - abort(HTTPStatus.BAD_REQUEST) - - webmention = db.get_or_404(SentWebmention, webmention_uuid) - - if webmention.source_uuid != source_uuid: - abort(HTTPStatus.NOT_FOUND) - - # Requeue the existing desired revision instead of inventing a new source - # revision. send_webmention() otherwise exits when this revision has already - # been processed. - webmention.clear_result_for_resend() - - db.session.commit() - - send_webmention(webmention_uuid) - - flash(f"Webmention {webmention.uuid} queued for resend.", "success") - - return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py index d9cec7f..78d1ed6 100644 --- a/webmentions_ssg/views/session.py +++ b/webmentions_ssg/views/session.py @@ -10,10 +10,10 @@ from flask.typing import ResponseReturnValue from flask_login import current_user, login_required, login_user, logout_user from werkzeug.security import check_password_hash -from .. import forms from ..auth import dummy_password_hash, is_safe_next_url, login_username_rate_limit_key from ..extensions import db, limiter -from ..models import User +from ..forms.user import LoginForm +from ..models.user import User from .blueprint import webmentions_bp @@ -50,7 +50,7 @@ def login() -> ResponseReturnValue: if current_user.is_authenticated: return redirect(url_for("webmentions.index")) - form = forms.LoginForm() + form = LoginForm() if form.validate_on_submit(): user = db.session.scalar( diff --git a/webmentions_ssg/views/sources.py b/webmentions_ssg/views/sources.py new file mode 100644 index 0000000..b68d2fd --- /dev/null +++ b/webmentions_ssg/views/sources.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated source document and sent Webmention views.""" + +import uuid +from http import HTTPStatus + +import sqlalchemy as sa +from flask import abort, flash, redirect, render_template, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.orm import selectinload + +from ..extensions import db +from ..forms.admin import ConfirmActionForm +from ..models.sources import SentWebmention, SourceDocument +from ..tasks.scanner import manual_scan_sources +from ..tasks.sender import send_webmention +from .blueprint import webmentions_bp + + +@webmentions_bp.route("/sources") +@login_required +def source_list() -> ResponseReturnValue: + """Display sources with sent Webmentions. + + :returns: Rendered page containing the paginated source list. + """ + sources = db.paginate( + sa.select(SourceDocument) + .options(selectinload(SourceDocument.sent_webmentions)) + .order_by(SourceDocument.last_seen_at.desc()), + per_page=25, + ) + return render_template( + "sent.html", + title="Sent Webmentions", + sources=sources, + action_form=ConfirmActionForm(), + ) + + +@webmentions_bp.post("/sources/rescan") +@login_required +def rescan_sources() -> ResponseReturnValue: + """Queue a manual scan of sent Webmention sources. + + :returns: Redirect response to the sent Webmentions view. + """ + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + manual_scan_sources() + flash("Source rescan queued.", "success") + + return redirect(url_for("webmentions.source_list")) + + +@webmentions_bp.route("/sources/<uuid:source_uuid>") +@login_required +def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: + """Display the sent Webmentions associated with a source. + + :param source_uuid: UUID of the source to display. + :returns: Rendered source details page. + """ + source = db.get_or_404(SourceDocument, source_uuid) + + webmentions = db.paginate( + sa.select(SentWebmention) + .where(SentWebmention.source_uuid == source.uuid) + .order_by(SentWebmention.target, SentWebmention.uuid), + per_page=25, + ) + + return render_template( + "sent_source.html", + title="Sent Webmentions", + source=source, + webmentions=webmentions, + action_form=ConfirmActionForm(), + ) + + +@webmentions_bp.post("/sources/<uuid:source_uuid>/<uuid:webmention_uuid>/resend") +@login_required +def resend_webmention( + source_uuid: uuid.UUID, webmention_uuid: uuid.UUID +) -> ResponseReturnValue: + """Queue an outgoing Webmention to be sent again. + + The current desired source revision is reopened for processing so the + sender task does not treat the Webmention as already completed. Historical + successful-send fields are preserved, while result fields from the previous + attempt are cleared. + + :param source_uuid: UUID of the source owning the Webmention. + :param webmention_uuid: UUID of the sent Webmention to resend. + :returns: Redirect response to the source details view. + """ + if not ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + webmention = db.get_or_404(SentWebmention, webmention_uuid) + + if webmention.source_uuid != source_uuid: + abort(HTTPStatus.NOT_FOUND) + + # Requeue the existing desired revision instead of inventing a new source + # revision. send_webmention() otherwise exits when this revision has already + # been processed. + webmention.clear_result_for_resend() + + db.session.commit() + + send_webmention(webmention_uuid) + + flash(f"Webmention {webmention.uuid} queued for resend.", "success") + + return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) |
