diff options
| author | Dennis Fink | 2026-11-17 14:44:00 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-17 14:44:00 +0100 |
| commit | 227092b57acc886469e16bcc4c0d3256e86d3d1e (patch) | |
| tree | c1de8d5f8f3a37562e482f84e8017213c08a05d6 | |
| parent | f98b0e446b41325c494bd97900b7bbf730a7d67e (diff) | |
| download | webmentions-ssg-227092b57acc886469e16bcc4c0d3256e86d3d1e.tar.gz webmentions-ssg-227092b57acc886469e16bcc4c0d3256e86d3d1e.zip | |
feat(receiver): add source domain access rules
Add persistent source block and allow lists with authenticated
management, single-row and bulk blocking actions, and optional
fail-closed allowlist mode.
Reject disallowed sources before enqueueing and recheck policy in
verification workers, while sharing canonical hostname normalization
across receiver, sender, and configuration paths.
25 files changed, 1036 insertions, 80 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index afc5621..882c86d 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -95,6 +95,13 @@ MAX_REDIRECTS = 20 # also used for outgoing requests # Webmention. Include each hostname you serve; wildcards are not supported. ALLOWED_TARGET_HOSTNAMES = {"example.com", "www.example.com"} +# Optional source allowlist mode. By default, every valid public source is +# accepted unless its hostname appears in the admin-managed blocklist. +# When enabled, a source must match an allowed domain; blocked entries always +# override allowed entries. Manage both lists in the authenticated "Source rules" +# page. Entries include subdomains; an empty allowlist rejects all sources. +SOURCE_ALLOWLIST_ENABLED = False + # Maximum size of an incoming HTTP request body, including form encoding. This # global Flask limit also applies to login and administration routes. MAX_CONTENT_LENGTH = 64 * 1024 diff --git a/tests/tasks/test_verification.py b/tests/tasks/test_verification.py index c336b19..c5550c8 100644 --- a/tests/tasks/test_verification.py +++ b/tests/tasks/test_verification.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from flask import Flask from webmentions_ssg import db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models import BlockedSource, ReceivedWebmention from webmentions_ssg.net import AddressResolutionError SOURCE_URL = "https://source.example/article" @@ -337,6 +337,7 @@ def test_source_mentions_target_rejects_unsupported_media_type( ], ) def test_determine_verification_outcome_returns_outcome( + app: Flask, verification: ModuleType, monkeypatch: pytest.MonkeyPatch, mentions: bool, @@ -350,10 +351,30 @@ def test_determine_verification_outcome_returns_outcome( uuid.uuid7(), SOURCE_URL, TARGET_URL, datetime.now(UTC) ) - assert verification.determine_verification_outcome(claim) == ( - expected_status, - expected_reason, + with app.app_context(): + assert verification.determine_verification_outcome(claim) == ( + expected_status, + expected_reason, + ) + + +def test_verify_webmention_refuses_domain_blocked_after_enqueue( + app: Flask, verification: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention(app) + with app.app_context(): + db.session.add(BlockedSource(domain="source.example")) + db.session.commit() + + fetch = Mock(return_value=True) + monkeypatch.setattr(verification, "source_mentions_target", fetch) + verification.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == ( + "failed", + "Source domain is blocked", ) + fetch.assert_not_called() @patch("webmentions_ssg.tasks.verification.source_mentions_target") diff --git a/tests/test_cli.py b/tests/test_cli.py index c98b2fc..f5535d2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -243,6 +243,20 @@ def test_upgrade_applies_pending_migrations_and_preserves_data( assert "sent_webmentions" in inspect(connection).get_table_names() +@pytest.mark.parametrize("existing_head", ("3e8c87a96b2d", "9a4e0c8211d6")) +def test_upgrade_merges_existing_migration_branches( + database_app: Flask, existing_head: str +) -> None: + """Existing installations on either branch can upgrade to a single head.""" + with database_app.app_context(): + migrate_upgrade(revision=existing_head) + migrate_upgrade() + + assert _database_revision(database_app) == _head_revision(database_app) + tables = set(inspect(db.engine).get_table_names()) + assert {"blocked_sources", "allowed_sources", "source_documents"} <= tables + + def test_upgrade_is_safe_to_run_when_already_current(database_app: Flask) -> None: runner = database_app.test_cli_runner() assert runner.invoke(args=["db", "create-db"]).exit_code == 0 diff --git a/tests/test_config.py b/tests/test_config.py index d075e7d..4a5f4cb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -39,6 +39,15 @@ def test_session_cookie_defaults( assert config_class.SESSION_COOKIE_SAMESITE == "Lax" +def test_source_allowlist_mode_requires_boolean_configuration(tmp_path: Path) -> None: + class BadSourceModeConfig(TestingConfig): + LOG_FILE = str(tmp_path / "mode-test.log") + SOURCE_ALLOWLIST_ENABLED = "yes" + + with pytest.raises(ValueError, match="SOURCE_ALLOWLIST_ENABLED"): + create_app(BadSourceModeConfig) + + def test_explicit_config_overrides_session_cookie_defaults() -> None: class CookieOverrideConfig(TestingConfig): SESSION_COOKIE_SECURE = True @@ -141,6 +150,7 @@ def test_allowed_hostnames_are_normalized_at_startup(tmp_path: Path) -> None: (["example.com", None], ValueError), (["example.com", ""], ValueError), (["example.com", " "], ValueError), + (["*.example.com"], ValueError), ], ) def test_invalid_allowed_hostnames_fail_at_startup( @@ -154,6 +164,23 @@ def test_invalid_allowed_hostnames_fail_at_startup( create_app(InvalidHostnameConfig) +def test_hostname_config_shares_admin_canonicalization(tmp_path: Path) -> None: + class InternationalizedHosts(TestingConfig): + LOG_FILE = str(tmp_path / "idn.log") + ALLOWED_TARGET_HOSTNAMES = {"BÜCHER.example.", "EXAMPLE.COM"} + IGNORED_TARGET_HOSTNAMES = {"*.BÜCHER.example.", "EXAMPLE.COM."} + + app = create_app(InternationalizedHosts) + assert app.config["ALLOWED_TARGET_HOSTNAMES"] == { + "xn--bcher-kva.example", + "example.com", + } + assert app.config["IGNORED_TARGET_HOSTNAMES"] == { + "*.xn--bcher-kva.example", + "example.com", + } + + def test_ignored_hostnames_are_normalized_at_startup(tmp_path: Path) -> None: class MixedCaseConfig(TestingConfig): LOG_FILE = str(tmp_path / "test.log") diff --git a/tests/test_hostname.py b/tests/test_hostname.py new file mode 100644 index 0000000..0faaacd --- /dev/null +++ b/tests/test_hostname.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression tests for the one shared hostname normalizer.""" + +import pytest + +from webmentions_ssg.hostname import normalize_hostname + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("Example.COM.", "example.com"), + ("blog.Example.COM", "blog.example.com"), + ("bücher.example", "xn--bcher-kva.example"), + ("198.51.100.10", "198.51.100.10"), + ("2001:db8::1", "2001:db8::1"), + ], +) +def test_normalize_bare_hostname(value: str, expected: str) -> None: + assert normalize_hostname(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + "", + " example.com", + "https://example.com", + "example.com/path", + "*.example.com", + "-bad.example", + "bad-.example", + "example..com", + "example.com:8080", + "hello_foo.example", + "a" * 64 + ".example", + ], +) +def test_reject_invalid_bare_hostname(value: str) -> None: + with pytest.raises(ValueError): + normalize_hostname(value) + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("https://EXAMPLE.com./path", "example.com"), + ("http://[2001:db8::1]/", "2001:db8::1"), + ("https://BÜCHER.example.:8443/post", "xn--bcher-kva.example"), + ], +) +def test_normalize_url_hostname(url: str, expected: str) -> None: + assert normalize_hostname(url, from_url=True) == expected + + +@pytest.mark.parametrize("url", ["https:///path", "http://[::1", "invalid"]) +def test_reject_url_without_valid_hostname(url: str) -> None: + with pytest.raises(ValueError): + normalize_hostname(url, from_url=True) + + +def test_wildcard_mode_is_explicit() -> None: + assert normalize_hostname("*.EXAMPLE.com.", allow_patterns=True) == "*.example.com" + assert ( + normalize_hostname("blog?.example.com", allow_patterns=True) + == "blog?.example.com" + ) + with pytest.raises(ValueError): + normalize_hostname("*.example.com") diff --git a/tests/test_net.py b/tests/test_net.py index 472ae91..6ff267d 100644 --- a/tests/test_net.py +++ b/tests/test_net.py @@ -25,7 +25,6 @@ from webmentions_ssg.net import ( is_http_url, is_public_address, media_type_of, - normalized_hostname, read_limited_body, resolve_public_addresses, ) @@ -344,19 +343,6 @@ def test_is_retryable_status(app: Flask, status: int, expected: bool) -> None: @pytest.mark.parametrize( - ("url", "expected"), - [ - pytest.param("https://EXAMPLE.com./path", "example.com", id="trailing-dot"), - pytest.param("http://[2001:db8::1]/", "2001:db8::1", id="ipv6"), - pytest.param("https:///path", None, id="missing-host"), - pytest.param("http://[::1", None, id="malformed-ipv6"), - ], -) -def test_normalized_hostname(url: str, expected: str | None) -> None: - assert normalized_hostname(url) == expected - - -@pytest.mark.parametrize( ("headers", "expected"), [ pytest.param({}, "", id="missing"), diff --git a/tests/test_source_policy.py b/tests/test_source_policy.py new file mode 100644 index 0000000..cddfa3a --- /dev/null +++ b/tests/test_source_policy.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain normalization and receiver admission policy.""" + +import pytest +from flask import Flask + +from webmentions_ssg import db +from webmentions_ssg.models import AllowedSource, BlockedSource +from webmentions_ssg.source_policy import ( + source_domain_from_url, + source_rejection_reason, +) + + +def test_source_domain_from_url() -> None: + assert ( + source_domain_from_url("https://BÜCHER.example.:8443/post") + == "xn--bcher-kva.example" + ) + assert source_domain_from_url("https://[2001:db8::1]/post") == "2001:db8::1" + assert source_domain_from_url("invalid") is None + + +def test_blocked_domain_matches_itself_and_subdomains_but_not_suffixes( + app: Flask, +) -> None: + with app.app_context(): + db.session.add(BlockedSource(domain="example.org")) + db.session.commit() + for source in ( + "https://example.org/post", + "https://blog.example.org/post", + "https://A.blog.Example.ORG./post", + ): + assert source_rejection_reason(source) == "Source domain is blocked" + for source in ( + "https://notexample.org/post", + "https://example.org.evil/post", + "https://other.net/post", + ): + assert source_rejection_reason(source) is None + + +def test_allowlist_is_fail_closed_and_blocklist_takes_precedence(app: Flask) -> None: + app.config["SOURCE_ALLOWLIST_ENABLED"] = True + with app.app_context(): + assert ( + source_rejection_reason("https://example.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(AllowedSource(domain="example.org")) + db.session.commit() + assert source_rejection_reason("https://blog.example.org/post") is None + assert ( + source_rejection_reason("https://notexample.org/post") + == "Source domain is not allowlisted" + ) + db.session.add(BlockedSource(domain="blog.example.org")) + db.session.commit() + assert ( + source_rejection_reason("https://blog.example.org/post") + == "Source domain is blocked" + ) + assert source_rejection_reason("https://example.org/post") is None diff --git a/tests/views/test_admin.py b/tests/views/test_admin.py index a475ad5..175bcfd 100644 --- a/tests/views/test_admin.py +++ b/tests/views/test_admin.py @@ -795,11 +795,12 @@ def test_received_bulk_toolbar_and_shared_confirmation_modal( assert 'id="received-select-all"' in html assert 'id="received-selected-count"' in html assert 'data-received-bulk-action="reverify"' in html + assert 'data-received-bulk-action="block-sources"' in html assert 'data-received-bulk-action="delete"' in html assert 'data-title="Delete all declined webmentions?"' in html assert html.count('id="confirmation-modal"') == 1 assert 'id="confirmation-modal-form"' in html - assert 'received-bulk-actions.js' in html + assert "received-bulk-actions.js" in html def test_received_delete_all_declined_available_without_matching_rows( @@ -815,12 +816,18 @@ def test_received_delete_all_declined_available_without_matching_rows( assert 'data-title="Delete all declined webmentions?"' in html assert 'id="confirmation-modal-form"' in html assert 'data-received-bulk-action="delete"' not in html + assert 'data-received-bulk-action="block-sources"' not in html assert 'id="received-select-all"' not in html @pytest.mark.parametrize( "path", - ["/received/bulk/reverify", "/received/bulk/delete", "/received/declined/delete"], + [ + "/received/bulk/reverify", + "/received/bulk/block-sources", + "/received/bulk/delete", + "/received/declined/delete", + ], ) def test_received_bulk_actions_require_login(client: FlaskClient, path: str) -> None: response = client.post(path) @@ -831,7 +838,12 @@ def test_received_bulk_actions_require_login(client: FlaskClient, path: str) -> @pytest.mark.parametrize( "path", - ["/received/bulk/reverify", "/received/bulk/delete", "/received/declined/delete"], + [ + "/received/bulk/reverify", + "/received/bulk/block-sources", + "/received/bulk/delete", + "/received/declined/delete", + ], ) def test_received_bulk_actions_reject_invalid_form( app: Flask, @@ -854,7 +866,12 @@ def test_received_bulk_actions_reject_invalid_form( @pytest.mark.parametrize( "path", - ["/received/bulk/reverify", "/received/bulk/delete", "/received/declined/delete"], + [ + "/received/bulk/reverify", + "/received/bulk/block-sources", + "/received/bulk/delete", + "/received/declined/delete", + ], ) def test_received_bulk_actions_reject_invalid_filters_before_mutating( app: Flask, @@ -881,7 +898,7 @@ def test_received_bulk_actions_reject_invalid_filters_before_mutating( verify.assert_not_called() -@pytest.mark.parametrize("action", ["reverify", "delete"]) +@pytest.mark.parametrize("action", ["reverify", "block-sources", "delete"]) @pytest.mark.parametrize("selection", [[], ["bad-uuid"]]) def test_received_bulk_actions_reject_empty_or_malformed_selection( app: Flask, @@ -907,7 +924,7 @@ def test_received_bulk_actions_reject_empty_or_malformed_selection( verify.assert_not_called() -@pytest.mark.parametrize("action", ["reverify", "delete"]) +@pytest.mark.parametrize("action", ["reverify", "block-sources", "delete"]) def test_received_bulk_actions_reject_duplicate_oversized_or_stale_selection( app: Flask, client: FlaskClient, @@ -932,9 +949,7 @@ def test_received_bulk_actions_reject_duplicate_oversized_or_stale_selection( assert stale.status_code == HTTPStatus.NOT_FOUND app.config["RECEIVED_MAX_PAGE_SIZE"] = 1 - oversized = client.post( - path, data={"webmention_uuids": [str(first), str(second)]} - ) + oversized = client.post(path, data={"webmention_uuids": [str(first), str(second)]}) assert oversized.status_code == HTTPStatus.BAD_REQUEST with app.app_context(): @@ -1022,7 +1037,9 @@ def test_delete_all_declined_received_webmentions_ignores_age_and_filters( monkeypatch: pytest.MonkeyPatch, ) -> None: log_in(app, client) - failed = create_received_webmention(app, status="failed", updated_at=datetime.now(UTC)) + failed = create_received_webmention( + app, status="failed", updated_at=datetime.now(UTC) + ) removed = create_received_webmention( app, source="https://other.example/post", @@ -1086,8 +1103,7 @@ def test_received_bulk_delete_requires_real_csrf_token( html = client.get("/received").get_data(as_text=True) match = re.search( - r'<input\b(?=[^>]*name="csrf_token")(?=[^>]*value="([^"]+)")[^>]*>', - html, + r'<input\b(?=[^>]*name="csrf_token")(?=[^>]*value="([^"]+)")[^>]*>', html ) assert match is not None valid = client.post( diff --git a/tests/views/test_source_domains.py b/tests/views/test_source_domains.py new file mode 100644 index 0000000..c9e72b2 --- /dev/null +++ b/tests/views/test_source_domains.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated source-domain management and early endpoint rejection.""" + +import uuid +from datetime import UTC, datetime +from http import HTTPStatus +from unittest.mock import Mock + +import pytest +import sqlalchemy as sa +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import db +from webmentions_ssg.models import ( + AllowedSource, + BlockedSource, + ReceivedWebmention, + ReceivedWebmentionStatus, +) +from webmentions_ssg.tasks.verification import VerificationClaim + +from .helpers import SOURCE_URL, TARGET_URL, create_received_webmention, log_in + + +def test_source_rules_require_login(client: FlaskClient) -> None: + for method, path, data in ( + ("get", "/source-domains", None), + ("post", "/source-domains/blocked/add", {"domain": "bad.example"}), + ("post", "/source-domains/blocked/1/delete", None), + ): + response = client.open(path, method=method.upper(), data=data) + assert response.status_code == HTTPStatus.FOUND + assert "/login" in response.headers["Location"] + + +def test_add_and_unblock_source_domain(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + response = client.post( + "/source-domains/blocked/add", data={"domain": "SPAM.Example."} + ) + assert response.status_code == HTTPStatus.FOUND + response = client.post( + "/source-domains/blocked/add", data={"domain": "spam.example"} + ) + assert response.status_code == HTTPStatus.FOUND + + with app.app_context(): + rows = db.session.scalars(sa.select(BlockedSource)).all() + assert len(rows) == 1 + assert rows[0].domain == "spam.example" + entry_id = rows[0].id + + response = client.get("/source-domains") + assert response.status_code == HTTPStatus.OK + assert b"spam.example" in response.data + assert b"Unblock" in response.data + assert b"Source rules" in response.data + + response = client.post(f"/source-domains/blocked/{entry_id}/delete") + assert response.status_code == HTTPStatus.FOUND + with app.app_context(): + assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + + +def test_allowlist_can_be_managed_while_mode_is_off( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.post( + "/source-domains/allowed/add", data={"domain": "friends.example"} + ) + assert response.status_code == HTTPStatus.FOUND + with app.app_context(): + row = db.session.scalar(sa.select(AllowedSource)) + assert row is not None + assert row.domain == "friends.example" + assert b"disabled" in client.get("/source-domains").data + + +@pytest.mark.parametrize( + "domain", ["https://spam.example", "*.example.com", "example.com/path", "bad host"] +) +def test_invalid_domains_are_rejected( + client: FlaskClient, app: Flask, domain: str +) -> None: + log_in(app, client) + response = client.post("/source-domains/blocked/add", data={"domain": domain}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_invalid_list_kind_and_missing_entry(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + assert ( + client.post("/source-domains/nope/add", data={"domain": "test.org"}).status_code + == 404 + ) + assert client.post("/source-domains/blocked/123/delete").status_code == 404 + + +def test_received_row_can_block_domain(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + mention_id = create_received_webmention(app, source="https://spam.example/article") + response = client.get("/received") + assert b"Block this domain" in response.data + response = client.post(f"/received/{mention_id}/block-source") + assert response.status_code == HTTPStatus.FOUND + with app.app_context(): + rule = db.session.scalar(sa.select(BlockedSource)) + assert rule is not None and rule.domain == "spam.example" + + +def test_block_selected_source_domains_deduplicates_and_preserves_mentions( + app: Flask, client: FlaskClient +) -> None: + from urllib.parse import parse_qs, urlsplit + + log_in(app, client) + first = create_received_webmention(app, source="https://SPAM.Example./article") + same_domain = create_received_webmention( + app, source="https://spam.example/another-article" + ) + second = create_received_webmention(app, source="https://second.example/article") + unselected = create_received_webmention(app, source="https://other.example/article") + with app.app_context(): + db.session.add(BlockedSource(domain="second.example")) + db.session.commit() + + response = client.post( + "/received/bulk/block-sources", + query_string={"page": 3, "status": "verified", "sort": "source"}, + data={"webmention_uuids": [str(first), str(same_domain), str(second)]}, + ) + + assert response.status_code == HTTPStatus.FOUND + assert parse_qs(urlsplit(response.headers["Location"]).query) == { + "page": ["3"], + "status": ["verified"], + "sort": ["source"], + } + with app.app_context(): + assert set(db.session.scalars(sa.select(BlockedSource.domain))) == { + "spam.example", + "second.example", + } + for identifier in (first, same_domain, second, unselected): + assert db.session.get(ReceivedWebmention, identifier) is not None + from webmentions_ssg.source_policy import source_rejection_reason + + assert ( + source_rejection_reason("https://child.spam.example/post") + == "Source domain is blocked" + ) + assert source_rejection_reason("https://other.example/post") is None + + # Repeating the action is idempotent. + response = client.post( + "/received/bulk/block-sources", + data={"webmention_uuids": [str(first), str(same_domain)]}, + ) + assert response.status_code == HTTPStatus.FOUND + with app.app_context(): + assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 2 + + +def test_bulk_block_rejects_invalid_source_without_partial_changes( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + valid = create_received_webmention(app, source="https://valid.example/post") + invalid = create_received_webmention(app, source="not-a-url") + + response = client.post( + "/received/bulk/block-sources", + data={"webmention_uuids": [str(valid), str(invalid)]}, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + with app.app_context(): + assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + + +def test_bulk_block_action_enforces_csrf(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_received_webmention(app) + app.config["WTF_CSRF_ENABLED"] = True + + response = client.post( + "/received/bulk/block-sources", data={"webmention_uuids": str(identifier)} + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + with app.app_context(): + assert db.session.scalar(sa.select(sa.func.count(BlockedSource.id))) == 0 + + +def test_endpoint_rejects_blocked_without_queue_or_insert( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg.views import endpoint + + enqueue = Mock() + monkeypatch.setattr(endpoint, "verify_webmention", enqueue) + with app.app_context(): + db.session.add(BlockedSource(domain="source.example")) + db.session.commit() + + response = client.post( + "/endpoint", data={"source": SOURCE_URL, "target": TARGET_URL} + ) + assert response.status_code == HTTPStatus.FORBIDDEN + assert response.json == {"error": "Source domain is blocked"} + enqueue.assert_not_called() + with app.app_context(): + assert db.session.scalar(sa.select(sa.func.count(ReceivedWebmention.uuid))) == 0 + + +def test_allowlist_mode_rejects_unlisted_and_allows_listed( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg.views import endpoint + + app.config["SOURCE_ALLOWLIST_ENABLED"] = True + enqueue = Mock() + monkeypatch.setattr(endpoint, "verify_webmention", enqueue) + data = {"source": SOURCE_URL, "target": TARGET_URL} + assert client.post("/endpoint", data=data).status_code == HTTPStatus.FORBIDDEN + + with app.app_context(): + db.session.add(AllowedSource(domain="source.example")) + db.session.commit() + + assert client.post("/endpoint", data=data).status_code == HTTPStatus.CREATED + enqueue.assert_called_once() + + +def test_worker_checks_rules_before_fetch( + app: Flask, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg.tasks import verification + + claim = VerificationClaim( + webmention_uuid=uuid.uuid7(), + source=SOURCE_URL, + target=TARGET_URL, + lease_started_at=datetime.now(UTC), + ) + fetch = Mock() + monkeypatch.setattr(verification, "source_mentions_target", fetch) + with app.app_context(): + db.session.add(BlockedSource(domain="source.example")) + db.session.commit() + assert verification.determine_verification_outcome(claim) == ( + ReceivedWebmentionStatus.FAILED, + "Source domain is blocked", + ) + fetch.assert_not_called() + + +def test_queued_worker_skips_fetch_after_source_is_blocked( + app: Flask, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg.tasks import verification + + with app.app_context(): + mention = ReceivedWebmention( + uuid=uuid.uuid7(), source=SOURCE_URL, target=TARGET_URL + ) + db.session.add(mention) + db.session.commit() + identifier = mention.uuid + + # A user blocks the domain after the submission has already been queued. + db.session.add(BlockedSource(domain="source.example")) + db.session.commit() + + fetch = Mock() + monkeypatch.setattr(verification, "source_mentions_target", fetch) + verification.verify_webmention.call_local(identifier) + fetch.assert_not_called() + with app.app_context(): + mention = db.session.get(ReceivedWebmention, identifier) + assert mention is not None + assert mention.status == ReceivedWebmentionStatus.FAILED + assert mention.failure_reason == "Source domain is blocked" + + +def test_admin_actions_enforce_csrf(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + app.config["WTF_CSRF_ENABLED"] = True + assert ( + client.post( + "/source-domains/blocked/add", data={"domain": "spam.example"} + ).status_code + == HTTPStatus.BAD_REQUEST + ) diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 1419b82..576f376 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -9,6 +9,8 @@ from urllib.parse import urlsplit from flask.config import Config +from .hostname import normalize_hostname + MIN_SECRET_KEY_LENGTH = 32 @@ -52,6 +54,9 @@ class DefaultConfig: # Receiver # Empty by default: no target hostnames are accepted until configured. ALLOWED_TARGET_HOSTNAMES = frozenset() + # When enabled, only source domains listed in AllowedSource are accepted. + # Entries in BlockedSource always take precedence. + SOURCE_ALLOWLIST_ENABLED = False # Cap incoming HTTP request bodies (also applies to other Flask routes). MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. @@ -137,29 +142,6 @@ class TestingConfig(DefaultConfig): ALLOWED_TARGET_HOSTNAMES = frozenset({"dennisfink.me"}) -def normalize_hostnames(value: object) -> set[str]: - """Validate and normalize a collection of hostnames. - - :param value: List, tuple, set, or frozenset of hostnames to normalize. - :returns: Normalized, lowercase hostnames without trailing dots. - :raises TypeError: If the input is not a supported collection type. - :raises ValueError: If a collection item is not a nonempty hostname string. - """ - if not isinstance(value, (list, tuple, set, frozenset)): - raise TypeError("Expected a list, tuple, or set of hostnames.") - - hostnames: set[str] = set() - for hostname in value: - if not isinstance(hostname, str) or not ( - normalized := hostname.strip().lower().rstrip(".") - ): - raise ValueError("Hostnames must be non-empty strings.") - - hostnames.add(normalized) - - return hostnames - - def validate_secret_key(value: object) -> str | bytes: """Require a non-placeholder, sufficiently long deployment signing key. @@ -214,10 +196,26 @@ def validated_config_overrides( normalized: dict[str, object] = {} for key in ("ALLOWED_TARGET_HOSTNAMES", "IGNORED_TARGET_HOSTNAMES"): try: - normalized[key] = normalize_hostnames(config[key]) + value = config[key] + if not isinstance(value, (list, tuple, set, frozenset)): + raise TypeError("Expected a list, tuple, or set of hostnames.") + hostnames: set[str] = set() + for hostname in value: + if not isinstance(hostname, str) or not hostname.strip(): + raise ValueError("Hostnames must be non-empty strings.") + hostnames.add( + normalize_hostname( + hostname.strip(), + allow_patterns=key == "IGNORED_TARGET_HOSTNAMES", + ) + ) + normalized[key] = hostnames except (TypeError, ValueError) as exc: raise type(exc)(f"{key}: {exc}") from exc + if type(config["SOURCE_ALLOWLIST_ENABLED"]) is not bool: + raise ValueError("SOURCE_ALLOWLIST_ENABLED must be a boolean.") + proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] if type(proxy_hops) is not int or proxy_hops < 0: raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.") diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py index ae969b3..3b61a8a 100644 --- a/webmentions_ssg/forms.py +++ b/webmentions_ssg/forms.py @@ -30,7 +30,8 @@ from wtforms.validators import ( StopValidation, ) -from .net import is_public_address, normalized_hostname +from .hostname import normalize_hostname +from .net import is_public_address from .validation import is_control_character RECEIVED_STATUS_CHOICES = { @@ -120,11 +121,12 @@ class AllowedTargetHostname: :param field: Field containing the URL to validate. :raises ValidationError: If the URL hostname is not configured as allowed. """ - hostname = normalized_hostname(field.data) - if ( - hostname is not None - and hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"] - ): + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError: + hostname = None + + if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: return raise ValidationError(self.message or field.gettext("Invalid input.")) @@ -133,9 +135,10 @@ class RejectNonPublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" def __call__(self, form: BaseForm, field: Field) -> None: - hostname = normalized_hostname(field.data) - if hostname is None: - raise ValidationError(field.gettext("Invalid input.")) + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError as exc: + raise ValidationError(field.gettext("Invalid input.")) from exc try: address = ipaddress.ip_address(hostname) @@ -249,3 +252,17 @@ class ConfirmActionForm(FlaskForm): """Form for confirming an administrative action.""" submit = SubmitField("Confirm") + + +class SourceDomainForm(FlaskForm): + """Add a domain to the source blocklist or allowlist.""" + + domain = StringField("Source domain", validators=[InputRequired()]) + submit = SubmitField("Add domain") + + def validate_domain(self, field: Field) -> None: + """Require a bare hostname and persist its canonical form.""" + try: + field.data = normalize_hostname(field.data) + except ValueError as exc: + raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/hostname.py b/webmentions_ssg/hostname.py new file mode 100644 index 0000000..4e55069 --- /dev/null +++ b/webmentions_ssg/hostname.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical hostname normalization shared by URL, config, and source rules.""" + +import ipaddress +import re +from urllib.parse import urlsplit + +LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") +PATTERN_LABEL = re.compile(r"[a-z0-9*?\[\]-]+\Z") + + +def normalize_hostname( + value: str, *, from_url: bool = False, allow_patterns: bool = False +) -> str: + """Return the canonical IDNA hostname or IP literal. + + Bare hostnames are required by default. ``from_url=True`` extracts a + hostname from a URL, including bracketed IPv6 literals. ``allow_patterns`` + permits sender-side ignored-hostname glob patterns; it must not be used + for receiver allow/block entries or allowed target hosts. + + Whitespace and malformed hosts are rejected. Callers normalizing config + collections may strip surrounding whitespace before calling this function. + + :raises ValueError: If the hostname is missing or malformed. + """ + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError("Enter a hostname, without a URL or whitespace.") + + if from_url: + try: + hostname = urlsplit(value).hostname + except ValueError as exc: + raise ValueError("Invalid hostname.") from exc + if hostname is None: + raise ValueError("Invalid hostname.") + else: + hostname = value + + hostname = hostname.lower().rstrip(".") + try: + return str(ipaddress.ip_address(hostname)) + except ValueError: + pass + + try: + hostname = hostname.encode("idna").decode("ascii").lower() + except (UnicodeError, ValueError) as exc: + raise ValueError("Invalid hostname.") from exc + + label_pattern = PATTERN_LABEL if allow_patterns else LABEL + if ( + not hostname + or len(hostname) > 253 + or any(not label_pattern.fullmatch(label) for label in hostname.split(".")) + ): + raise ValueError("Invalid hostname.") + return hostname diff --git a/webmentions_ssg/migrations/versions/3e8c87a96b2d_source_domain_rules.py b/webmentions_ssg/migrations/versions/3e8c87a96b2d_source_domain_rules.py new file mode 100644 index 0000000..04f5b8d --- /dev/null +++ b/webmentions_ssg/migrations/versions/3e8c87a96b2d_source_domain_rules.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Add persistent source-domain block and allow lists. + +Revision ID: 3e8c87a96b2d +Revises: e62b1a7c4d90 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "3e8c87a96b2d" +down_revision = "e62b1a7c4d90" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "blocked_sources", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("domain", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("domain"), + ) + op.create_table( + "allowed_sources", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("domain", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("domain"), + ) + + +def downgrade(): + op.drop_table("allowed_sources") + op.drop_table("blocked_sources") diff --git a/webmentions_ssg/migrations/versions/b8179a2e46c3_merge_domain_rules_and_name_changes.py b/webmentions_ssg/migrations/versions/b8179a2e46c3_merge_domain_rules_and_name_changes.py new file mode 100644 index 0000000..860e042 --- /dev/null +++ b/webmentions_ssg/migrations/versions/b8179a2e46c3_merge_domain_rules_and_name_changes.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Merge source-domain rules with Webmention lifecycle naming changes. + +Revision ID: b8179a2e46c3 +Revises: 3e8c87a96b2d, 9a4e0c8211d6 +""" + +revision = "b8179a2e46c3" +down_revision = ("3e8c87a96b2d", "9a4e0c8211d6") +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Both parent migrations have already applied their schema changes.""" + + +def downgrade() -> None: + """Downgrading past the merge leaves Alembic's two parent branches.""" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 2b375dd..2304788 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -101,6 +101,24 @@ class User(UserMixin, Base): return check_password_hash(self.password_hash, password) +class BlockedSource(Base): + """A source domain denied before verification or outbound fetching.""" + + __tablename__ = "blocked_sources" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + +class AllowedSource(Base): + """A source domain accepted when allowlist mode is enabled.""" + + __tablename__ = "allowed_sources" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + class ReceivedWebmention(Base): """Represent a Webmention received for verification.""" diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py index b827180..df62ca7 100644 --- a/webmentions_ssg/net.py +++ b/webmentions_ssg/net.py @@ -279,19 +279,6 @@ def resolve_public_addresses( return tuple(str(address) for address in addresses) -def normalized_hostname(url: str) -> str | None: - """Return a normalized URL hostname, or None for malformed URLs. - - :param url: URL from which to extract a hostname. - :returns: Normalized hostname, or ``None`` when the URL is invalid or has no host. - """ - try: - hostname = urlsplit(url).hostname - except TypeError, ValueError: - return None - return hostname.rstrip(".") if hostname is not None else None - - def media_type_of(headers: Mapping[str, str]) -> str: """Return a normalized media type without Content-Type parameters. diff --git a/webmentions_ssg/source_policy.py b/webmentions_ssg/source_policy.py new file mode 100644 index 0000000..2c62527 --- /dev/null +++ b/webmentions_ssg/source_policy.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Admission policy for incoming Webmention source hostnames.""" + +import ipaddress + +import sqlalchemy as sa +from flask import current_app + +from .extensions import db +from .hostname import normalize_hostname +from .models import AllowedSource, BlockedSource + + +def source_domain_from_url(source: str) -> str | None: + """Extract and canonicalize the source hostname without a network request.""" + try: + return normalize_hostname(source, from_url=True) + except ValueError: + return None + + +def has_matching_domain( + model: type[AllowedSource | BlockedSource], domain: str +) -> bool: + """Match the hostname itself and parent domains, never partial labels.""" + candidates = [domain] + # An IP address can only match itself, not a suffix of its textual form. + try: + ipaddress.ip_address(domain) + except ValueError: + parts = domain.split(".") + candidates.extend(".".join(parts[i:]) for i in range(1, len(parts))) + + return ( + db.session.scalar( + sa.select(model.id).where(model.domain.in_(candidates)).limit(1) + ) + is not None + ) + + +def source_rejection_reason(source: str) -> str | None: + """Return the reason a source cannot be received, or None when accepted. + + The blocklist always wins. In allowlist mode an empty allowlist rejects + every source. The database, not a stale per-worker cache, is consulted. + """ + if (domain := source_domain_from_url(source)) is None: + return "Invalid source hostname" + + if has_matching_domain(BlockedSource, domain): + return "Source domain is blocked" + + if current_app.config["SOURCE_ALLOWLIST_ENABLED"] and not has_matching_domain( + AllowedSource, domain + ): + return "Source domain is not allowlisted" + + return None diff --git a/webmentions_ssg/static/js/received-bulk-actions.js b/webmentions_ssg/static/js/received-bulk-actions.js index c4ac31b..06ab912 100644 --- a/webmentions_ssg/static/js/received-bulk-actions.js +++ b/webmentions_ssg/static/js/received-bulk-actions.js @@ -49,8 +49,11 @@ document.addEventListener("DOMContentLoaded", () => { const selection = selected(); const message = modal.querySelector("#confirmation-modal-message"); - if (event.relatedTarget.dataset.receivedBulkAction === "delete") { + const action = event.relatedTarget.dataset.receivedBulkAction; + if (action === "delete") { message.textContent = `This permanently deletes ${selection.length} selected webmentions and cannot be undone.`; + } else if (action === "block-sources") { + message.textContent = `Source domains from ${selection.length} selected webmentions (and their subdomains) will be blocked. Existing webmentions will be kept.`; } else { message.textContent = `${selection.length} selected webmentions will be fetched and verified again.`; } diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index e063525..bd0700a 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -18,7 +18,8 @@ import xxhash from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement -from ..net import is_http_url, normalized_hostname +from ..hostname import normalize_hostname +from ..net import is_http_url REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") @@ -284,7 +285,10 @@ def iter_outgoing_targets( for value in chain(hrefs, reactions): target = normalize_target_url(value, base_url=base_url, source_url=source_url) if target is not None: - hostname = normalized_hostname(target) or "" + try: + hostname = normalize_hostname(target, from_url=True) + except ValueError: + continue if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): yield target diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index b0f4d0f..d55608c 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -25,10 +25,10 @@ from ..net import ( media_type_of, read_limited_body, ) +from ..source_policy import source_rejection_reason from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target - RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" @@ -200,6 +200,10 @@ def determine_verification_outcome( retried. :raises httpx.RequestError: If a transient HTTP request error should be retried. """ + # Recheck at worker time: a source may have been blocked since submission. + if reason := source_rejection_reason(claim.source): + return ReceivedWebmentionStatus.FAILED, reason + try: mentions_target = source_mentions_target(claim.source, claim.target) except SourceGoneError as exc: diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index 69e5215..d9c8e87 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -45,6 +45,8 @@ SPDX-License-Identifier: BSD-3-Clause <div class="navbar-nav flex-row gap-3"> <a class="nav-link{% if request.endpoint == 'webmentions.received_list' %} active{% endif %}" href='{{ url_for("webmentions.received_list") }}'>Received</a> + <a class="nav-link{% if request.endpoint and request.endpoint.startswith('webmentions.source_domain') %} active{% endif %}" + href="{{ url_for('webmentions.source_domain_list') }}">Source rules</a> <a class="nav-link{% if request.endpoint == 'webmentions.source_list' %} active{% endif %}" href='{{ url_for("webmentions.source_list") }}'>Sent</a> <form action="{{ url_for('webmentions.logout') }}" diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index de3a9b6..5fd9866 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -127,6 +127,17 @@ SPDX-License-Identifier: BSD-3-Clause data-confirm-label="Reverify selected" data-confirm-class="btn-primary">Reverify selected</button> <button type="button" + class="btn btn-sm btn-outline-warning" + disabled + data-bs-toggle="modal" + data-bs-target="#confirmation-modal" + data-received-bulk-action="block-sources" + data-action="{{ url_for('webmentions.block_selected_received_sources', page=webmentions.page, **filters) }}" + data-title="Block selected source domains?" + data-message="Selected source domains and their subdomains will be blocked from future Webmentions. Existing Webmentions will be kept." + data-confirm-label="Block selected domains" + data-confirm-class="btn-warning">Block selected source domains</button> + <button type="button" class="btn btn-sm btn-outline-danger" disabled data-bs-toggle="modal" @@ -206,6 +217,15 @@ SPDX-License-Identifier: BSD-3-Clause data-confirm-label="Reverify" data-confirm-class="btn-primary">Reverify</button> <button type="button" + class="btn btn-sm btn-warning" + data-bs-toggle="modal" + data-bs-target="#confirmation-modal" + data-action="{{ url_for('webmentions.block_received_source', webmention_uuid=webmention.uuid, page=webmentions.page, **filters) }}" + data-title="Block this source domain?" + data-message="Future Webmentions from this source hostname and its subdomains will be rejected." + data-confirm-label="Block domain" + data-confirm-class="btn-warning">Block this domain</button> + <button type="button" class="btn btn-sm btn-danger" data-bs-toggle="modal" data-bs-target="#confirmation-modal" diff --git a/webmentions_ssg/templates/source_domains.html b/webmentions_ssg/templates/source_domains.html new file mode 100644 index 0000000..2db942f --- /dev/null +++ b/webmentions_ssg/templates/source_domains.html @@ -0,0 +1,68 @@ +<!-- +SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> + +SPDX-License-Identifier: BSD-3-Clause +--> +{% extends "base.html" %} + +{% block content %} + <h1 class="mb-3">Source domain rules</h1> + <p> + Entries match the named hostname and all of its subdomains (not similarly named domains). + Blocked domains always take precedence over allowed domains. Changes affect new + submissions and future verification attempts; existing mentions are not deleted. + </p> + <p> + Allowlist mode is <strong>{{ "enabled" if allowlist_enabled else "disabled" }}</strong>. + Set <code>SOURCE_ALLOWLIST_ENABLED = True</code> in the server configuration to + accept only listed source domains. An empty enabled allowlist rejects all sources. + </p> + {% for kind, entries in (("blocked", blocked), ("allowed", allowed)) %} + <section class="mb-4" aria-labelledby="{{ kind }}-heading"> + <h2 id="{{ kind }}-heading">{{ "Blocked" if kind == "blocked" else "Allowed" }} domains</h2> + <form action="{{ url_for('webmentions.source_domain_add', kind=kind) }}" + method="post" + class="row g-2 align-items-end mb-3"> + <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> + <div class="col-sm-8 col-md-5"> + <label class="form-label" for="{{ kind }}-domain">Source domain</label> + <input class="form-control" + type="text" + id="{{ kind }}-domain" + name="domain" + placeholder="example.org" + required> + </div> + <div class="col-auto"> + <button type="submit" class="btn btn-primary">Add domain</button> + </div> + </form> + {% if entries %} + <table class="table table-striped align-middle"> + <thead> + <tr> + <th scope="col">Domain</th> + <th scope="col">Action</th> + </tr> + </thead> + <tbody> + {% for entry in entries %} + <tr> + <td class="text-break">{{ entry.domain }}</td> + <td> + <form action="{{ url_for('webmentions.source_domain_delete', kind=kind, entry_id=entry.id) }}" + method="post"> + <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"> + <button type="submit" class="btn btn-sm btn-outline-danger">{{ "Unblock" if kind == "blocked" else "Remove" }}</button> + </form> + </td> + </tr> + {% endfor %} + </tbody> + </table> + {% else %} + <p class="text-body-secondary">No {{ kind }} domains.</p> + {% endif %} + </section> + {% endfor %} +{% endblock %} diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py index 6754443..eb5a5e7 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/admin.py @@ -13,17 +13,21 @@ import sqlalchemy as sa from flask import abort, current_app, flash, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from flask_login import login_required +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import selectinload from .. import forms from ..db_helpers import rows_affected from ..extensions import db from ..models import ( + AllowedSource, + BlockedSource, ReceivedWebmention, ReceivedWebmentionStatus, SentWebmention, SourceDocument, ) +from ..source_policy import source_domain_from_url from ..tasks.scanner import manual_scan_sources from ..tasks.sender import send_webmention from ..tasks.verification import verify_webmention @@ -38,6 +42,7 @@ RECEIVED_STATUS_FILTERS = { ReceivedWebmentionStatus.SOURCE_REMOVED, ), } +SOURCE_DOMAIN_MODELS = {"blocked": BlockedSource, "allowed": AllowedSource} class ReceivedFilterArgs(TypedDict, total=False): @@ -344,6 +349,141 @@ def delete_all_declined_received_webmentions() -> ResponseReturnValue: ) +def block_source_domains(domains: set[str]) -> int: + """Persist distinct source domains, tolerating concurrent blocklist inserts. + + Savepoints isolate unique-constraint races so one preexisting entry cannot + prevent other selected domains from being blocked. + """ + existing = set( + db.session.scalars( + sa.select(BlockedSource.domain).where(BlockedSource.domain.in_(domains)) + ) + ) + added = 0 + for domain in sorted(domains - existing): + try: + with db.session.begin_nested(): + db.session.add(BlockedSource(domain=domain)) + db.session.flush() + except IntegrityError: + # A concurrent admin action has already blocked this hostname. + continue + added += 1 + db.session.commit() + return added + + +@webmentions_bp.post("/received/bulk/block-sources") +@login_required +def block_selected_received_sources() -> ResponseReturnValue: + """Block the unique source domains of selected received Webmentions.""" + if not forms.ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + filters = received_filter_args(received_filter_form()) + webmentions = selected_received_webmentions() + domains = {source_domain_from_url(webmention.source) for webmention in webmentions} + if None in domains: + # Fail the entire selection instead of silently skipping a source. + abort(HTTPStatus.BAD_REQUEST) + + added = block_source_domains(cast(set[str], domains)) + flash( + f"{added} source domains newly blocked from {len(webmentions)} selected Webmentions.", + "success", + ) + return redirect( + url_for( + "webmentions.received_list", + page=request.args.get("page", 1, type=int), + **filters, + ) + ) + + +@webmentions_bp.post("/received/<uuid:webmention_uuid>/block-source") +@login_required +def block_received_source(webmention_uuid: uuid.UUID) -> ResponseReturnValue: + """Add a received Webmention's source hostname to the blocklist.""" + if not forms.ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + filters = received_filter_args(received_filter_form()) + webmention = db.get_or_404(ReceivedWebmention, webmention_uuid) + if (domain := source_domain_from_url(webmention.source)) is None: + abort(HTTPStatus.BAD_REQUEST) + + block_source_domains({domain}) + + flash(f"Source domain {domain} blocked.", "success") + return redirect( + url_for( + "webmentions.received_list", + page=request.args.get("page", 1, type=int), + **filters, + ) + ) + + +@webmentions_bp.get("/source-domains") +@login_required +def source_domain_list() -> ResponseReturnValue: + """Manage blocklist and allowlist entries used by the receiver.""" + return render_template( + "source_domains.html", + title="Source domain rules", + blocked=db.session.scalars( + sa.select(BlockedSource).order_by(BlockedSource.domain) + ).all(), + allowed=db.session.scalars( + sa.select(AllowedSource).order_by(AllowedSource.domain) + ).all(), + allowlist_enabled=current_app.config["SOURCE_ALLOWLIST_ENABLED"], + ) + + +@webmentions_bp.post("/source-domains/<kind>/add") +@login_required +def source_domain_add(kind: str) -> ResponseReturnValue: + """Validate and add one normalized hostname to the selected list.""" + if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + form = forms.SourceDomainForm() + if not form.validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + domain = cast(str, form.domain.data) + if db.session.scalar(sa.select(model.id).where(model.domain == domain)) is None: + db.session.add(model(domain=domain)) + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + + flash(f"Source domain {domain} added to {kind} list.", "success") + return redirect(url_for("webmentions.source_domain_list")) + + +@webmentions_bp.post("/source-domains/<kind>/<int:entry_id>/delete") +@login_required +def source_domain_delete(kind: str, entry_id: int) -> ResponseReturnValue: + """Remove one entry; a blocked domain becomes eligible for new submissions.""" + if (model := SOURCE_DOMAIN_MODELS.get(kind)) is None: + abort(HTTPStatus.NOT_FOUND) + + if not forms.ConfirmActionForm().validate_on_submit(): + abort(HTTPStatus.BAD_REQUEST) + + entry = db.get_or_404(model, entry_id) + domain = entry.domain + db.session.delete(entry) + db.session.commit() + flash(f"Source domain {domain} removed from {kind} list.", "success") + return redirect(url_for("webmentions.source_domain_list")) + + @webmentions_bp.route("/sources") @login_required def source_list() -> ResponseReturnValue: diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index 934ffad..6d84d92 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -18,6 +18,7 @@ from .. import forms from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter from ..models import ReceivedWebmention, source_target_digest +from ..source_policy import source_rejection_reason from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -152,6 +153,9 @@ def receive_webmention() -> ResponseReturnValue: source = cast(str, form.source.data) target = cast(str, form.target.data) + if reason := source_rejection_reason(source): + return {"error": reason}, HTTPStatus.FORBIDDEN + result = record_submission(source, target) if result is None: |
