diff options
| author | Dennis Fink | 2026-10-31 17:42:00 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-31 17:42:00 +0100 |
| commit | 6924ad7e52033365c55d5afacdd916c502cc80cb (patch) | |
| tree | 8a52b8e8fbae7a6bcc85d250cac2436a2c42e748 /tests | |
| parent | 3494eab1208dc18478485e02c61e292556502fa6 (diff) | |
| download | webmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.tar.gz webmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.zip | |
feat(receiver): add resource limits and retention
Bound incoming request and URL sizes so untrusted Webmention submissions
cannot consume unbounded application or database resources.
Purge old failed and deleted received Webmentions on a schedule, and use
portable SHA-256 URL fingerprints to preserve uniqueness for long URLs.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/cli/test_database.py | 44 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 23 | ||||
| -rw-r--r-- | tests/forms/test_validators.py | 32 | ||||
| -rw-r--r-- | tests/tasks/test_consumer.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_retention.py | 69 | ||||
| -rw-r--r-- | tests/test_config.py | 22 | ||||
| -rw-r--r-- | tests/test_url_digest.py | 100 | ||||
| -rw-r--r-- | tests/test_views.py | 12 |
8 files changed, 305 insertions, 5 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 1c36d1f..7095d51 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -213,3 +213,47 @@ def test_create_db_preserves_logging( database_app.logger.warning("Logging still works after migration") assert "Logging still works after migration" in caplog.text + + +def test_upgrade_backfills_portable_digest_for_existing_received_rows( + database_app: Flask, +) -> None: + """Upgrade a pre-fingerprint SQLite database without losing historical rows.""" + from webmentions_ssg.url_digest import received_url_digest + + with database_app.app_context(): + migrate_upgrade(revision="c0272f25ce17") + with DATABASE.engine.begin() as connection: + connection.execute( + sa.text( + "INSERT INTO received_webmentions " + "(uuid, source, target, status, updated_at) " + "VALUES (:uuid, :source, :target, :status, :updated_at)" + ), + { + "uuid": "0" * 31 + "1", + "source": "https://source.example/old", + "target": "https://dennisfink.me/target", + "status": "verified", + "updated_at": "2026-01-01 00:00:00", + }, + ) + migrate_upgrade() + with DATABASE.engine.connect() as connection: + result = connection.execute( + sa.text( + "SELECT source, target, status, url_digest " + "FROM received_webmentions" + ) + ).one() + assert result.status == "verified" + assert result.url_digest == received_url_digest( + result.source, result.target + ) + constraints = inspect(connection).get_unique_constraints( + "received_webmentions" + ) + assert any( + constraint["column_names"] == ["url_digest"] + for constraint in constraints + ) diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index 0f5cb23..2a68509 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -145,3 +145,26 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: ) assert not form.validate() assert "Invalid input." in form.errors["target"] + + +def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( + app: Flask, +) -> None: + with app.test_request_context("/endpoint", method="POST"): + app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_SOURCE.encode("utf-8")) + form = EndpointForm( + formdata=MultiDict({"source": VALID_SOURCE + "x", "target": VALID_TARGET}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "source" in form.errors + assert any("bytes" in message for message in form.errors["source"]) + + app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_TARGET.encode("utf-8")) + form = EndpointForm( + formdata=MultiDict({"source": VALID_SOURCE, "target": VALID_TARGET + "x"}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "target" in form.errors + assert any("bytes" in message for message in form.errors["target"]) diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py index 2858b4c..10796e9 100644 --- a/tests/forms/test_validators.py +++ b/tests/forms/test_validators.py @@ -9,7 +9,12 @@ from flask import Flask from werkzeug.datastructures import MultiDict from wtforms import Form, StringField, ValidationError -from webmentions_ssg.forms.validators import AllowedHostname, NotEqualTo, PublicURL +from webmentions_ssg.forms.validators import ( + AllowedHostname, + NotEqualTo, + PublicURL, + URLMaxBytes, +) from webmentions_ssg.url_security import AddressResolutionError @@ -118,3 +123,28 @@ def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: form = URLForm(MultiDict({"url": "http://[::1"})) with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): AllowedHostname()(form, form.url) + + +def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: + validator = URLMaxBytes() + form = URLForm(MultiDict({"url": "é" * 3})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 6 + validator(form, form.url) + app.config["MAX_WEBMENTION_URL_BYTES"] = 5 + with pytest.raises(ValidationError, match="5 bytes"): + validator(form, form.url) + + +def test_url_max_bytes_custom_message(app: Flask) -> None: + form = URLForm(MultiDict({"url": "too-long"})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 2 + with pytest.raises(ValidationError, match="URL too long"): + URLMaxBytes(message="URL too long")(form, form.url) + + +def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: + form = URLForm(MultiDict({"url": "\ud800"})) + with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): + URLMaxBytes()(form, form.url) diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py index dd93331..7604691 100644 --- a/tests/tasks/test_consumer.py +++ b/tests/tasks/test_consumer.py @@ -13,7 +13,7 @@ from flask import Flask from webmentions_ssg import HUEY -def test_consumer_registers_scanner_digest_and_verification_reaper( +def test_consumer_registers_scanner_digest_reaper_and_retention( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: import webmentions_ssg @@ -21,6 +21,7 @@ def test_consumer_registers_scanner_digest_and_verification_reaper( app.config["SCANNER_SCHEDULE"] = "*/15 * * * *" app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *" app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *" + app.config["RECEIVED_RETENTION_SCHEDULE"] = "0 4 * * *" registered_tasks: list[object] = [] @@ -36,12 +37,13 @@ def test_consumer_registers_scanner_digest_and_verification_reaper( monkeypatch.setattr(HUEY, "periodic_task", periodic_task) consumer = importlib.import_module("webmentions_ssg.tasks.consumer") - from webmentions_ssg.tasks import notifications, receiver, scanner + from webmentions_ssg.tasks import notifications, receiver, retention, scanner assert consumer.huey is HUEY.huey - assert periodic_task.call_count == 3 + assert periodic_task.call_count == 4 assert registered_tasks == [ scanner.scan_sources, receiver.recover_stale_webmentions, notifications.send_webmention_digest, + retention.purge_old_received_webmentions, ] diff --git a/tests/tasks/test_retention.py b/tests/tasks/test_retention.py new file mode 100644 index 0000000..3f421f9 --- /dev/null +++ b/tests/tasks/test_retention.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test scheduled removal of terminal received Webmentions.""" + +import uuid +from datetime import UTC, datetime, timedelta + +import sqlalchemy as sa +from flask import Flask + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.tasks.retention import purge_old_received_webmentions + + +def test_retention_only_removes_old_failed_or_deleted_rows(app: Flask) -> None: + now = datetime.now(UTC) + app.config["RECEIVED_RETENTION_DAYS"] = 30 + identifiers = {} + with app.app_context(): + for suffix, status, age_days in ( + ("old-failed", "failed", 31), + ("old-deleted", "deleted", 31), + ("recent-failed", "failed", 2), + ("recent-deleted", "deleted", 2), + ("old-verified", "verified", 50), + ("old-received", "received", 50), + ("old-verifying", "verifying", 50), + ): + identifier = uuid.uuid7() + identifiers[suffix] = identifier + db.session.add( + ReceivedWebmention( + uuid=identifier, + source=f"https://source.example/{suffix}", + target="https://dennisfink.me/target", + status=status, + updated_at=now - timedelta(days=age_days), + ) + ) + db.session.commit() + + assert purge_old_received_webmentions() == 2 + assert purge_old_received_webmentions() == 0 + + remaining = set(db.session.scalars(sa.select(ReceivedWebmention.uuid))) + assert remaining == { + identifier + for key, identifier in identifiers.items() + if key not in {"old-failed", "old-deleted"} + } + + +def test_retention_uses_admin_configured_days(app: Flask) -> None: + app.config["RECEIVED_RETENTION_DAYS"] = 7 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/one", + target="https://dennisfink.me/target", + status="failed", + updated_at=datetime.now(UTC) - timedelta(days=10), + ) + ) + db.session.commit() + assert purge_old_received_webmentions() == 1 diff --git a/tests/test_config.py b/tests/test_config.py index dd02c01..933f437 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -232,3 +232,25 @@ def test_development_rejects_placeholder_even_with_debug(tmp_path: Path) -> None with pytest.raises(RuntimeError, match="SECRET_KEY"): create_app(LocalDevelopment) + + +def test_resource_limits_have_safe_defaults() -> None: + assert DefaultConfig.MAX_CONTENT_LENGTH == 64 * 1024 + assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192 + assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30 + assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *" + + +@pytest.mark.parametrize( + "key", ["MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS"] +) +@pytest.mark.parametrize("value", [None, "100", 0, -1, True, 1.5]) +def test_invalid_resource_limits_fail_at_startup( + tmp_path: Path, key: str, value: object +) -> None: + class InvalidResourceLimit(TestingConfig): + LOG_FILE = str(tmp_path / "invalid-resource.log") + + setattr(InvalidResourceLimit, key, value) + with pytest.raises(ValueError, match=key): + create_app(InvalidResourceLimit) diff --git a/tests/test_url_digest.py b/tests/test_url_digest.py new file mode 100644 index 0000000..0c04364 --- /dev/null +++ b/tests/test_url_digest.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test the database-independent Webmention URL fingerprints.""" + +import uuid +from datetime import UTC, datetime + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention, SentWebmention, Source +from webmentions_ssg.url_digest import received_url_digest, sent_target_digest + + +def test_received_pair_fingerprint_is_unambiguous() -> None: + assert received_url_digest("a", "bc") != received_url_digest("ab", "c") + assert received_url_digest("a", "b") != received_url_digest("b", "a") + + +def test_models_hash_url_fields_when_created_or_changed() -> None: + received = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/a", + target="https://site.example/b", + ) + assert received.url_digest == received_url_digest(received.source, received.target) + received.source = "https://source.example/c" + assert received.url_digest == received_url_digest(received.source, received.target) + received.target = "https://site.example/d" + assert received.url_digest == received_url_digest(received.source, received.target) + + sent = SentWebmention(target="https://site.example/a", desired_revision=1) + assert sent.target_digest == sent_target_digest(sent.target) + sent.target = "https://site.example/b" + assert sent.target_digest == sent_target_digest(sent.target) + + +def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: + long_source = "https://source.example/" + "a" * 4000 + long_target = "https://dennisfink.me/" + "b" * 4000 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + db.session.commit() + match = db.session.scalar( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.url_digest + == received_url_digest(long_source, long_target) + ) + ) + assert match is not None + assert match.source == long_source + assert match.target == long_target + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() + + +def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( + app: Flask, +) -> None: + now = datetime.now(UTC) + long_target = "https://destination.example/" + "a" * 4000 + with app.app_context(): + source = Source( + path="long/index.html", + url="https://example.com/long", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + db.session.add(source) + db.session.commit() + assert source.sent_webmentions[0].target_digest == sent_target_digest( + long_target + ) + + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/test_views.py b/tests/test_views.py index 21a33bf..f5e301f 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -760,7 +760,7 @@ def test_endpoint_recovers_from_concurrent_insert( initial_result = Mock() initial_result.scalar_one_or_none.return_value = None competing_result = Mock() - competing_result.scalar_one.return_value = competing_webmention + competing_result.scalar_one_or_none.return_value = competing_webmention execute = Mock(side_effect=[initial_result, competing_result]) add = Mock() @@ -1184,3 +1184,13 @@ def test_admin_only_links_safe_existing_status_urls( html = response.get_data(as_text=True) assert (f'href="{status_url}"' in html) is safe assert status_url in html + + +def test_endpoint_rejects_request_bodies_over_configured_maximum( + app: Flask, client: FlaskClient +) -> None: + app.config["MAX_CONTENT_LENGTH"] = 128 + response = client.post( + "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} + ) + assert response.status_code == 413 |
