diff options
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 13 | ||||
| -rw-r--r-- | tests/cli/test_database.py | 44 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 23 | ||||
| -rw-r--r-- | tests/forms/test_validators.py | 32 | ||||
| -rw-r--r-- | tests/tasks/test_consumer.py | 8 | ||||
| -rw-r--r-- | tests/tasks/test_retention.py | 69 | ||||
| -rw-r--r-- | tests/test_config.py | 22 | ||||
| -rw-r--r-- | tests/test_url_digest.py | 100 | ||||
| -rw-r--r-- | tests/test_views.py | 12 | ||||
| -rw-r--r-- | webmentions_ssg/__init__.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 26 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py | 109 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 22 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 41 | ||||
| -rw-r--r-- | webmentions_ssg/url_digest.py | 21 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 23 |
19 files changed, 575 insertions, 16 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 2b7b115..45ae297 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -48,6 +48,19 @@ LOGFILE_BACKUP_COUNT = 10 # Webmention. Include each hostname you serve; wildcards are not supported. ALLOWED_HOSTNAMES = {"example.com", "www.example.com"} +# Maximum size of an incoming HTTP request body, including form encoding. +# This global Flask limit also applies to login and administration routes. +MAX_CONTENT_LENGTH = 64 * 1024 +# Independent UTF-8 byte limit for each URL submitted to /endpoint. +# Increase cautiously: a longer URL consumes additional storage and network IO. +MAX_WEBMENTION_URL_BYTES = 8192 +# Delete failed/deleted received Webmentions this many days after their most +# recent update; verified, pending, and outgoing records are never purged. +# Deleted status pages return 404; the same pair can be resubmitted afterward. +RECEIVED_RETENTION_DAYS = 30 +# Daily at 03:00 UTC (Huey's default timezone). +RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" + # Defense in depth: restrict the Huey worker service account with outbound # firewall/network-policy rules that block loopback, RFC1918, link-local, ULA, # and cloud metadata ranges, including IPv6. The application also pins diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 1c36d1f..7095d51 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -213,3 +213,47 @@ def test_create_db_preserves_logging( database_app.logger.warning("Logging still works after migration") assert "Logging still works after migration" in caplog.text + + +def test_upgrade_backfills_portable_digest_for_existing_received_rows( + database_app: Flask, +) -> None: + """Upgrade a pre-fingerprint SQLite database without losing historical rows.""" + from webmentions_ssg.url_digest import received_url_digest + + with database_app.app_context(): + migrate_upgrade(revision="c0272f25ce17") + with DATABASE.engine.begin() as connection: + connection.execute( + sa.text( + "INSERT INTO received_webmentions " + "(uuid, source, target, status, updated_at) " + "VALUES (:uuid, :source, :target, :status, :updated_at)" + ), + { + "uuid": "0" * 31 + "1", + "source": "https://source.example/old", + "target": "https://dennisfink.me/target", + "status": "verified", + "updated_at": "2026-01-01 00:00:00", + }, + ) + migrate_upgrade() + with DATABASE.engine.connect() as connection: + result = connection.execute( + sa.text( + "SELECT source, target, status, url_digest " + "FROM received_webmentions" + ) + ).one() + assert result.status == "verified" + assert result.url_digest == received_url_digest( + result.source, result.target + ) + constraints = inspect(connection).get_unique_constraints( + "received_webmentions" + ) + assert any( + constraint["column_names"] == ["url_digest"] + for constraint in constraints + ) diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index 0f5cb23..2a68509 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -145,3 +145,26 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: ) assert not form.validate() assert "Invalid input." in form.errors["target"] + + +def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( + app: Flask, +) -> None: + with app.test_request_context("/endpoint", method="POST"): + app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_SOURCE.encode("utf-8")) + form = EndpointForm( + formdata=MultiDict({"source": VALID_SOURCE + "x", "target": VALID_TARGET}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "source" in form.errors + assert any("bytes" in message for message in form.errors["source"]) + + app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_TARGET.encode("utf-8")) + form = EndpointForm( + formdata=MultiDict({"source": VALID_SOURCE, "target": VALID_TARGET + "x"}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "target" in form.errors + assert any("bytes" in message for message in form.errors["target"]) diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py index 2858b4c..10796e9 100644 --- a/tests/forms/test_validators.py +++ b/tests/forms/test_validators.py @@ -9,7 +9,12 @@ from flask import Flask from werkzeug.datastructures import MultiDict from wtforms import Form, StringField, ValidationError -from webmentions_ssg.forms.validators import AllowedHostname, NotEqualTo, PublicURL +from webmentions_ssg.forms.validators import ( + AllowedHostname, + NotEqualTo, + PublicURL, + URLMaxBytes, +) from webmentions_ssg.url_security import AddressResolutionError @@ -118,3 +123,28 @@ def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: form = URLForm(MultiDict({"url": "http://[::1"})) with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): AllowedHostname()(form, form.url) + + +def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None: + validator = URLMaxBytes() + form = URLForm(MultiDict({"url": "é" * 3})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 6 + validator(form, form.url) + app.config["MAX_WEBMENTION_URL_BYTES"] = 5 + with pytest.raises(ValidationError, match="5 bytes"): + validator(form, form.url) + + +def test_url_max_bytes_custom_message(app: Flask) -> None: + form = URLForm(MultiDict({"url": "too-long"})) + with app.app_context(): + app.config["MAX_WEBMENTION_URL_BYTES"] = 2 + with pytest.raises(ValidationError, match="URL too long"): + URLMaxBytes(message="URL too long")(form, form.url) + + +def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None: + form = URLForm(MultiDict({"url": "\ud800"})) + with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"): + URLMaxBytes()(form, form.url) diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py index dd93331..7604691 100644 --- a/tests/tasks/test_consumer.py +++ b/tests/tasks/test_consumer.py @@ -13,7 +13,7 @@ from flask import Flask from webmentions_ssg import HUEY -def test_consumer_registers_scanner_digest_and_verification_reaper( +def test_consumer_registers_scanner_digest_reaper_and_retention( app: Flask, monkeypatch: pytest.MonkeyPatch ) -> None: import webmentions_ssg @@ -21,6 +21,7 @@ def test_consumer_registers_scanner_digest_and_verification_reaper( app.config["SCANNER_SCHEDULE"] = "*/15 * * * *" app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *" app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *" + app.config["RECEIVED_RETENTION_SCHEDULE"] = "0 4 * * *" registered_tasks: list[object] = [] @@ -36,12 +37,13 @@ def test_consumer_registers_scanner_digest_and_verification_reaper( monkeypatch.setattr(HUEY, "periodic_task", periodic_task) consumer = importlib.import_module("webmentions_ssg.tasks.consumer") - from webmentions_ssg.tasks import notifications, receiver, scanner + from webmentions_ssg.tasks import notifications, receiver, retention, scanner assert consumer.huey is HUEY.huey - assert periodic_task.call_count == 3 + assert periodic_task.call_count == 4 assert registered_tasks == [ scanner.scan_sources, receiver.recover_stale_webmentions, notifications.send_webmention_digest, + retention.purge_old_received_webmentions, ] diff --git a/tests/tasks/test_retention.py b/tests/tasks/test_retention.py new file mode 100644 index 0000000..3f421f9 --- /dev/null +++ b/tests/tasks/test_retention.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test scheduled removal of terminal received Webmentions.""" + +import uuid +from datetime import UTC, datetime, timedelta + +import sqlalchemy as sa +from flask import Flask + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.tasks.retention import purge_old_received_webmentions + + +def test_retention_only_removes_old_failed_or_deleted_rows(app: Flask) -> None: + now = datetime.now(UTC) + app.config["RECEIVED_RETENTION_DAYS"] = 30 + identifiers = {} + with app.app_context(): + for suffix, status, age_days in ( + ("old-failed", "failed", 31), + ("old-deleted", "deleted", 31), + ("recent-failed", "failed", 2), + ("recent-deleted", "deleted", 2), + ("old-verified", "verified", 50), + ("old-received", "received", 50), + ("old-verifying", "verifying", 50), + ): + identifier = uuid.uuid7() + identifiers[suffix] = identifier + db.session.add( + ReceivedWebmention( + uuid=identifier, + source=f"https://source.example/{suffix}", + target="https://dennisfink.me/target", + status=status, + updated_at=now - timedelta(days=age_days), + ) + ) + db.session.commit() + + assert purge_old_received_webmentions() == 2 + assert purge_old_received_webmentions() == 0 + + remaining = set(db.session.scalars(sa.select(ReceivedWebmention.uuid))) + assert remaining == { + identifier + for key, identifier in identifiers.items() + if key not in {"old-failed", "old-deleted"} + } + + +def test_retention_uses_admin_configured_days(app: Flask) -> None: + app.config["RECEIVED_RETENTION_DAYS"] = 7 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/one", + target="https://dennisfink.me/target", + status="failed", + updated_at=datetime.now(UTC) - timedelta(days=10), + ) + ) + db.session.commit() + assert purge_old_received_webmentions() == 1 diff --git a/tests/test_config.py b/tests/test_config.py index dd02c01..933f437 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -232,3 +232,25 @@ def test_development_rejects_placeholder_even_with_debug(tmp_path: Path) -> None with pytest.raises(RuntimeError, match="SECRET_KEY"): create_app(LocalDevelopment) + + +def test_resource_limits_have_safe_defaults() -> None: + assert DefaultConfig.MAX_CONTENT_LENGTH == 64 * 1024 + assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192 + assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30 + assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *" + + +@pytest.mark.parametrize( + "key", ["MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS"] +) +@pytest.mark.parametrize("value", [None, "100", 0, -1, True, 1.5]) +def test_invalid_resource_limits_fail_at_startup( + tmp_path: Path, key: str, value: object +) -> None: + class InvalidResourceLimit(TestingConfig): + LOG_FILE = str(tmp_path / "invalid-resource.log") + + setattr(InvalidResourceLimit, key, value) + with pytest.raises(ValueError, match=key): + create_app(InvalidResourceLimit) diff --git a/tests/test_url_digest.py b/tests/test_url_digest.py new file mode 100644 index 0000000..0c04364 --- /dev/null +++ b/tests/test_url_digest.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test the database-independent Webmention URL fingerprints.""" + +import uuid +from datetime import UTC, datetime + +import pytest +import sqlalchemy as sa +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention, SentWebmention, Source +from webmentions_ssg.url_digest import received_url_digest, sent_target_digest + + +def test_received_pair_fingerprint_is_unambiguous() -> None: + assert received_url_digest("a", "bc") != received_url_digest("ab", "c") + assert received_url_digest("a", "b") != received_url_digest("b", "a") + + +def test_models_hash_url_fields_when_created_or_changed() -> None: + received = ReceivedWebmention( + uuid=uuid.uuid7(), + source="https://source.example/a", + target="https://site.example/b", + ) + assert received.url_digest == received_url_digest(received.source, received.target) + received.source = "https://source.example/c" + assert received.url_digest == received_url_digest(received.source, received.target) + received.target = "https://site.example/d" + assert received.url_digest == received_url_digest(received.source, received.target) + + sent = SentWebmention(target="https://site.example/a", desired_revision=1) + assert sent.target_digest == sent_target_digest(sent.target) + sent.target = "https://site.example/b" + assert sent.target_digest == sent_target_digest(sent.target) + + +def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None: + long_source = "https://source.example/" + "a" * 4000 + long_target = "https://dennisfink.me/" + "b" * 4000 + with app.app_context(): + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + db.session.commit() + match = db.session.scalar( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.url_digest + == received_url_digest(long_source, long_target) + ) + ) + assert match is not None + assert match.source == long_source + assert match.target == long_target + db.session.add( + ReceivedWebmention( + uuid=uuid.uuid7(), source=long_source, target=long_target + ) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() + + +def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text( + app: Flask, +) -> None: + now = datetime.now(UTC) + long_target = "https://destination.example/" + "a" * 4000 + with app.app_context(): + source = Source( + path="long/index.html", + url="https://example.com/long", + content_hash="0" * 32, + revision=1, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + db.session.add(source) + db.session.commit() + assert source.sent_webmentions[0].target_digest == sent_target_digest( + long_target + ) + + source.sent_webmentions.append( + SentWebmention(target=long_target, desired_revision=1) + ) + with pytest.raises(IntegrityError): + db.session.commit() + db.session.rollback() diff --git a/tests/test_views.py b/tests/test_views.py index 21a33bf..f5e301f 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -760,7 +760,7 @@ def test_endpoint_recovers_from_concurrent_insert( initial_result = Mock() initial_result.scalar_one_or_none.return_value = None competing_result = Mock() - competing_result.scalar_one.return_value = competing_webmention + competing_result.scalar_one_or_none.return_value = competing_webmention execute = Mock(side_effect=[initial_result, competing_result]) add = Mock() @@ -1184,3 +1184,13 @@ def test_admin_only_links_safe_existing_status_urls( html = response.get_data(as_text=True) assert (f'href="{status_url}"' in html) is safe assert status_url in html + + +def test_endpoint_rejects_request_bodies_over_configured_maximum( + app: Flask, client: FlaskClient +) -> None: + app.config["MAX_CONTENT_LENGTH"] = 128 + response = client.post( + "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} + ) + assert response.status_code == 413 diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 51635ef..e4a0a30 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -96,6 +96,14 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames( app.config["ALLOWED_HOSTNAMES"] ) + for key in ( + "MAX_CONTENT_LENGTH", + "MAX_WEBMENTION_URL_BYTES", + "RECEIVED_RETENTION_DAYS", + ): + value = app.config[key] + if type(value) is not int or value <= 0: + raise ValueError(f"{key} must be a positive integer.") if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]: raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 28cea64..97dd682 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -42,6 +42,13 @@ class DefaultConfig: # Empty by default: no target hostnames are accepted until configured. ALLOWED_HOSTNAMES = set() + # Cap incoming HTTP request bodies (also applies to other Flask routes). + MAX_CONTENT_LENGTH = 64 * 1024 + # Independent UTF-8 byte limit for each submitted source/target URL. + MAX_WEBMENTION_URL_BYTES = 8192 + # Purge terminal failed/deleted received Webmentions after 30 days. + RECEIVED_RETENTION_DAYS = 30 + RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index f698026..5006365 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,7 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral +from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes class LoginForm(FlaskForm): @@ -35,6 +35,7 @@ class EndpointForm(FlaskForm): "source", validators=[ InputRequired(), + URLMaxBytes(), URL(), Regexp( "^https?://.*", @@ -49,6 +50,7 @@ class EndpointForm(FlaskForm): "target", validators=[ InputRequired(), + URLMaxBytes(), URL(), Regexp( "^https?://.*", diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index 2ce2b5b..36f36ce 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -14,6 +14,32 @@ from wtforms.form import BaseForm from ..url_security import AddressResolutionError, is_public_url +class URLMaxBytes: + """Check the UTF-8 byte length of a submitted URL using the current config.""" + + def __init__( + self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None + ) -> None: + self.config_key = config_key + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + if field.data is None: + return + + max_bytes = current_app.config[self.config_key] + + try: + length = len(field.data.encode("utf-8")) + except UnicodeEncodeError as exc: + raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc + + if length > max_bytes: + raise ValidationError( + self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") + ) + + class NotEqualTo: """ Validate that a field does not equal another field. diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py new file mode 100644 index 0000000..0157df3 --- /dev/null +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Replace long URL uniqueness indexes with portable SHA-256 fingerprints. + +Revision ID: d845aa7fbc19 +Revises: c0272f25ce17 +""" + +import sqlalchemy as sa +from alembic import op + +from webmentions_ssg.url_digest import received_url_digest, sent_target_digest + +revision = "d845aa7fbc19" +down_revision = "c0272f25ce17" +branch_labels = None +depends_on = None + + +def _backfill_received() -> None: + table = sa.table( + "received_webmentions", + sa.column("uuid", sa.Uuid()), + sa.column("source", sa.Text()), + sa.column("target", sa.Text()), + sa.column("url_digest", sa.String(64)), + ) + conn = op.get_bind() + while True: + # Bounded batches avoid reading the entire production database into RAM. + rows = conn.execute( + sa.select(table.c.uuid, table.c.source, table.c.target) + .where(table.c.url_digest.is_(None)) + .limit(500) + ).all() + if not rows: + return + for identifier, source, target in rows: + conn.execute( + table.update() + .where(table.c.uuid == identifier) + .values(url_digest=received_url_digest(source, target)) + ) + + +def _backfill_sent() -> None: + table = sa.table( + "sent_webmentions", + sa.column("uuid", sa.Uuid()), + sa.column("target", sa.Text()), + sa.column("target_digest", sa.String(64)), + ) + conn = op.get_bind() + while True: + rows = conn.execute( + sa.select(table.c.uuid, table.c.target) + .where(table.c.target_digest.is_(None)) + .limit(500) + ).all() + if not rows: + return + for identifier, target in rows: + conn.execute( + table.update() + .where(table.c.uuid == identifier) + .values(target_digest=sent_target_digest(target)) + ) + + +def upgrade() -> None: + with op.batch_alter_table("received_webmentions") as batch: + batch.add_column(sa.Column("url_digest", sa.String(64), nullable=True)) + with op.batch_alter_table("sent_webmentions") as batch: + batch.add_column(sa.Column("target_digest", sa.String(64), nullable=True)) + + _backfill_received() + _backfill_sent() + + # Batch operations are required for portable SQLite support. The source + # and target Text columns remain unchanged, with no vendor-specific SQL. + with op.batch_alter_table("received_webmentions") as batch: + batch.alter_column("url_digest", existing_type=sa.String(64), nullable=False) + batch.drop_constraint("uq_webmention_source_target", type_="unique") + batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"]) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.alter_column("target_digest", existing_type=sa.String(64), nullable=False) + batch.drop_constraint("uq_sent_webmention_source_target", type_="unique") + batch.create_unique_constraint( + "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"] + ) + + +def downgrade() -> None: + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique") + batch.create_unique_constraint( + "uq_sent_webmention_source_target", ["source_id", "target"] + ) + batch.drop_column("target_digest") + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_url_digest", type_="unique") + batch.create_unique_constraint( + "uq_webmention_source_target", ["source", "target"] + ) + batch.drop_column("url_digest") diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 7cab879..0dfbb78 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -25,10 +25,11 @@ from sqlalchemy import ( UniqueConstraint, Uuid, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from werkzeug.security import check_password_hash, generate_password_hash from . import Base +from .url_digest import received_url_digest, sent_target_digest class SentWebmentionStatus(StrEnum): @@ -109,6 +110,15 @@ class ReceivedWebmention(Base): source: Mapped[str] = mapped_column(Text(), nullable=False) target: Mapped[str] = mapped_column(Text(), nullable=False) + url_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("source", "target") + def _update_url_digest(self, key: str, value: str) -> str: + source = value if key == "source" else self.__dict__.get("source") + target = value if key == "target" else self.__dict__.get("target") + if source is not None and target is not None: + self.url_digest = received_url_digest(source, target) + return value status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") failure_reason: Mapped[str | None] = mapped_column( @@ -127,7 +137,7 @@ class ReceivedWebmention(Base): ) __table_args__ = ( - UniqueConstraint("source", "target", name="uq_webmention_source_target"), + UniqueConstraint("url_digest", name="uq_webmention_url_digest"), Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) @@ -226,6 +236,12 @@ class SentWebmention(Base): ) target: Mapped[str] = mapped_column(Text(), nullable=False) + target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("target") + def _update_target_digest(self, key: str, value: str) -> str: + self.target_digest = sent_target_digest(value) + return value active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) @@ -275,7 +291,7 @@ class SentWebmention(Base): __table_args__ = ( UniqueConstraint( - "source_id", "target", name="uq_sent_webmention_source_target" + "source_id", "target_digest", name="uq_sent_webmention_source_target_digest" ), ) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index afae6e0..fb1d46d 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,7 +21,7 @@ def create_consumer() -> Huey: """ app = create_app() - from . import notifications, receiver, scanner, sender # noqa: F401 + from . import notifications, receiver, retention, scanner, sender # noqa: F401 schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) @@ -36,6 +36,11 @@ def create_consumer() -> Huey: ) HUEY.periodic_task(notification_schedule)(notifications.send_webmention_digest) + retention_schedule = crontab( + *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True + ) + HUEY.periodic_task(retention_schedule)(retention.purge_old_received_webmentions) + return HUEY.huey diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py new file mode 100644 index 0000000..727b186 --- /dev/null +++ b/webmentions_ssg/tasks/retention.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Delete old received Webmentions in terminal unsuccessful states.""" + +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +import sqlalchemy as sa +from flask import current_app +from sqlalchemy.engine import CursorResult + +from .. import DATABASE as db +from ..models import ReceivedWebmention + + +def purge_old_received_webmentions() -> int: + """Delete old failed/deleted received rows, retaining verified and in-flight ones. + + The status/time predicate is checked by the database in one DELETE, so a + row recently requeued for verification is not purged based on a stale read. + Outgoing sources and Webmentions are intentionally unaffected. + """ + cutoff = datetime.now(UTC) - timedelta( + days=current_app.config["RECEIVED_RETENTION_DAYS"] + ) + result = cast( + CursorResult[Any], + db.session.execute( + sa.delete(ReceivedWebmention).where( + ReceivedWebmention.status.in_(("failed", "deleted")), + ReceivedWebmention.updated_at < cutoff, + ) + ), + ) + db.session.commit() + removed = result.rowcount + if removed: + current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + return removed diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py new file mode 100644 index 0000000..256d037 --- /dev/null +++ b/webmentions_ssg/url_digest.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Portable SHA-256 fingerprints for indexed Webmention URLs.""" + +import hashlib +import json + + +def received_url_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair.""" + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def sent_target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID).""" + return hashlib.sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index 1dcdd2e..fcb5f2d 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -33,6 +33,7 @@ from .models import ReceivedWebmention, SentWebmention, Source, User from .tasks.receiver import verify_webmention from .tasks.scanner import manual_scan_sources from .tasks.sender import send_webmention +from .url_digest import received_url_digest from .url_security import is_safe_next_url root_page = Blueprint("root", __name__) @@ -339,12 +340,18 @@ def endpoint() -> ResponseReturnValue: source = cast(str, form.source.data) target = cast(str, form.target.data) + url_digest = received_url_digest(source, target) webmention = db.session.execute( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.source == source, ReceivedWebmention.target == target - ) + sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) ).scalar_one_or_none() + # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. + if webmention is not None and (webmention.source, webmention.target) != ( + source, + target, + ): + return {"error": "Webmention URL fingerprint conflict"}, 409 + should_verify = False if webmention is None: @@ -364,10 +371,14 @@ def endpoint() -> ResponseReturnValue: # after our SELECT but before our COMMIT. webmention = db.session.execute( sa.select(ReceivedWebmention).where( - ReceivedWebmention.source == source, - ReceivedWebmention.target == target, + ReceivedWebmention.url_digest == url_digest ) - ).scalar_one() + ).scalar_one_or_none() + if webmention is None or (webmention.source, webmention.target) != ( + source, + target, + ): + return {"error": "Webmention URL fingerprint conflict"}, 409 else: now = datetime.now(UTC) |
