aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--contrib/webmentions-ssg.example.cfg13
-rw-r--r--tests/cli/test_database.py44
-rw-r--r--tests/forms/test_init.py23
-rw-r--r--tests/forms/test_validators.py32
-rw-r--r--tests/tasks/test_consumer.py8
-rw-r--r--tests/tasks/test_retention.py69
-rw-r--r--tests/test_config.py22
-rw-r--r--tests/test_url_digest.py100
-rw-r--r--tests/test_views.py12
-rw-r--r--webmentions_ssg/__init__.py8
-rw-r--r--webmentions_ssg/config.py7
-rw-r--r--webmentions_ssg/forms/__init__.py4
-rw-r--r--webmentions_ssg/forms/validators.py26
-rw-r--r--webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py109
-rw-r--r--webmentions_ssg/models.py22
-rw-r--r--webmentions_ssg/tasks/consumer.py7
-rw-r--r--webmentions_ssg/tasks/retention.py41
-rw-r--r--webmentions_ssg/url_digest.py21
-rw-r--r--webmentions_ssg/views.py23
19 files changed, 575 insertions, 16 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg
index 2b7b115..45ae297 100644
--- a/contrib/webmentions-ssg.example.cfg
+++ b/contrib/webmentions-ssg.example.cfg
@@ -48,6 +48,19 @@ LOGFILE_BACKUP_COUNT = 10
# Webmention. Include each hostname you serve; wildcards are not supported.
ALLOWED_HOSTNAMES = {"example.com", "www.example.com"}
+# Maximum size of an incoming HTTP request body, including form encoding.
+# This global Flask limit also applies to login and administration routes.
+MAX_CONTENT_LENGTH = 64 * 1024
+# Independent UTF-8 byte limit for each URL submitted to /endpoint.
+# Increase cautiously: a longer URL consumes additional storage and network IO.
+MAX_WEBMENTION_URL_BYTES = 8192
+# Delete failed/deleted received Webmentions this many days after their most
+# recent update; verified, pending, and outgoing records are never purged.
+# Deleted status pages return 404; the same pair can be resubmitted afterward.
+RECEIVED_RETENTION_DAYS = 30
+# Daily at 03:00 UTC (Huey's default timezone).
+RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
+
# Defense in depth: restrict the Huey worker service account with outbound
# firewall/network-policy rules that block loopback, RFC1918, link-local, ULA,
# and cloud metadata ranges, including IPv6. The application also pins
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
index 1c36d1f..7095d51 100644
--- a/tests/cli/test_database.py
+++ b/tests/cli/test_database.py
@@ -213,3 +213,47 @@ def test_create_db_preserves_logging(
database_app.logger.warning("Logging still works after migration")
assert "Logging still works after migration" in caplog.text
+
+
+def test_upgrade_backfills_portable_digest_for_existing_received_rows(
+ database_app: Flask,
+) -> None:
+ """Upgrade a pre-fingerprint SQLite database without losing historical rows."""
+ from webmentions_ssg.url_digest import received_url_digest
+
+ with database_app.app_context():
+ migrate_upgrade(revision="c0272f25ce17")
+ with DATABASE.engine.begin() as connection:
+ connection.execute(
+ sa.text(
+ "INSERT INTO received_webmentions "
+ "(uuid, source, target, status, updated_at) "
+ "VALUES (:uuid, :source, :target, :status, :updated_at)"
+ ),
+ {
+ "uuid": "0" * 31 + "1",
+ "source": "https://source.example/old",
+ "target": "https://dennisfink.me/target",
+ "status": "verified",
+ "updated_at": "2026-01-01 00:00:00",
+ },
+ )
+ migrate_upgrade()
+ with DATABASE.engine.connect() as connection:
+ result = connection.execute(
+ sa.text(
+ "SELECT source, target, status, url_digest "
+ "FROM received_webmentions"
+ )
+ ).one()
+ assert result.status == "verified"
+ assert result.url_digest == received_url_digest(
+ result.source, result.target
+ )
+ constraints = inspect(connection).get_unique_constraints(
+ "received_webmentions"
+ )
+ assert any(
+ constraint["column_names"] == ["url_digest"]
+ for constraint in constraints
+ )
diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py
index 0f5cb23..2a68509 100644
--- a/tests/forms/test_init.py
+++ b/tests/forms/test_init.py
@@ -145,3 +145,26 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None:
)
assert not form.validate()
assert "Invalid input." in form.errors["target"]
+
+
+def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields(
+ app: Flask,
+) -> None:
+ with app.test_request_context("/endpoint", method="POST"):
+ app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_SOURCE.encode("utf-8"))
+ form = EndpointForm(
+ formdata=MultiDict({"source": VALID_SOURCE + "x", "target": VALID_TARGET}),
+ meta={"csrf": False},
+ )
+ assert not form.validate()
+ assert "source" in form.errors
+ assert any("bytes" in message for message in form.errors["source"])
+
+ app.config["MAX_WEBMENTION_URL_BYTES"] = len(VALID_TARGET.encode("utf-8"))
+ form = EndpointForm(
+ formdata=MultiDict({"source": VALID_SOURCE, "target": VALID_TARGET + "x"}),
+ meta={"csrf": False},
+ )
+ assert not form.validate()
+ assert "target" in form.errors
+ assert any("bytes" in message for message in form.errors["target"])
diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py
index 2858b4c..10796e9 100644
--- a/tests/forms/test_validators.py
+++ b/tests/forms/test_validators.py
@@ -9,7 +9,12 @@ from flask import Flask
from werkzeug.datastructures import MultiDict
from wtforms import Form, StringField, ValidationError
-from webmentions_ssg.forms.validators import AllowedHostname, NotEqualTo, PublicURL
+from webmentions_ssg.forms.validators import (
+ AllowedHostname,
+ NotEqualTo,
+ PublicURL,
+ URLMaxBytes,
+)
from webmentions_ssg.url_security import AddressResolutionError
@@ -118,3 +123,28 @@ def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None:
form = URLForm(MultiDict({"url": "http://[::1"}))
with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
AllowedHostname()(form, form.url)
+
+
+def test_url_max_bytes_respects_utf8_byte_length_and_current_config(app: Flask) -> None:
+ validator = URLMaxBytes()
+ form = URLForm(MultiDict({"url": "é" * 3}))
+ with app.app_context():
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 6
+ validator(form, form.url)
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 5
+ with pytest.raises(ValidationError, match="5 bytes"):
+ validator(form, form.url)
+
+
+def test_url_max_bytes_custom_message(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "too-long"}))
+ with app.app_context():
+ app.config["MAX_WEBMENTION_URL_BYTES"] = 2
+ with pytest.raises(ValidationError, match="URL too long"):
+ URLMaxBytes(message="URL too long")(form, form.url)
+
+
+def test_url_max_bytes_rejects_malformed_unicode(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "\ud800"}))
+ with app.app_context(), pytest.raises(ValidationError, match="valid UTF-8"):
+ URLMaxBytes()(form, form.url)
diff --git a/tests/tasks/test_consumer.py b/tests/tasks/test_consumer.py
index dd93331..7604691 100644
--- a/tests/tasks/test_consumer.py
+++ b/tests/tasks/test_consumer.py
@@ -13,7 +13,7 @@ from flask import Flask
from webmentions_ssg import HUEY
-def test_consumer_registers_scanner_digest_and_verification_reaper(
+def test_consumer_registers_scanner_digest_reaper_and_retention(
app: Flask, monkeypatch: pytest.MonkeyPatch
) -> None:
import webmentions_ssg
@@ -21,6 +21,7 @@ def test_consumer_registers_scanner_digest_and_verification_reaper(
app.config["SCANNER_SCHEDULE"] = "*/15 * * * *"
app.config["NOTIFICATION_SCHEDULE"] = "0 9 * * *"
app.config["VERIFICATION_REAPER_SCHEDULE"] = "*/3 * * * *"
+ app.config["RECEIVED_RETENTION_SCHEDULE"] = "0 4 * * *"
registered_tasks: list[object] = []
@@ -36,12 +37,13 @@ def test_consumer_registers_scanner_digest_and_verification_reaper(
monkeypatch.setattr(HUEY, "periodic_task", periodic_task)
consumer = importlib.import_module("webmentions_ssg.tasks.consumer")
- from webmentions_ssg.tasks import notifications, receiver, scanner
+ from webmentions_ssg.tasks import notifications, receiver, retention, scanner
assert consumer.huey is HUEY.huey
- assert periodic_task.call_count == 3
+ assert periodic_task.call_count == 4
assert registered_tasks == [
scanner.scan_sources,
receiver.recover_stale_webmentions,
notifications.send_webmention_digest,
+ retention.purge_old_received_webmentions,
]
diff --git a/tests/tasks/test_retention.py b/tests/tasks/test_retention.py
new file mode 100644
index 0000000..3f421f9
--- /dev/null
+++ b/tests/tasks/test_retention.py
@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Test scheduled removal of terminal received Webmentions."""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import sqlalchemy as sa
+from flask import Flask
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import ReceivedWebmention
+from webmentions_ssg.tasks.retention import purge_old_received_webmentions
+
+
+def test_retention_only_removes_old_failed_or_deleted_rows(app: Flask) -> None:
+ now = datetime.now(UTC)
+ app.config["RECEIVED_RETENTION_DAYS"] = 30
+ identifiers = {}
+ with app.app_context():
+ for suffix, status, age_days in (
+ ("old-failed", "failed", 31),
+ ("old-deleted", "deleted", 31),
+ ("recent-failed", "failed", 2),
+ ("recent-deleted", "deleted", 2),
+ ("old-verified", "verified", 50),
+ ("old-received", "received", 50),
+ ("old-verifying", "verifying", 50),
+ ):
+ identifier = uuid.uuid7()
+ identifiers[suffix] = identifier
+ db.session.add(
+ ReceivedWebmention(
+ uuid=identifier,
+ source=f"https://source.example/{suffix}",
+ target="https://dennisfink.me/target",
+ status=status,
+ updated_at=now - timedelta(days=age_days),
+ )
+ )
+ db.session.commit()
+
+ assert purge_old_received_webmentions() == 2
+ assert purge_old_received_webmentions() == 0
+
+ remaining = set(db.session.scalars(sa.select(ReceivedWebmention.uuid)))
+ assert remaining == {
+ identifier
+ for key, identifier in identifiers.items()
+ if key not in {"old-failed", "old-deleted"}
+ }
+
+
+def test_retention_uses_admin_configured_days(app: Flask) -> None:
+ app.config["RECEIVED_RETENTION_DAYS"] = 7
+ with app.app_context():
+ db.session.add(
+ ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source="https://source.example/one",
+ target="https://dennisfink.me/target",
+ status="failed",
+ updated_at=datetime.now(UTC) - timedelta(days=10),
+ )
+ )
+ db.session.commit()
+ assert purge_old_received_webmentions() == 1
diff --git a/tests/test_config.py b/tests/test_config.py
index dd02c01..933f437 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -232,3 +232,25 @@ def test_development_rejects_placeholder_even_with_debug(tmp_path: Path) -> None
with pytest.raises(RuntimeError, match="SECRET_KEY"):
create_app(LocalDevelopment)
+
+
+def test_resource_limits_have_safe_defaults() -> None:
+ assert DefaultConfig.MAX_CONTENT_LENGTH == 64 * 1024
+ assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192
+ assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30
+ assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *"
+
+
+@pytest.mark.parametrize(
+ "key", ["MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS"]
+)
+@pytest.mark.parametrize("value", [None, "100", 0, -1, True, 1.5])
+def test_invalid_resource_limits_fail_at_startup(
+ tmp_path: Path, key: str, value: object
+) -> None:
+ class InvalidResourceLimit(TestingConfig):
+ LOG_FILE = str(tmp_path / "invalid-resource.log")
+
+ setattr(InvalidResourceLimit, key, value)
+ with pytest.raises(ValueError, match=key):
+ create_app(InvalidResourceLimit)
diff --git a/tests/test_url_digest.py b/tests/test_url_digest.py
new file mode 100644
index 0000000..0c04364
--- /dev/null
+++ b/tests/test_url_digest.py
@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Test the database-independent Webmention URL fingerprints."""
+
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+import sqlalchemy as sa
+from flask import Flask
+from sqlalchemy.exc import IntegrityError
+
+from webmentions_ssg import DATABASE as db
+from webmentions_ssg.models import ReceivedWebmention, SentWebmention, Source
+from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
+
+
+def test_received_pair_fingerprint_is_unambiguous() -> None:
+ assert received_url_digest("a", "bc") != received_url_digest("ab", "c")
+ assert received_url_digest("a", "b") != received_url_digest("b", "a")
+
+
+def test_models_hash_url_fields_when_created_or_changed() -> None:
+ received = ReceivedWebmention(
+ uuid=uuid.uuid7(),
+ source="https://source.example/a",
+ target="https://site.example/b",
+ )
+ assert received.url_digest == received_url_digest(received.source, received.target)
+ received.source = "https://source.example/c"
+ assert received.url_digest == received_url_digest(received.source, received.target)
+ received.target = "https://site.example/d"
+ assert received.url_digest == received_url_digest(received.source, received.target)
+
+ sent = SentWebmention(target="https://site.example/a", desired_revision=1)
+ assert sent.target_digest == sent_target_digest(sent.target)
+ sent.target = "https://site.example/b"
+ assert sent.target_digest == sent_target_digest(sent.target)
+
+
+def test_long_received_pair_can_be_stored_with_unique_digest(app: Flask) -> None:
+ long_source = "https://source.example/" + "a" * 4000
+ long_target = "https://dennisfink.me/" + "b" * 4000
+ with app.app_context():
+ db.session.add(
+ ReceivedWebmention(
+ uuid=uuid.uuid7(), source=long_source, target=long_target
+ )
+ )
+ db.session.commit()
+ match = db.session.scalar(
+ sa.select(ReceivedWebmention).where(
+ ReceivedWebmention.url_digest
+ == received_url_digest(long_source, long_target)
+ )
+ )
+ assert match is not None
+ assert match.source == long_source
+ assert match.target == long_target
+ db.session.add(
+ ReceivedWebmention(
+ uuid=uuid.uuid7(), source=long_source, target=long_target
+ )
+ )
+ with pytest.raises(IntegrityError):
+ db.session.commit()
+ db.session.rollback()
+
+
+def test_long_sent_targets_enforce_uniqueness_without_indexing_url_text(
+ app: Flask,
+) -> None:
+ now = datetime.now(UTC)
+ long_target = "https://destination.example/" + "a" * 4000
+ with app.app_context():
+ source = Source(
+ path="long/index.html",
+ url="https://example.com/long",
+ content_hash="0" * 32,
+ revision=1,
+ last_seen_at=now,
+ revised_at=now,
+ )
+ source.sent_webmentions.append(
+ SentWebmention(target=long_target, desired_revision=1)
+ )
+ db.session.add(source)
+ db.session.commit()
+ assert source.sent_webmentions[0].target_digest == sent_target_digest(
+ long_target
+ )
+
+ source.sent_webmentions.append(
+ SentWebmention(target=long_target, desired_revision=1)
+ )
+ with pytest.raises(IntegrityError):
+ db.session.commit()
+ db.session.rollback()
diff --git a/tests/test_views.py b/tests/test_views.py
index 21a33bf..f5e301f 100644
--- a/tests/test_views.py
+++ b/tests/test_views.py
@@ -760,7 +760,7 @@ def test_endpoint_recovers_from_concurrent_insert(
initial_result = Mock()
initial_result.scalar_one_or_none.return_value = None
competing_result = Mock()
- competing_result.scalar_one.return_value = competing_webmention
+ competing_result.scalar_one_or_none.return_value = competing_webmention
execute = Mock(side_effect=[initial_result, competing_result])
add = Mock()
@@ -1184,3 +1184,13 @@ def test_admin_only_links_safe_existing_status_urls(
html = response.get_data(as_text=True)
assert (f'href="{status_url}"' in html) is safe
assert status_url in html
+
+
+def test_endpoint_rejects_request_bodies_over_configured_maximum(
+ app: Flask, client: FlaskClient
+) -> None:
+ app.config["MAX_CONTENT_LENGTH"] = 128
+ response = client.post(
+ "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL}
+ )
+ assert response.status_code == 413
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index 51635ef..e4a0a30 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -96,6 +96,14 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
app.config["ALLOWED_HOSTNAMES"]
)
+ for key in (
+ "MAX_CONTENT_LENGTH",
+ "MAX_WEBMENTION_URL_BYTES",
+ "RECEIVED_RETENTION_DAYS",
+ ):
+ value = app.config[key]
+ if type(value) is not int or value <= 0:
+ raise ValueError(f"{key} must be a positive integer.")
if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]:
raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.")
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 28cea64..97dd682 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -42,6 +42,13 @@ class DefaultConfig:
# Empty by default: no target hostnames are accepted until configured.
ALLOWED_HOSTNAMES = set()
+ # Cap incoming HTTP request bodies (also applies to other Flask routes).
+ MAX_CONTENT_LENGTH = 64 * 1024
+ # Independent UTF-8 byte limit for each submitted source/target URL.
+ MAX_WEBMENTION_URL_BYTES = 8192
+ # Purge terminal failed/deleted received Webmentions after 30 days.
+ RECEIVED_RETENTION_DAYS = 30
+ RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
index f698026..5006365 100644
--- a/webmentions_ssg/forms/__init__.py
+++ b/webmentions_ssg/forms/__init__.py
@@ -10,7 +10,7 @@ from flask_wtf import FlaskForm
from wtforms import PasswordField, StringField, SubmitField
from wtforms.validators import URL, InputRequired, Regexp
-from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral
+from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes
class LoginForm(FlaskForm):
@@ -35,6 +35,7 @@ class EndpointForm(FlaskForm):
"source",
validators=[
InputRequired(),
+ URLMaxBytes(),
URL(),
Regexp(
"^https?://.*",
@@ -49,6 +50,7 @@ class EndpointForm(FlaskForm):
"target",
validators=[
InputRequired(),
+ URLMaxBytes(),
URL(),
Regexp(
"^https?://.*",
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index 2ce2b5b..36f36ce 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -14,6 +14,32 @@ from wtforms.form import BaseForm
from ..url_security import AddressResolutionError, is_public_url
+class URLMaxBytes:
+ """Check the UTF-8 byte length of a submitted URL using the current config."""
+
+ def __init__(
+ self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None
+ ) -> None:
+ self.config_key = config_key
+ self.message = message
+
+ def __call__(self, form: BaseForm, field: Field) -> None:
+ if field.data is None:
+ return
+
+ max_bytes = current_app.config[self.config_key]
+
+ try:
+ length = len(field.data.encode("utf-8"))
+ except UnicodeEncodeError as exc:
+ raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc
+
+ if length > max_bytes:
+ raise ValidationError(
+ self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.")
+ )
+
+
class NotEqualTo:
"""
Validate that a field does not equal another field.
diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
new file mode 100644
index 0000000..0157df3
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Replace long URL uniqueness indexes with portable SHA-256 fingerprints.
+
+Revision ID: d845aa7fbc19
+Revises: c0272f25ce17
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
+
+revision = "d845aa7fbc19"
+down_revision = "c0272f25ce17"
+branch_labels = None
+depends_on = None
+
+
+def _backfill_received() -> None:
+ table = sa.table(
+ "received_webmentions",
+ sa.column("uuid", sa.Uuid()),
+ sa.column("source", sa.Text()),
+ sa.column("target", sa.Text()),
+ sa.column("url_digest", sa.String(64)),
+ )
+ conn = op.get_bind()
+ while True:
+ # Bounded batches avoid reading the entire production database into RAM.
+ rows = conn.execute(
+ sa.select(table.c.uuid, table.c.source, table.c.target)
+ .where(table.c.url_digest.is_(None))
+ .limit(500)
+ ).all()
+ if not rows:
+ return
+ for identifier, source, target in rows:
+ conn.execute(
+ table.update()
+ .where(table.c.uuid == identifier)
+ .values(url_digest=received_url_digest(source, target))
+ )
+
+
+def _backfill_sent() -> None:
+ table = sa.table(
+ "sent_webmentions",
+ sa.column("uuid", sa.Uuid()),
+ sa.column("target", sa.Text()),
+ sa.column("target_digest", sa.String(64)),
+ )
+ conn = op.get_bind()
+ while True:
+ rows = conn.execute(
+ sa.select(table.c.uuid, table.c.target)
+ .where(table.c.target_digest.is_(None))
+ .limit(500)
+ ).all()
+ if not rows:
+ return
+ for identifier, target in rows:
+ conn.execute(
+ table.update()
+ .where(table.c.uuid == identifier)
+ .values(target_digest=sent_target_digest(target))
+ )
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.add_column(sa.Column("url_digest", sa.String(64), nullable=True))
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.add_column(sa.Column("target_digest", sa.String(64), nullable=True))
+
+ _backfill_received()
+ _backfill_sent()
+
+ # Batch operations are required for portable SQLite support. The source
+ # and target Text columns remain unchanged, with no vendor-specific SQL.
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.alter_column("url_digest", existing_type=sa.String(64), nullable=False)
+ batch.drop_constraint("uq_webmention_source_target", type_="unique")
+ batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"])
+
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.alter_column("target_digest", existing_type=sa.String(64), nullable=False)
+ batch.drop_constraint("uq_sent_webmention_source_target", type_="unique")
+ batch.create_unique_constraint(
+ "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"]
+ )
+
+
+def downgrade() -> None:
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique")
+ batch.create_unique_constraint(
+ "uq_sent_webmention_source_target", ["source_id", "target"]
+ )
+ batch.drop_column("target_digest")
+
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.drop_constraint("uq_webmention_url_digest", type_="unique")
+ batch.create_unique_constraint(
+ "uq_webmention_source_target", ["source", "target"]
+ )
+ batch.drop_column("url_digest")
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 7cab879..0dfbb78 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -25,10 +25,11 @@ from sqlalchemy import (
UniqueConstraint,
Uuid,
)
-from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
from werkzeug.security import check_password_hash, generate_password_hash
from . import Base
+from .url_digest import received_url_digest, sent_target_digest
class SentWebmentionStatus(StrEnum):
@@ -109,6 +110,15 @@ class ReceivedWebmention(Base):
source: Mapped[str] = mapped_column(Text(), nullable=False)
target: Mapped[str] = mapped_column(Text(), nullable=False)
+ url_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+ @validates("source", "target")
+ def _update_url_digest(self, key: str, value: str) -> str:
+ source = value if key == "source" else self.__dict__.get("source")
+ target = value if key == "target" else self.__dict__.get("target")
+ if source is not None and target is not None:
+ self.url_digest = received_url_digest(source, target)
+ return value
status: Mapped[str] = mapped_column(Text(), nullable=False, default="received")
failure_reason: Mapped[str | None] = mapped_column(
@@ -127,7 +137,7 @@ class ReceivedWebmention(Base):
)
__table_args__ = (
- UniqueConstraint("source", "target", name="uq_webmention_source_target"),
+ UniqueConstraint("url_digest", name="uq_webmention_url_digest"),
Index("ix_received_webmentions_status_updated_at", "status", "updated_at"),
)
@@ -226,6 +236,12 @@ class SentWebmention(Base):
)
target: Mapped[str] = mapped_column(Text(), nullable=False)
+ target_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+ @validates("target")
+ def _update_target_digest(self, key: str, value: str) -> str:
+ self.target_digest = sent_target_digest(value)
+ return value
active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True)
@@ -275,7 +291,7 @@ class SentWebmention(Base):
__table_args__ = (
UniqueConstraint(
- "source_id", "target", name="uq_sent_webmention_source_target"
+ "source_id", "target_digest", name="uq_sent_webmention_source_target_digest"
),
)
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index afae6e0..fb1d46d 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,7 +21,7 @@ def create_consumer() -> Huey:
"""
app = create_app()
- from . import notifications, receiver, scanner, sender # noqa: F401
+ from . import notifications, receiver, retention, scanner, sender # noqa: F401
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
@@ -36,6 +36,11 @@ def create_consumer() -> Huey:
)
HUEY.periodic_task(notification_schedule)(notifications.send_webmention_digest)
+ retention_schedule = crontab(
+ *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True
+ )
+ HUEY.periodic_task(retention_schedule)(retention.purge_old_received_webmentions)
+
return HUEY.huey
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
new file mode 100644
index 0000000..727b186
--- /dev/null
+++ b/webmentions_ssg/tasks/retention.py
@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Delete old received Webmentions in terminal unsuccessful states."""
+
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+
+import sqlalchemy as sa
+from flask import current_app
+from sqlalchemy.engine import CursorResult
+
+from .. import DATABASE as db
+from ..models import ReceivedWebmention
+
+
+def purge_old_received_webmentions() -> int:
+ """Delete old failed/deleted received rows, retaining verified and in-flight ones.
+
+ The status/time predicate is checked by the database in one DELETE, so a
+ row recently requeued for verification is not purged based on a stale read.
+ Outgoing sources and Webmentions are intentionally unaffected.
+ """
+ cutoff = datetime.now(UTC) - timedelta(
+ days=current_app.config["RECEIVED_RETENTION_DAYS"]
+ )
+ result = cast(
+ CursorResult[Any],
+ db.session.execute(
+ sa.delete(ReceivedWebmention).where(
+ ReceivedWebmention.status.in_(("failed", "deleted")),
+ ReceivedWebmention.updated_at < cutoff,
+ )
+ ),
+ )
+ db.session.commit()
+ removed = result.rowcount
+ if removed:
+ current_app.logger.info("Purged %d old failed/deleted Webmentions", removed)
+ return removed
diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py
new file mode 100644
index 0000000..256d037
--- /dev/null
+++ b/webmentions_ssg/url_digest.py
@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Portable SHA-256 fingerprints for indexed Webmention URLs."""
+
+import hashlib
+import json
+
+
+def received_url_digest(source: str, target: str) -> str:
+ """Fingerprint an unambiguous, ordered source/target pair."""
+ encoded = json.dumps(
+ [source, target], ensure_ascii=False, separators=(",", ":")
+ ).encode("utf-8")
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def sent_target_digest(target: str) -> str:
+ """Fingerprint an outgoing target (unique within each source ID)."""
+ return hashlib.sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
index 1dcdd2e..fcb5f2d 100644
--- a/webmentions_ssg/views.py
+++ b/webmentions_ssg/views.py
@@ -33,6 +33,7 @@ from .models import ReceivedWebmention, SentWebmention, Source, User
from .tasks.receiver import verify_webmention
from .tasks.scanner import manual_scan_sources
from .tasks.sender import send_webmention
+from .url_digest import received_url_digest
from .url_security import is_safe_next_url
root_page = Blueprint("root", __name__)
@@ -339,12 +340,18 @@ def endpoint() -> ResponseReturnValue:
source = cast(str, form.source.data)
target = cast(str, form.target.data)
+ url_digest = received_url_digest(source, target)
webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(
- ReceivedWebmention.source == source, ReceivedWebmention.target == target
- )
+ sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
).scalar_one_or_none()
+ # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
+ if webmention is not None and (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return {"error": "Webmention URL fingerprint conflict"}, 409
+
should_verify = False
if webmention is None:
@@ -364,10 +371,14 @@ def endpoint() -> ResponseReturnValue:
# after our SELECT but before our COMMIT.
webmention = db.session.execute(
sa.select(ReceivedWebmention).where(
- ReceivedWebmention.source == source,
- ReceivedWebmention.target == target,
+ ReceivedWebmention.url_digest == url_digest
)
- ).scalar_one()
+ ).scalar_one_or_none()
+ if webmention is None or (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return {"error": "Webmention URL fingerprint conflict"}, 409
else:
now = datetime.now(UTC)