aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
authorDennis Fink2026-10-31 17:42:00 +0100
committerDennis Fink2026-10-31 17:42:00 +0100
commit6924ad7e52033365c55d5afacdd916c502cc80cb (patch)
tree8a52b8e8fbae7a6bcc85d250cac2436a2c42e748 /webmentions_ssg
parent3494eab1208dc18478485e02c61e292556502fa6 (diff)
downloadwebmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.tar.gz
webmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.zip
feat(receiver): add resource limits and retention
Bound incoming request and URL sizes so untrusted Webmention submissions cannot consume unbounded application or database resources. Purge old failed and deleted received Webmentions on a schedule, and use portable SHA-256 URL fingerprints to preserve uniqueness for long URLs.
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/__init__.py8
-rw-r--r--webmentions_ssg/config.py7
-rw-r--r--webmentions_ssg/forms/__init__.py4
-rw-r--r--webmentions_ssg/forms/validators.py26
-rw-r--r--webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py109
-rw-r--r--webmentions_ssg/models.py22
-rw-r--r--webmentions_ssg/tasks/consumer.py7
-rw-r--r--webmentions_ssg/tasks/retention.py41
-rw-r--r--webmentions_ssg/url_digest.py21
-rw-r--r--webmentions_ssg/views.py23
10 files changed, 257 insertions, 11 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index 51635ef..e4a0a30 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -96,6 +96,14 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
app.config["ALLOWED_HOSTNAMES"]
)
+ for key in (
+ "MAX_CONTENT_LENGTH",
+ "MAX_WEBMENTION_URL_BYTES",
+ "RECEIVED_RETENTION_DAYS",
+ ):
+ value = app.config[key]
+ if type(value) is not int or value <= 0:
+ raise ValueError(f"{key} must be a positive integer.")
if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]:
raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.")
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 28cea64..97dd682 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -42,6 +42,13 @@ class DefaultConfig:
# Empty by default: no target hostnames are accepted until configured.
ALLOWED_HOSTNAMES = set()
+ # Cap incoming HTTP request bodies (also applies to other Flask routes).
+ MAX_CONTENT_LENGTH = 64 * 1024
+ # Independent UTF-8 byte limit for each submitted source/target URL.
+ MAX_WEBMENTION_URL_BYTES = 8192
+ # Purge terminal failed/deleted received Webmentions after 30 days.
+ RECEIVED_RETENTION_DAYS = 30
+ RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
index f698026..5006365 100644
--- a/webmentions_ssg/forms/__init__.py
+++ b/webmentions_ssg/forms/__init__.py
@@ -10,7 +10,7 @@ from flask_wtf import FlaskForm
from wtforms import PasswordField, StringField, SubmitField
from wtforms.validators import URL, InputRequired, Regexp
-from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral
+from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes
class LoginForm(FlaskForm):
@@ -35,6 +35,7 @@ class EndpointForm(FlaskForm):
"source",
validators=[
InputRequired(),
+ URLMaxBytes(),
URL(),
Regexp(
"^https?://.*",
@@ -49,6 +50,7 @@ class EndpointForm(FlaskForm):
"target",
validators=[
InputRequired(),
+ URLMaxBytes(),
URL(),
Regexp(
"^https?://.*",
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index 2ce2b5b..36f36ce 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -14,6 +14,32 @@ from wtforms.form import BaseForm
from ..url_security import AddressResolutionError, is_public_url
+class URLMaxBytes:
+ """Check the UTF-8 byte length of a submitted URL using the current config."""
+
+ def __init__(
+ self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None
+ ) -> None:
+ self.config_key = config_key
+ self.message = message
+
+ def __call__(self, form: BaseForm, field: Field) -> None:
+ if field.data is None:
+ return
+
+ max_bytes = current_app.config[self.config_key]
+
+ try:
+ length = len(field.data.encode("utf-8"))
+ except UnicodeEncodeError as exc:
+ raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc
+
+ if length > max_bytes:
+ raise ValidationError(
+ self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.")
+ )
+
+
class NotEqualTo:
"""
Validate that a field does not equal another field.
diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
new file mode 100644
index 0000000..0157df3
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Replace long URL uniqueness indexes with portable SHA-256 fingerprints.
+
+Revision ID: d845aa7fbc19
+Revises: c0272f25ce17
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
+
+revision = "d845aa7fbc19"
+down_revision = "c0272f25ce17"
+branch_labels = None
+depends_on = None
+
+
+def _backfill_received() -> None:
+ table = sa.table(
+ "received_webmentions",
+ sa.column("uuid", sa.Uuid()),
+ sa.column("source", sa.Text()),
+ sa.column("target", sa.Text()),
+ sa.column("url_digest", sa.String(64)),
+ )
+ conn = op.get_bind()
+ while True:
+ # Bounded batches avoid reading the entire production database into RAM.
+ rows = conn.execute(
+ sa.select(table.c.uuid, table.c.source, table.c.target)
+ .where(table.c.url_digest.is_(None))
+ .limit(500)
+ ).all()
+ if not rows:
+ return
+ for identifier, source, target in rows:
+ conn.execute(
+ table.update()
+ .where(table.c.uuid == identifier)
+ .values(url_digest=received_url_digest(source, target))
+ )
+
+
+def _backfill_sent() -> None:
+ table = sa.table(
+ "sent_webmentions",
+ sa.column("uuid", sa.Uuid()),
+ sa.column("target", sa.Text()),
+ sa.column("target_digest", sa.String(64)),
+ )
+ conn = op.get_bind()
+ while True:
+ rows = conn.execute(
+ sa.select(table.c.uuid, table.c.target)
+ .where(table.c.target_digest.is_(None))
+ .limit(500)
+ ).all()
+ if not rows:
+ return
+ for identifier, target in rows:
+ conn.execute(
+ table.update()
+ .where(table.c.uuid == identifier)
+ .values(target_digest=sent_target_digest(target))
+ )
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.add_column(sa.Column("url_digest", sa.String(64), nullable=True))
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.add_column(sa.Column("target_digest", sa.String(64), nullable=True))
+
+ _backfill_received()
+ _backfill_sent()
+
+ # Batch operations are required for portable SQLite support. The source
+ # and target Text columns remain unchanged, with no vendor-specific SQL.
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.alter_column("url_digest", existing_type=sa.String(64), nullable=False)
+ batch.drop_constraint("uq_webmention_source_target", type_="unique")
+ batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"])
+
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.alter_column("target_digest", existing_type=sa.String(64), nullable=False)
+ batch.drop_constraint("uq_sent_webmention_source_target", type_="unique")
+ batch.create_unique_constraint(
+ "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"]
+ )
+
+
+def downgrade() -> None:
+ with op.batch_alter_table("sent_webmentions") as batch:
+ batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique")
+ batch.create_unique_constraint(
+ "uq_sent_webmention_source_target", ["source_id", "target"]
+ )
+ batch.drop_column("target_digest")
+
+ with op.batch_alter_table("received_webmentions") as batch:
+ batch.drop_constraint("uq_webmention_url_digest", type_="unique")
+ batch.create_unique_constraint(
+ "uq_webmention_source_target", ["source", "target"]
+ )
+ batch.drop_column("url_digest")
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 7cab879..0dfbb78 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -25,10 +25,11 @@ from sqlalchemy import (
UniqueConstraint,
Uuid,
)
-from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
from werkzeug.security import check_password_hash, generate_password_hash
from . import Base
+from .url_digest import received_url_digest, sent_target_digest
class SentWebmentionStatus(StrEnum):
@@ -109,6 +110,15 @@ class ReceivedWebmention(Base):
source: Mapped[str] = mapped_column(Text(), nullable=False)
target: Mapped[str] = mapped_column(Text(), nullable=False)
+ url_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+ @validates("source", "target")
+ def _update_url_digest(self, key: str, value: str) -> str:
+ source = value if key == "source" else self.__dict__.get("source")
+ target = value if key == "target" else self.__dict__.get("target")
+ if source is not None and target is not None:
+ self.url_digest = received_url_digest(source, target)
+ return value
status: Mapped[str] = mapped_column(Text(), nullable=False, default="received")
failure_reason: Mapped[str | None] = mapped_column(
@@ -127,7 +137,7 @@ class ReceivedWebmention(Base):
)
__table_args__ = (
- UniqueConstraint("source", "target", name="uq_webmention_source_target"),
+ UniqueConstraint("url_digest", name="uq_webmention_url_digest"),
Index("ix_received_webmentions_status_updated_at", "status", "updated_at"),
)
@@ -226,6 +236,12 @@ class SentWebmention(Base):
)
target: Mapped[str] = mapped_column(Text(), nullable=False)
+ target_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+ @validates("target")
+ def _update_target_digest(self, key: str, value: str) -> str:
+ self.target_digest = sent_target_digest(value)
+ return value
active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True)
@@ -275,7 +291,7 @@ class SentWebmention(Base):
__table_args__ = (
UniqueConstraint(
- "source_id", "target", name="uq_sent_webmention_source_target"
+ "source_id", "target_digest", name="uq_sent_webmention_source_target_digest"
),
)
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index afae6e0..fb1d46d 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,7 +21,7 @@ def create_consumer() -> Huey:
"""
app = create_app()
- from . import notifications, receiver, scanner, sender # noqa: F401
+ from . import notifications, receiver, retention, scanner, sender # noqa: F401
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
@@ -36,6 +36,11 @@ def create_consumer() -> Huey:
)
HUEY.periodic_task(notification_schedule)(notifications.send_webmention_digest)
+ retention_schedule = crontab(
+ *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True
+ )
+ HUEY.periodic_task(retention_schedule)(retention.purge_old_received_webmentions)
+
return HUEY.huey
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
new file mode 100644
index 0000000..727b186
--- /dev/null
+++ b/webmentions_ssg/tasks/retention.py
@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Delete old received Webmentions in terminal unsuccessful states."""
+
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+
+import sqlalchemy as sa
+from flask import current_app
+from sqlalchemy.engine import CursorResult
+
+from .. import DATABASE as db
+from ..models import ReceivedWebmention
+
+
+def purge_old_received_webmentions() -> int:
+ """Delete old failed/deleted received rows, retaining verified and in-flight ones.
+
+ The status/time predicate is checked by the database in one DELETE, so a
+ row recently requeued for verification is not purged based on a stale read.
+ Outgoing sources and Webmentions are intentionally unaffected.
+ """
+ cutoff = datetime.now(UTC) - timedelta(
+ days=current_app.config["RECEIVED_RETENTION_DAYS"]
+ )
+ result = cast(
+ CursorResult[Any],
+ db.session.execute(
+ sa.delete(ReceivedWebmention).where(
+ ReceivedWebmention.status.in_(("failed", "deleted")),
+ ReceivedWebmention.updated_at < cutoff,
+ )
+ ),
+ )
+ db.session.commit()
+ removed = result.rowcount
+ if removed:
+ current_app.logger.info("Purged %d old failed/deleted Webmentions", removed)
+ return removed
diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py
new file mode 100644
index 0000000..256d037
--- /dev/null
+++ b/webmentions_ssg/url_digest.py
@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Portable SHA-256 fingerprints for indexed Webmention URLs."""
+
+import hashlib
+import json
+
+
+def received_url_digest(source: str, target: str) -> str:
+ """Fingerprint an unambiguous, ordered source/target pair."""
+ encoded = json.dumps(
+ [source, target], ensure_ascii=False, separators=(",", ":")
+ ).encode("utf-8")
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def sent_target_digest(target: str) -> str:
+ """Fingerprint an outgoing target (unique within each source ID)."""
+ return hashlib.sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
index 1dcdd2e..fcb5f2d 100644
--- a/webmentions_ssg/views.py
+++ b/webmentions_ssg/views.py
@@ -33,6 +33,7 @@ from .models import ReceivedWebmention, SentWebmention, Source, User
from .tasks.receiver import verify_webmention
from .tasks.scanner import manual_scan_sources
from .tasks.sender import send_webmention
+from .url_digest import received_url_digest
from .url_security import is_safe_next_url
root_page = Blueprint("root", __name__)
@@ -339,12 +340,18 @@ def endpoint() -> ResponseReturnValue:
source = cast(str, form.source.data)
target = cast(str, form.target.data)
+ url_digest = received_url_digest(source, target)
webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(
- ReceivedWebmention.source == source, ReceivedWebmention.target == target
- )
+ sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
).scalar_one_or_none()
+ # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
+ if webmention is not None and (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return {"error": "Webmention URL fingerprint conflict"}, 409
+
should_verify = False
if webmention is None:
@@ -364,10 +371,14 @@ def endpoint() -> ResponseReturnValue:
# after our SELECT but before our COMMIT.
webmention = db.session.execute(
sa.select(ReceivedWebmention).where(
- ReceivedWebmention.source == source,
- ReceivedWebmention.target == target,
+ ReceivedWebmention.url_digest == url_digest
)
- ).scalar_one()
+ ).scalar_one_or_none()
+ if webmention is None or (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return {"error": "Webmention URL fingerprint conflict"}, 409
else:
now = datetime.now(UTC)