diff options
| author | Dennis Fink | 2026-10-31 17:42:00 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-31 17:42:00 +0100 |
| commit | 6924ad7e52033365c55d5afacdd916c502cc80cb (patch) | |
| tree | 8a52b8e8fbae7a6bcc85d250cac2436a2c42e748 /webmentions_ssg | |
| parent | 3494eab1208dc18478485e02c61e292556502fa6 (diff) | |
| download | webmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.tar.gz webmentions-ssg-6924ad7e52033365c55d5afacdd916c502cc80cb.zip | |
feat(receiver): add resource limits and retention
Bound incoming request and URL sizes so untrusted Webmention submissions
cannot consume unbounded application or database resources.
Purge old failed and deleted received Webmentions on a schedule, and use
portable SHA-256 URL fingerprints to preserve uniqueness for long URLs.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/__init__.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 26 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py | 109 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 22 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 41 | ||||
| -rw-r--r-- | webmentions_ssg/url_digest.py | 21 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 23 |
10 files changed, 257 insertions, 11 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 51635ef..e4a0a30 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -96,6 +96,14 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames( app.config["ALLOWED_HOSTNAMES"] ) + for key in ( + "MAX_CONTENT_LENGTH", + "MAX_WEBMENTION_URL_BYTES", + "RECEIVED_RETENTION_DAYS", + ): + value = app.config[key] + if type(value) is not int or value <= 0: + raise ValueError(f"{key} must be a positive integer.") if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]: raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 28cea64..97dd682 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -42,6 +42,13 @@ class DefaultConfig: # Empty by default: no target hostnames are accepted until configured. ALLOWED_HOSTNAMES = set() + # Cap incoming HTTP request bodies (also applies to other Flask routes). + MAX_CONTENT_LENGTH = 64 * 1024 + # Independent UTF-8 byte limit for each submitted source/target URL. + MAX_WEBMENTION_URL_BYTES = 8192 + # Purge terminal failed/deleted received Webmentions after 30 days. + RECEIVED_RETENTION_DAYS = 30 + RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index f698026..5006365 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,7 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral +from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes class LoginForm(FlaskForm): @@ -35,6 +35,7 @@ class EndpointForm(FlaskForm): "source", validators=[ InputRequired(), + URLMaxBytes(), URL(), Regexp( "^https?://.*", @@ -49,6 +50,7 @@ class EndpointForm(FlaskForm): "target", validators=[ InputRequired(), + URLMaxBytes(), URL(), Regexp( "^https?://.*", diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index 2ce2b5b..36f36ce 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -14,6 +14,32 @@ from wtforms.form import BaseForm from ..url_security import AddressResolutionError, is_public_url +class URLMaxBytes: + """Check the UTF-8 byte length of a submitted URL using the current config.""" + + def __init__( + self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None + ) -> None: + self.config_key = config_key + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + if field.data is None: + return + + max_bytes = current_app.config[self.config_key] + + try: + length = len(field.data.encode("utf-8")) + except UnicodeEncodeError as exc: + raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc + + if length > max_bytes: + raise ValidationError( + self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") + ) + + class NotEqualTo: """ Validate that a field does not equal another field. diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py new file mode 100644 index 0000000..0157df3 --- /dev/null +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Replace long URL uniqueness indexes with portable SHA-256 fingerprints. + +Revision ID: d845aa7fbc19 +Revises: c0272f25ce17 +""" + +import sqlalchemy as sa +from alembic import op + +from webmentions_ssg.url_digest import received_url_digest, sent_target_digest + +revision = "d845aa7fbc19" +down_revision = "c0272f25ce17" +branch_labels = None +depends_on = None + + +def _backfill_received() -> None: + table = sa.table( + "received_webmentions", + sa.column("uuid", sa.Uuid()), + sa.column("source", sa.Text()), + sa.column("target", sa.Text()), + sa.column("url_digest", sa.String(64)), + ) + conn = op.get_bind() + while True: + # Bounded batches avoid reading the entire production database into RAM. + rows = conn.execute( + sa.select(table.c.uuid, table.c.source, table.c.target) + .where(table.c.url_digest.is_(None)) + .limit(500) + ).all() + if not rows: + return + for identifier, source, target in rows: + conn.execute( + table.update() + .where(table.c.uuid == identifier) + .values(url_digest=received_url_digest(source, target)) + ) + + +def _backfill_sent() -> None: + table = sa.table( + "sent_webmentions", + sa.column("uuid", sa.Uuid()), + sa.column("target", sa.Text()), + sa.column("target_digest", sa.String(64)), + ) + conn = op.get_bind() + while True: + rows = conn.execute( + sa.select(table.c.uuid, table.c.target) + .where(table.c.target_digest.is_(None)) + .limit(500) + ).all() + if not rows: + return + for identifier, target in rows: + conn.execute( + table.update() + .where(table.c.uuid == identifier) + .values(target_digest=sent_target_digest(target)) + ) + + +def upgrade() -> None: + with op.batch_alter_table("received_webmentions") as batch: + batch.add_column(sa.Column("url_digest", sa.String(64), nullable=True)) + with op.batch_alter_table("sent_webmentions") as batch: + batch.add_column(sa.Column("target_digest", sa.String(64), nullable=True)) + + _backfill_received() + _backfill_sent() + + # Batch operations are required for portable SQLite support. The source + # and target Text columns remain unchanged, with no vendor-specific SQL. + with op.batch_alter_table("received_webmentions") as batch: + batch.alter_column("url_digest", existing_type=sa.String(64), nullable=False) + batch.drop_constraint("uq_webmention_source_target", type_="unique") + batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"]) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.alter_column("target_digest", existing_type=sa.String(64), nullable=False) + batch.drop_constraint("uq_sent_webmention_source_target", type_="unique") + batch.create_unique_constraint( + "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"] + ) + + +def downgrade() -> None: + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique") + batch.create_unique_constraint( + "uq_sent_webmention_source_target", ["source_id", "target"] + ) + batch.drop_column("target_digest") + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_url_digest", type_="unique") + batch.create_unique_constraint( + "uq_webmention_source_target", ["source", "target"] + ) + batch.drop_column("url_digest") diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 7cab879..0dfbb78 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -25,10 +25,11 @@ from sqlalchemy import ( UniqueConstraint, Uuid, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from werkzeug.security import check_password_hash, generate_password_hash from . import Base +from .url_digest import received_url_digest, sent_target_digest class SentWebmentionStatus(StrEnum): @@ -109,6 +110,15 @@ class ReceivedWebmention(Base): source: Mapped[str] = mapped_column(Text(), nullable=False) target: Mapped[str] = mapped_column(Text(), nullable=False) + url_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("source", "target") + def _update_url_digest(self, key: str, value: str) -> str: + source = value if key == "source" else self.__dict__.get("source") + target = value if key == "target" else self.__dict__.get("target") + if source is not None and target is not None: + self.url_digest = received_url_digest(source, target) + return value status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") failure_reason: Mapped[str | None] = mapped_column( @@ -127,7 +137,7 @@ class ReceivedWebmention(Base): ) __table_args__ = ( - UniqueConstraint("source", "target", name="uq_webmention_source_target"), + UniqueConstraint("url_digest", name="uq_webmention_url_digest"), Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) @@ -226,6 +236,12 @@ class SentWebmention(Base): ) target: Mapped[str] = mapped_column(Text(), nullable=False) + target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("target") + def _update_target_digest(self, key: str, value: str) -> str: + self.target_digest = sent_target_digest(value) + return value active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) @@ -275,7 +291,7 @@ class SentWebmention(Base): __table_args__ = ( UniqueConstraint( - "source_id", "target", name="uq_sent_webmention_source_target" + "source_id", "target_digest", name="uq_sent_webmention_source_target_digest" ), ) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index afae6e0..fb1d46d 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,7 +21,7 @@ def create_consumer() -> Huey: """ app = create_app() - from . import notifications, receiver, scanner, sender # noqa: F401 + from . import notifications, receiver, retention, scanner, sender # noqa: F401 schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) @@ -36,6 +36,11 @@ def create_consumer() -> Huey: ) HUEY.periodic_task(notification_schedule)(notifications.send_webmention_digest) + retention_schedule = crontab( + *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True + ) + HUEY.periodic_task(retention_schedule)(retention.purge_old_received_webmentions) + return HUEY.huey diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py new file mode 100644 index 0000000..727b186 --- /dev/null +++ b/webmentions_ssg/tasks/retention.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Delete old received Webmentions in terminal unsuccessful states.""" + +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +import sqlalchemy as sa +from flask import current_app +from sqlalchemy.engine import CursorResult + +from .. import DATABASE as db +from ..models import ReceivedWebmention + + +def purge_old_received_webmentions() -> int: + """Delete old failed/deleted received rows, retaining verified and in-flight ones. + + The status/time predicate is checked by the database in one DELETE, so a + row recently requeued for verification is not purged based on a stale read. + Outgoing sources and Webmentions are intentionally unaffected. + """ + cutoff = datetime.now(UTC) - timedelta( + days=current_app.config["RECEIVED_RETENTION_DAYS"] + ) + result = cast( + CursorResult[Any], + db.session.execute( + sa.delete(ReceivedWebmention).where( + ReceivedWebmention.status.in_(("failed", "deleted")), + ReceivedWebmention.updated_at < cutoff, + ) + ), + ) + db.session.commit() + removed = result.rowcount + if removed: + current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + return removed diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py new file mode 100644 index 0000000..256d037 --- /dev/null +++ b/webmentions_ssg/url_digest.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Portable SHA-256 fingerprints for indexed Webmention URLs.""" + +import hashlib +import json + + +def received_url_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair.""" + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def sent_target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID).""" + return hashlib.sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index 1dcdd2e..fcb5f2d 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -33,6 +33,7 @@ from .models import ReceivedWebmention, SentWebmention, Source, User from .tasks.receiver import verify_webmention from .tasks.scanner import manual_scan_sources from .tasks.sender import send_webmention +from .url_digest import received_url_digest from .url_security import is_safe_next_url root_page = Blueprint("root", __name__) @@ -339,12 +340,18 @@ def endpoint() -> ResponseReturnValue: source = cast(str, form.source.data) target = cast(str, form.target.data) + url_digest = received_url_digest(source, target) webmention = db.session.execute( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.source == source, ReceivedWebmention.target == target - ) + sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) ).scalar_one_or_none() + # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. + if webmention is not None and (webmention.source, webmention.target) != ( + source, + target, + ): + return {"error": "Webmention URL fingerprint conflict"}, 409 + should_verify = False if webmention is None: @@ -364,10 +371,14 @@ def endpoint() -> ResponseReturnValue: # after our SELECT but before our COMMIT. webmention = db.session.execute( sa.select(ReceivedWebmention).where( - ReceivedWebmention.source == source, - ReceivedWebmention.target == target, + ReceivedWebmention.url_digest == url_digest ) - ).scalar_one() + ).scalar_one_or_none() + if webmention is None or (webmention.source, webmention.target) != ( + source, + target, + ): + return {"error": "Webmention URL fingerprint conflict"}, 409 else: now = datetime.now(UTC) |
