aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/__init__.py117
-rw-r--r--webmentions_ssg/auth.py53
-rw-r--r--webmentions_ssg/cli.py (renamed from webmentions_ssg/cli/users.py)94
-rw-r--r--webmentions_ssg/cli/__init__.py5
-rw-r--r--webmentions_ssg/cli/database.py49
-rw-r--r--webmentions_ssg/cli/mail.py53
-rw-r--r--webmentions_ssg/config.py139
-rw-r--r--webmentions_ssg/extensions.py29
-rw-r--r--webmentions_ssg/forms.py (renamed from webmentions_ssg/forms/validators.py)107
-rw-r--r--webmentions_ssg/forms/__init__.py78
-rw-r--r--webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py4
-rw-r--r--webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py4
-rw-r--r--webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py4
-rw-r--r--webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py2
-rw-r--r--webmentions_ssg/models.py68
-rw-r--r--webmentions_ssg/net.py (renamed from webmentions_ssg/public_transport.py)135
-rw-r--r--webmentions_ssg/security.py51
-rw-r--r--webmentions_ssg/tasks/consumer.py2
-rw-r--r--webmentions_ssg/tasks/discovery.py231
-rw-r--r--webmentions_ssg/tasks/extension.py53
-rw-r--r--webmentions_ssg/tasks/json_serializer.py133
-rw-r--r--webmentions_ssg/tasks/mentions.py99
-rw-r--r--webmentions_ssg/tasks/notifications.py11
-rw-r--r--webmentions_ssg/tasks/receiver.py281
-rw-r--r--webmentions_ssg/tasks/retention.py11
-rw-r--r--webmentions_ssg/tasks/scanner.py414
-rw-r--r--webmentions_ssg/tasks/sender.py405
-rw-r--r--webmentions_ssg/tasks/source_parser.py390
-rw-r--r--webmentions_ssg/templates/received.html51
-rw-r--r--webmentions_ssg/templates/sent_source.html42
-rw-r--r--webmentions_ssg/url_digest.py21
-rw-r--r--webmentions_ssg/url_security.py133
-rw-r--r--webmentions_ssg/views.py454
-rw-r--r--webmentions_ssg/views/__init__.py16
-rw-r--r--webmentions_ssg/views/admin.py217
-rw-r--r--webmentions_ssg/views/endpoint.py150
-rw-r--r--webmentions_ssg/views/session.py93
37 files changed, 2086 insertions, 2113 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index 8fed1a7..63e9f13 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -8,47 +8,20 @@ import logging
import logging.handlers
import os
from pathlib import Path
-from urllib.parse import urlsplit
import coloredlogs
from flask import Flask
-from flask_bootstrap import Bootstrap5
-from flask_limiter import Limiter
-from flask_limiter.util import get_remote_address
-from flask_mail import Mail
-from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy
-from flask_wtf.csrf import CSRFProtect
-from sqlalchemy.orm import DeclarativeBase
from werkzeug.middleware.proxy_fix import ProxyFix
-from webmentions_ssg.config import (
- DefaultConfig,
- DevelopmentConfig,
- ProductionConfig,
- normalize_allowed_hostnames,
- normalize_ignored_hostnames,
-)
-
-from .security import validate_secret_key
-from .tasks import Huey
-from .url_security import is_http_url
-
APP_NAME = "webmentions-ssg"
VERSION = "0.0.2"
-
-class Base(DeclarativeBase):
- pass
-
-
-CSRF = CSRFProtect()
-BOOTSTRAP = Bootstrap5()
-DATABASE = SQLAlchemy(model_class=Base)
-MIGRATE = Migrate()
-HUEY = Huey()
-MAIL = Mail()
-LIMITER = Limiter(key_func=get_remote_address)
+from .auth import AUTH
+from .cli import DB_CLI, MAIL_CLI, USERS_CLI
+from .config import DefaultConfig, DevelopmentConfig, ProductionConfig, validate_config
+from .extensions import BOOTSTRAP, CSRF, DATABASE, HUEY, LIMITER, MAIL, MIGRATE
+from .net import is_http_url
+from .views import register_routes
def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
@@ -91,71 +64,15 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config.from_prefixed_env(prefix="FLASK")
app.config.from_prefixed_env(prefix="WEBMENTIONS_SSG")
- # A fixed key is convenient only for local debugging. All other
- # deployments must supply their own persistent, random signing key.
- using_local_dev_key = (
- issubclass(config_class, DevelopmentConfig)
- and app.debug
- and app.config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY
- )
- if not app.testing and not using_local_dev_key:
- validate_secret_key(app.config.get("SECRET_KEY"))
+ app.config.update(validate_config(app.config, config_class=config_class))
- app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
- app.config["ALLOWED_HOSTNAMES"]
- )
- app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames(
- app.config["IGNORED_HOSTNAMES"]
- )
- proxy_hops = app.config["PROXY_FIX_TRUSTED_HOPS"]
- if type(proxy_hops) is not int or proxy_hops < 0:
- raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.")
- if proxy_hops:
- # Trust only the configured number of proxies; never trust forwarded
- # host, port or URL prefix headers for rate limiting.
+ if proxy_hops := app.config["PROXY_FIX_TRUSTED_HOPS"]:
+ # Do not trust forwarded host, port, or URL prefixes.
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=proxy_hops, x_proto=proxy_hops)
- for key in (
- "MAX_CONTENT_LENGTH",
- "MAX_WEBMENTION_URL_BYTES",
- "RECEIVED_RETENTION_DAYS",
- ):
- value = app.config[key]
- if type(value) is not int or value <= 0:
- raise ValueError(f"{key} must be a positive integer.")
- if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]:
- raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.")
-
- if (
- not app.testing
- and not app.debug
- and app.config["RATELIMIT_STORAGE_URI"] == "memory://"
- ):
- raise RuntimeError(
- "Configure RATELIMIT_STORAGE_URI with shared storage before running in production."
- )
-
- if not app.testing and not app.debug:
- # HUEY_URL overrides the legacy HUEY.url option; absent either, Huey
- # falls back to an in-process queue that cannot serve a separate worker.
- huey_url = app.config.get(
- "HUEY_URL", app.config.get("HUEY", {}).get("url", "memory://")
- )
- if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in {
- "memory",
- "blackhole",
- }:
- raise RuntimeError(
- "Configure HUEY_URL with a shared, persistent queue before "
- "running in production (memory:// and blackhole:// are unsafe)."
- )
-
- # Local development and tests should start from a fresh checkout: their
- # default log directory and development SQLite instance are generated.
- if app.testing or app.debug:
- Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True)
- if app.debug:
+ if app.debug or app.testing:
Path(app.instance_path).mkdir(parents=True, exist_ok=True)
+ Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True)
default_logging_formatter = logging.Formatter(app.config["LOG_FORMAT"])
@@ -197,8 +114,6 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
LIMITER.init_app(app)
MAIL.init_app(app)
- from .auth import AUTH
-
AUTH.init_app(app)
BOOTSTRAP.init_app(app)
HUEY.init_app(app)
@@ -214,20 +129,12 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.add_template_filter(is_http_url)
- from .views import root_page
-
- app.register_blueprint(root_page)
-
- from .cli.users import USERS_CLI
+ app.register_blueprint(register_routes())
app.cli.add_command(USERS_CLI)
- from .cli.mail import MAIL_CLI
-
app.cli.add_command(MAIL_CLI)
- from .cli.database import DB_CLI
-
app.cli.add_command(DB_CLI)
return app
diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py
index 147220e..633aa82 100644
--- a/webmentions_ssg/auth.py
+++ b/webmentions_ssg/auth.py
@@ -4,17 +4,43 @@
"""Configure user authentication and session loading."""
+import hashlib
+import secrets
from hmac import compare_digest
+from urllib.parse import urlsplit
+from flask import request
+from flask_limiter.util import get_remote_address
from flask_login import LoginManager
-from . import DATABASE as db
+from .extensions import DATABASE as db
from .models import User
AUTH = LoginManager()
AUTH.login_view = "root.login" # pyright: ignore[reportAttributeAccessIssue]
+MIN_PASSWORD_LENGTH = 8
+
+DUMMY_USER = User(username=secrets.token_urlsafe())
+DUMMY_USER.password = secrets.token_urlsafe()
+
+
+def validate_password(password: str) -> None:
+ """Validate newly chosen passwords without arbitrary composition rules.
+
+ Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases,
+ but reject empty passwords and control characters.
+ """
+ if not password:
+ raise ValueError("Password must not be empty.")
+ if len(password) < MIN_PASSWORD_LENGTH:
+ raise ValueError(
+ f"Password must be at least {MIN_PASSWORD_LENGTH} characters long."
+ )
+ if any(ord(char) < 32 or ord(char) == 127 for char in password):
+ raise ValueError("Password must not contain control characters.")
+
@AUTH.user_loader
def load_user(session_id: str) -> User | None:
@@ -44,3 +70,28 @@ def load_user(session_id: str) -> User | None:
if not compare_digest(session_id, user.get_id()):
return None
return user
+
+
+def login_username_limit_key() -> str:
+ """Group usernames across IPs without exposing plaintext to rate-limit storage."""
+ username = request.form.get("username", "").strip().casefold()
+ if not username:
+ return f"missing-username:{get_remote_address()}"
+ digest = hashlib.sha256(username.encode("utf-8")).hexdigest()
+ return f"username:{digest}"
+
+
+def is_safe_next_url(next_url: str) -> bool:
+ """Allow redirects only to paths on this site."""
+ if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"):
+ return False
+ try:
+ target = urlsplit(next_url)
+ except ValueError:
+ return False
+ return (
+ not target.scheme
+ and not target.netloc
+ and target.path.startswith("/")
+ and not target.path.startswith("//")
+ )
diff --git a/webmentions_ssg/cli/users.py b/webmentions_ssg/cli.py
index a48fc71..0387ba6 100644
--- a/webmentions_ssg/cli/users.py
+++ b/webmentions_ssg/cli.py
@@ -2,22 +2,26 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Provide commands for managing application user accounts."""
+"""Manage users, mail settings, and database migrations."""
import click
import sqlalchemy as sa
+from flask import current_app
from flask.cli import AppGroup
+from flask_mail import Message
+from flask_migrate import upgrade as migrate_upgrade
+from sqlalchemy import inspect
from sqlalchemy.exc import IntegrityError
-from .. import DATABASE
-from ..security import validate_password
+from .auth import validate_password
+from .extensions import DATABASE, MAIL
+from .models import User
USERS_CLI = AppGroup("users", help="Manage users.")
def _require_user(username: str):
"""Return a user by username or report a CLI-friendly error."""
- from ..models import User
user = DATABASE.session.scalar(sa.select(User).where(User.username == username))
if user is None:
@@ -36,7 +40,6 @@ def create_user(username: str, password: str) -> None:
:param password: Plain-text password to hash and store.
:raises click.ClickException: If the credentials are invalid or the user exists.
"""
- from ..models import User
username = username.strip()
if not username:
@@ -63,7 +66,6 @@ def create_user(username: str, password: str) -> None:
@USERS_CLI.command("list", help="List registered users.")
def list_users() -> None:
"""List user IDs and usernames in alphabetical order."""
- from ..models import User
users = DATABASE.session.scalars(sa.select(User).order_by(User.username)).all()
if not users:
@@ -115,3 +117,83 @@ def delete_user(username: str, yes: bool) -> None:
DATABASE.session.delete(user)
DATABASE.session.commit()
click.echo(f"Deleted user {username!r}.")
+
+
+MAIL_CLI = AppGroup("mail", help="Handle mail")
+
+
+@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration")
+@click.option(
+ "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)."
+)
+def send_test_mail(recipient: str | None) -> None:
+ """
+ Send a test email using the configured SMTP server.
+
+ :param recipient: Optional recipient overriding NOTIFICATION_EMAIL.
+ :raises click.ClickException: If configuration is missing or sending fails.
+ """
+ recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL")
+ if not recipient:
+ raise click.ClickException(
+ "No recipient specified. Set NOTIFICATION_EMAIL or pass --to."
+ )
+
+ sender = current_app.config.get("MAIL_DEFAULT_SENDER")
+ if not sender:
+ raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.")
+
+ message = Message(
+ subject="webmentions-ssg: test email",
+ sender=sender,
+ recipients=[recipient],
+ body=(
+ "This is a test email from webmentions-ssg.\n\n"
+ "If you received this message, the outgoing mail configuration works.\n"
+ ),
+ )
+
+ try:
+ MAIL.send(message)
+ except Exception as exc:
+ raise click.ClickException(f"Could not send test email: {exc}") from exc
+
+ click.echo(f"Test email sent to {recipient}.")
+
+
+DB_CLI = AppGroup("db", help="Handle database")
+
+
+def _table_names() -> set[str]:
+ """Return existing table names without modifying the database schema."""
+ with DATABASE.engine.connect() as connection:
+ return set(inspect(connection).get_table_names())
+
+
+@DB_CLI.command("create-db", help="Create a new database using all migrations")
+def create_db() -> None:
+ """Initialize an empty database and record its Alembic revision."""
+ if _table_names():
+ raise click.ClickException(
+ "The database already contains tables. "
+ "Use 'flask db upgrade' for an existing migrated database."
+ )
+
+ # Applying migrations (rather than create_all()) keeps alembic_version in sync.
+ migrate_upgrade()
+ click.echo("Database created at the latest migration revision.")
+
+
+@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision")
+def upgrade_db() -> None:
+ """Apply pending migrations to a previously initialized database."""
+ if "alembic_version" not in _table_names():
+ raise click.ClickException(
+ "No Flask-Migrate revision table found. "
+ "For a new, empty database run 'flask db create-db'. "
+ "For a legacy database, inspect its schema and use "
+ "'flask migrate stamp' only if appropriate."
+ )
+
+ migrate_upgrade()
+ click.echo("Database upgraded to the latest migration revision.")
diff --git a/webmentions_ssg/cli/__init__.py b/webmentions_ssg/cli/__init__.py
deleted file mode 100644
index accaaa3..0000000
--- a/webmentions_ssg/cli/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Group the application command-line management modules."""
diff --git a/webmentions_ssg/cli/database.py b/webmentions_ssg/cli/database.py
deleted file mode 100644
index e918600..0000000
--- a/webmentions_ssg/cli/database.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Simple database commands built on Flask-Migrate."""
-
-import click
-from flask.cli import AppGroup
-from flask_migrate import upgrade as migrate_upgrade
-from sqlalchemy import inspect
-
-from .. import DATABASE
-
-DB_CLI = AppGroup("db", help="Handle database")
-
-
-def _table_names() -> set[str]:
- """Return existing table names without modifying the database schema."""
- with DATABASE.engine.connect() as connection:
- return set(inspect(connection).get_table_names())
-
-
-@DB_CLI.command("create-db", help="Create a new database using all migrations")
-def create_db() -> None:
- """Initialize an empty database and record its Alembic revision."""
- if _table_names():
- raise click.ClickException(
- "The database already contains tables. "
- "Use 'flask db upgrade' for an existing migrated database."
- )
-
- # Applying migrations (rather than create_all()) keeps alembic_version in sync.
- migrate_upgrade()
- click.echo("Database created at the latest migration revision.")
-
-
-@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision")
-def upgrade_db() -> None:
- """Apply pending migrations to a previously initialized database."""
- if "alembic_version" not in _table_names():
- raise click.ClickException(
- "No Flask-Migrate revision table found. "
- "For a new, empty database run 'flask db create-db'. "
- "For a legacy database, inspect its schema and use "
- "'flask migrate stamp' only if appropriate."
- )
-
- migrate_upgrade()
- click.echo("Database upgraded to the latest migration revision.")
diff --git a/webmentions_ssg/cli/mail.py b/webmentions_ssg/cli/mail.py
deleted file mode 100644
index 47687e2..0000000
--- a/webmentions_ssg/cli/mail.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Provide a command for testing outgoing email configuration."""
-
-import click
-from flask import current_app
-from flask.cli import AppGroup
-from flask_mail import Message
-
-from .. import MAIL
-
-MAIL_CLI = AppGroup("mail", help="Handle mail")
-
-
-@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration")
-@click.option(
- "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)."
-)
-def send_test_mail(recipient: str | None) -> None:
- """
- Send a test email using the configured SMTP server.
-
- :param recipient: Optional recipient overriding NOTIFICATION_EMAIL.
- :raises click.ClickException: If configuration is missing or sending fails.
- """
- recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL")
- if not recipient:
- raise click.ClickException(
- "No recipient specified. Set NOTIFICATION_EMAIL or pass --to."
- )
-
- sender = current_app.config.get("MAIL_DEFAULT_SENDER")
- if not sender:
- raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.")
-
- message = Message(
- subject="webmentions-ssg: test email",
- sender=sender,
- recipients=[recipient],
- body=(
- "This is a test email from webmentions-ssg.\n\n"
- "If you received this message, the outgoing mail configuration works.\n"
- ),
- )
-
- try:
- MAIL.send(message)
- except Exception as exc:
- raise click.ClickException(f"Could not send test email: {exc}") from exc
-
- click.echo(f"Test email sent to {recipient}.")
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 8a3e58d..8251da3 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -5,39 +5,11 @@
"""Define application settings for production, development, and testing."""
import logging
+from urllib.parse import urlsplit
+from flask.config import Config
-def normalize_allowed_hostnames(value: object) -> set[str]:
- """Validate and normalize the configured collection of allowed hostnames."""
- if not isinstance(value, (list, tuple, set, frozenset)):
- raise TypeError(
- "ALLOWED_HOSTNAMES must be a list, tuple, or set of hostnames, not a string."
- )
-
- hostnames: set[str] = set()
- for hostname in value:
- if not isinstance(hostname, str) or not hostname.strip():
- raise ValueError("ALLOWED_HOSTNAMES must contain non-empty strings only.")
- hostnames.add(hostname.strip().lower())
- return hostnames
-
-
-def normalize_ignored_hostnames(value: object) -> tuple[str, ...]:
- """Validate and normalize hostname patterns ignored by the source scanner."""
- if not isinstance(value, (list, tuple, set, frozenset)):
- raise TypeError(
- "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, "
- "not a string."
- )
-
- patterns: set[str] = set()
- for pattern in value:
- if not isinstance(pattern, str) or not (
- normalized := pattern.strip().lower().rstrip(".")
- ):
- raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.")
- patterns.add(normalized)
- return tuple(sorted(patterns))
+MIN_SECRET_KEY_LENGTH = 32
class DefaultConfig:
@@ -69,6 +41,7 @@ class DefaultConfig:
RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
+ MAX_TARGET_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
# Absolute wall-clock budget for each verification or delivery attempt.
REQUEST_TOTAL_TIMEOUT = 30.0
@@ -144,3 +117,107 @@ class TestingConfig(DefaultConfig):
HUEY_URL = "memory://"
ALLOWED_HOSTNAMES = {"dennisfink.me"}
+
+
+def normalize_hostnames(value: object) -> set[str]:
+ """Validate and normalize a collection of hostnames."""
+ if not isinstance(value, (list, tuple, set, frozenset)):
+ raise TypeError("Expected a list, tuple, or set of hostnames.")
+
+ hostnames: set[str] = set()
+ for hostname in value:
+ if not isinstance(hostname, str) or not (
+ normalized := hostname.strip().lower().rstrip(".")
+ ):
+ raise ValueError("Hostnames must be non-empty strings.")
+
+ hostnames.add(normalized)
+
+ return hostnames
+
+
+def validate_secret_key(value: object) -> str | bytes:
+ """Require a non-placeholder, sufficiently long deployment signing key."""
+ if not isinstance(value, (str, bytes)):
+ raise TypeError(
+ "Configure a persistent SECRET_KEY of at least 32 random bytes "
+ "before starting."
+ )
+
+ # Flask accepts either text or bytes as its signing key. Reject common
+ # example/development values even when their lengths look sufficient.
+ key = value.encode("utf-8") if isinstance(value, str) else value
+ if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip():
+ raise RuntimeError(
+ "Configure a persistent SECRET_KEY of at least 32 random bytes "
+ "before starting."
+ )
+
+ normalized = key.strip().lower()
+ if normalized.startswith(
+ (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_")
+ ):
+ raise RuntimeError(
+ "SECRET_KEY is an example/development placeholder; "
+ "generate and configure a real random key."
+ )
+
+ return value
+
+
+def validate_config(
+ config: Config, *, config_class: type[DefaultConfig]
+) -> dict[str, object]:
+ """Validate runtime configuration and return normalized values.
+
+ The caller applies the returned values once; validation does not mutate
+ the Flask application or install extensions.
+ """
+ normalized: dict[str, object] = {}
+ for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"):
+ try:
+ normalized[key] = normalize_hostnames(config[key])
+ except (TypeError, ValueError) as exc:
+ raise type(exc)(f"{key}: {exc}") from exc
+
+ proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"]
+ if type(proxy_hops) is not int or proxy_hops < 0:
+ raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.")
+
+ for key in (
+ "MAX_CONTENT_LENGTH",
+ "MAX_WEBMENTION_URL_BYTES",
+ "RECEIVED_RETENTION_DAYS",
+ ):
+ value = config[key]
+ if type(value) is not int or value <= 0:
+ raise ValueError(f"{key} must be a positive integer.")
+
+ if not config["TESTING"] and not (
+ issubclass(config_class, DevelopmentConfig)
+ and config["DEBUG"]
+ and config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY
+ ):
+ normalized["SECRET_KEY"] = validate_secret_key(config.get("SECRET_KEY"))
+
+ if not config["DEBUG"] and not config["TESTING"]:
+ if not normalized["ALLOWED_HOSTNAMES"]:
+ raise RuntimeError(
+ "Configure ALLOWED_HOSTNAMES before running in production."
+ )
+ if config["RATELIMIT_STORAGE_URI"] == "memory://":
+ raise RuntimeError(
+ "Configure RATELIMIT_STORAGE_URI with shared storage before running in production."
+ )
+ huey_url = config.get(
+ "HUEY_URL", config.get("HUEY", {}).get("url", "memory://")
+ )
+ if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in {
+ "memory",
+ "blackhole",
+ }:
+ raise RuntimeError(
+ "Configure HUEY_URL with a shared, persistent queue before "
+ "running in production (memory:// and blackhole:// are unsafe)."
+ )
+ return normalized
diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py
new file mode 100644
index 0000000..0afec9c
--- /dev/null
+++ b/webmentions_ssg/extensions.py
@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Unbound extensions shared across Flask application instances."""
+
+from flask_bootstrap import Bootstrap5
+from flask_limiter import Limiter
+from flask_limiter.util import get_remote_address
+from flask_mail import Mail
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from flask_wtf.csrf import CSRFProtect
+from sqlalchemy.orm import DeclarativeBase
+
+from .tasks.extension import Huey
+
+
+class Base(DeclarativeBase):
+ pass
+
+
+CSRF = CSRFProtect()
+BOOTSTRAP = Bootstrap5()
+DATABASE = SQLAlchemy(model_class=Base)
+MIGRATE = Migrate()
+HUEY = Huey()
+MAIL = Mail()
+LIMITER = Limiter(key_func=get_remote_address)
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms.py
index 384c7a1..a721dc6 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms.py
@@ -2,19 +2,21 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Provide validators for URL safety and form field constraints."""
+"""Webmention forms and URL validation."""
import ipaddress
+import re
import unicodedata
from urllib.parse import urlsplit
import httpx
from flask import current_app
-from wtforms import Field, ValidationError
-from wtforms.validators import StopValidation
+from flask_wtf import FlaskForm
+from wtforms import Field, PasswordField, StringField, SubmitField, ValidationError
from wtforms.form import BaseForm
+from wtforms.validators import URL, InputRequired, Regexp, StopValidation
-from ..url_security import AddressResolutionError, is_public_address, is_public_url
+from .net import is_public_address
class URLMaxBytes:
@@ -143,7 +145,7 @@ class AllowedHostname:
:raises ValidationError: If the URL hostname is not configured as allowed.
"""
try:
- hostname = urlsplit(field.data).hostname
+ hostname = (urlsplit(field.data).hostname or "").rstrip(".")
except ValueError, TypeError:
hostname = None
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
@@ -151,42 +153,6 @@ class AllowedHostname:
raise ValidationError(self.message or field.gettext("Invalid input."))
-class PublicURL:
- """
- Validate that a URL resolves exclusively to public IP addresses.
-
- :param message: Optional validation error message.
- """
-
- def __init__(self, message: str | None = None) -> None:
- """
- Initialize the validator.
-
- :param message: Optional validation error message.
- """
- self.message = message
-
- def __call__(self, form: BaseForm, field: Field) -> None:
- """
- Validate that a URL resolves exclusively to public IP addresses.
-
- :param form: Form containing the field.
- :param field: Field containing the URL to validate.
- :raises ValidationError: If the URL cannot be resolved, has no hostname, or
- resolves to a non-public address.
- """
- message = self.message
-
- if message is None:
- message = field.gettext("URL must resolve to a public address.")
-
- try:
- if not is_public_url(field.data):
- raise ValidationError(message)
- except (AddressResolutionError, ValueError) as exc:
- raise ValidationError(message) from exc
-
-
class PublicIPLiteral:
"""Reject non-public IP literals without performing DNS in the web worker."""
@@ -206,3 +172,62 @@ class PublicIPLiteral:
if not is_public_address(address):
raise ValidationError(field.gettext("URL must use a public address."))
+
+
+class LoginForm(FlaskForm):
+ """
+ Form for authenticating a user.
+ """
+
+ username = StringField("Username", validators=[InputRequired()])
+ password = PasswordField("Password", validators=[InputRequired()])
+ submit = SubmitField("Sign In")
+
+
+class EndpointForm(FlaskForm):
+ """
+ Form for receiving a Webmention.
+
+ The source must be a public HTTP or HTTPS URL distinct from the target.
+ The target must be an HTTP or HTTPS URL using an allowed hostname.
+ """
+
+ source = StringField(
+ "source",
+ validators=[
+ InputRequired(),
+ URLMaxBytes(),
+ HTTPXURL(),
+ URL(),
+ Regexp(
+ "^https?://.*",
+ flags=re.IGNORECASE,
+ message="source must begin with http or https",
+ ),
+ NotEqualTo("target"),
+ PublicIPLiteral(),
+ ],
+ )
+ target = StringField(
+ "target",
+ validators=[
+ InputRequired(),
+ URLMaxBytes(),
+ HTTPXURL(),
+ URL(),
+ Regexp(
+ "^https?://.*",
+ flags=re.IGNORECASE,
+ message="target must begin with http or https",
+ ),
+ AllowedHostname(),
+ ],
+ )
+
+
+class AdminActionForm(FlaskForm):
+ """
+ Form for confirming an administrative action.
+ """
+
+ submit = SubmitField("Confirm")
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
deleted file mode 100644
index 589cbd9..0000000
--- a/webmentions_ssg/forms/__init__.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Define forms for login, Webmention submission, and administration."""
-
-import re
-
-from flask_wtf import FlaskForm
-from wtforms import PasswordField, StringField, SubmitField
-from wtforms.validators import URL, InputRequired, Regexp
-
-from .validators import (
- AllowedHostname,
- HTTPXURL,
- NotEqualTo,
- PublicIPLiteral,
- URLMaxBytes,
-)
-
-
-class LoginForm(FlaskForm):
- """
- Form for authenticating a user.
- """
-
- username = StringField("Username", validators=[InputRequired()])
- password = PasswordField("Password", validators=[InputRequired()])
- submit = SubmitField("Sign In")
-
-
-class EndpointForm(FlaskForm):
- """
- Form for receiving a Webmention.
-
- The source must be a public HTTP or HTTPS URL distinct from the target.
- The target must be an HTTP or HTTPS URL using an allowed hostname.
- """
-
- source = StringField(
- "source",
- validators=[
- InputRequired(),
- URLMaxBytes(),
- HTTPXURL(),
- URL(),
- Regexp(
- "^https?://.*",
- flags=re.IGNORECASE,
- message="source must begin with http or https",
- ),
- NotEqualTo("target"),
- PublicIPLiteral(),
- ],
- )
- target = StringField(
- "target",
- validators=[
- InputRequired(),
- URLMaxBytes(),
- HTTPXURL(),
- URL(),
- Regexp(
- "^https?://.*",
- flags=re.IGNORECASE,
- message="target must begin with http or https",
- ),
- AllowedHostname(),
- ],
- )
-
-
-class AdminActionForm(FlaskForm):
- """
- Form for confirming an administrative action.
- """
-
- submit = SubmitField("Confirm")
diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
index 905b2c2..a263a2f 100644
--- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
+++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Track webmention digest notifications
Revision ID: 408fc2eca9ce
diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
index c4eac52..ee6ad09 100644
--- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
+++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Add Webmention retry backoff
Revision ID: 85f2b226ec99
diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
index cbb40ef..08dd45f 100644
--- a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
+++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
"""Index received webmentions for expired verification recovery.
Revision ID: c0272f25ce17
diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
index 0157df3..41c5376 100644
--- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
+++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py
@@ -11,7 +11,7 @@ Revises: c0272f25ce17
import sqlalchemy as sa
from alembic import op
-from webmentions_ssg.url_digest import received_url_digest, sent_target_digest
+from webmentions_ssg.models import received_url_digest, sent_target_digest
revision = "d845aa7fbc19"
down_revision = "c0272f25ce17"
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 31516d5..9442309 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -6,6 +6,7 @@
from __future__ import annotations
+import json
import uuid
from datetime import UTC, datetime
from enum import StrEnum
@@ -28,8 +29,17 @@ from sqlalchemy import (
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
from werkzeug.security import check_password_hash, generate_password_hash
-from . import Base
-from .url_digest import received_url_digest, sent_target_digest
+from .extensions import Base
+
+
+class ReceivedStatus(StrEnum):
+ """Persisted lifecycle state of an incoming Webmention."""
+
+ RECEIVED = "received"
+ VERIFYING = "verifying"
+ VERIFIED = "verified"
+ FAILED = "failed"
+ DELETED = "deleted"
class SentWebmentionStatus(StrEnum):
@@ -120,7 +130,9 @@ class ReceivedWebmention(Base):
self.url_digest = received_url_digest(source, target)
return value
- status: Mapped[str] = mapped_column(Text(), nullable=False, default="received")
+ status: Mapped[str] = mapped_column(
+ Text(), nullable=False, default=ReceivedStatus.RECEIVED
+ )
# Counts stale-row requeues, not Huey's bounded in-task retries. A fresh
# explicit submission resets this counter.
recovery_attempts: Mapped[int] = mapped_column(
@@ -154,6 +166,24 @@ class ReceivedWebmention(Base):
"""
return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>"
+ @staticmethod
+ def verification_reset_values(
+ *, at: datetime, recoveries: int = 0
+ ) -> dict[str, object]:
+ """Return the columns to reset when verification is explicitly reopened."""
+ return {
+ "status": ReceivedStatus.RECEIVED,
+ "failure_reason": None,
+ "recovery_attempts": recoveries,
+ "updated_at": at,
+ }
+
+ def reset_for_verification(self) -> ReceivedWebmention:
+ """Reopen an existing row for manual verification and return it."""
+ for key, value in self.verification_reset_values(at=datetime.now(UTC)).items():
+ setattr(self, key, value)
+ return self
+
@property
def verified(self) -> bool:
"""
@@ -161,7 +191,7 @@ class ReceivedWebmention(Base):
:return: Whether the Webmention status is ``verified``.
"""
- return self.status == "verified"
+ return self.status == ReceivedStatus.VERIFIED
@property
def created_at(self) -> datetime:
@@ -308,6 +338,23 @@ class SentWebmention(Base):
"""
return f"<SentWebmention {self.target!r}>"
+ def reopen(
+ self, revision: int, active: bool, *, resend: bool = False
+ ) -> SentWebmention:
+ """Reopen delivery for a revision while preserving send history."""
+ self.desired_revision = revision
+ self.active = active
+ self.attempt_count = 0
+ self.next_retry_at = None
+ if resend:
+ self.processed_revision = None
+ self.status = None
+ self.failure_reason = None
+ self.endpoint = None
+ self.response_status = None
+ self.status_url = None
+ return self
+
@property
def pending(self) -> bool:
"""
@@ -338,3 +385,16 @@ def uuid7_to_datetime(identifier: uuid.UUID) -> datetime:
:return: Timestamp as a timezone-aware UTC datetime.
"""
return datetime.fromtimestamp(identifier.time / 1000, tz=UTC)
+
+
+def received_url_digest(source: str, target: str) -> str:
+ """Fingerprint an unambiguous, ordered source/target pair."""
+ encoded = json.dumps(
+ [source, target], ensure_ascii=False, separators=(",", ":")
+ ).encode("utf-8")
+ return sha256(encoded).hexdigest()
+
+
+def sent_target_digest(target: str) -> str:
+ """Fingerprint an outgoing target (unique within each source ID)."""
+ return sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/net.py
index 131cf55..01a9a80 100644
--- a/webmentions_ssg/public_transport.py
+++ b/webmentions_ssg/net.py
@@ -2,23 +2,31 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""HTTP transport that connects only to validated DNS answers.
-
-httpcore receives the original host for Host and TLS SNI/certificate checks,
-but the socket is opened to an IP from a single, bounded DNS lookup. This
-protects the receiver and sender, including redirects and advertised URLs.
-"""
+"""Public-only HTTP transport, URL checks, and bounded response reads."""
import ipaddress
import socket
import time
from collections.abc import Iterable
+from urllib.parse import urlsplit
+import dns.exception
+import dns.resolver
import httpcore
import httpx
-from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation
+from flask import current_app
+from httpcore._backends.sync import SyncStream
+
+NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
+IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
+
+
+class AddressResolutionError(Exception):
+ """A hostname could not be resolved (potentially transient)."""
-from .url_security import public_addresses
+
+class UnsafeAddressError(ValueError):
+ """A hostname or IP literal identifies a non-public network address."""
class NetworkDeadline:
@@ -142,3 +150,114 @@ class PublicHTTPTransport(httpx.HTTPTransport):
ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
)
+
+
+def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
+ """Return the hardened HTTP client shared by receiver and sender."""
+ headers = {"User-Agent": user_agent}
+ if accept is not None:
+ headers["Accept"] = accept
+ return httpx.Client(
+ headers=headers,
+ timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]),
+ follow_redirects=True,
+ max_redirects=current_app.config["MAX_REDIRECTS"],
+ trust_env=False,
+ transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]),
+ )
+
+
+def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+ """Reject non-public addresses, including IPv4 hidden in IPv6 forms.
+
+ ``is_global`` alone does not reliably exclude IPv6 transition addresses
+ across Python versions or networks with NAT64/6to4 routing.
+ """
+ if isinstance(address, ipaddress.IPv6Address):
+ # The well-known NAT64 prefix can translate an otherwise global IPv6
+ # address into a private IPv4 destination. IPv4-compatible addresses
+ # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
+ if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
+ return False
+
+ # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
+ # destination through ipaddress; validate that destination as well.
+ for embedded in (address.sixtofour, address.ipv4_mapped):
+ if embedded is not None and not embedded.is_global:
+ return False
+
+ return address.is_global
+
+
+def public_addresses(
+ hostname: str, *, lifetime: float | None = None
+) -> tuple[str, ...]:
+ """Resolve *once* and return only checked, globally routable IP addresses.
+
+ A DNS name with a mixture of public and private answers is rejected in its
+ entirety. IP literals are handled without DNS, including IPv6 literals.
+ The returned strings must be used for the actual TCP connection; checking
+ them and then connecting to ``hostname`` reintroduces DNS rebinding.
+ """
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ addresses = (literal,)
+ else:
+ try:
+ answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
+ addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
+ except (dns.exception.DNSException, ValueError) as exc:
+ raise AddressResolutionError(
+ f"Could not resolve hostname {hostname!r}"
+ ) from exc
+
+ if not addresses:
+ raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
+
+ if any(not is_public_address(address) for address in addresses):
+ raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
+
+ return tuple(str(address) for address in addresses)
+
+
+def is_http_url(url: str) -> bool:
+ """
+ Check whether a URL is an absolute HTTP or HTTPS URL.
+
+ :param url: URL to validate.
+ :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ """
+ try:
+ parsed = urlsplit(url)
+ except ValueError:
+ return False
+
+ return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
+
+
+def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
+ """Read a streaming response, rejecting oversized declared or actual bodies."""
+ if (content_length := response.headers.get("Content-Length")) is not None:
+ try:
+ declared_length = int(content_length)
+ except ValueError:
+ pass
+ else:
+ if declared_length > max_bytes:
+ raise ValueError(f"{label} document is too large")
+
+ body = bytearray()
+ for chunk in response.iter_bytes(chunk_size=64 * 1024):
+ body.extend(chunk)
+ if len(body) > max_bytes:
+ raise ValueError(f"{label} document is too large")
+ return bytes(body)
+
+
+def is_temporary_status(status: int) -> bool:
+ """Return whether an HTTP status is normally retryable."""
+ return status in {408, 425, 429} or 500 <= status <= 599
diff --git a/webmentions_ssg/security.py b/webmentions_ssg/security.py
deleted file mode 100644
index f4841a4..0000000
--- a/webmentions_ssg/security.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Validate persistent signing keys and administrator passwords."""
-
-MIN_PASSWORD_LENGTH = 8
-MIN_SECRET_KEY_LENGTH = 32
-
-
-def validate_secret_key(value: object) -> None:
- """Require a non-placeholder, sufficiently long deployment signing key."""
- if not isinstance(value, (str, bytes)):
- raise TypeError(
- "Configure a persistent SECRET_KEY of at least 32 random bytes "
- "before starting."
- )
-
- # Flask accepts either text or bytes as its signing key. Reject common
- # example/development values even when their lengths look sufficient.
- key = value.encode("utf-8") if isinstance(value, str) else value
- if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip():
- raise RuntimeError(
- "Configure a persistent SECRET_KEY of at least 32 random bytes "
- "before starting."
- )
-
- normalized = key.strip().lower()
- if normalized.startswith(
- (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_")
- ):
- raise RuntimeError(
- "SECRET_KEY is an example/development placeholder; "
- "generate and configure a real random key."
- )
-
-
-def validate_password(password: str) -> None:
- """Validate newly chosen passwords without arbitrary composition rules.
-
- Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases,
- but reject empty passwords and control characters.
- """
- if not password:
- raise ValueError("Password must not be empty.")
- if len(password) < MIN_PASSWORD_LENGTH:
- raise ValueError(
- f"Password must be at least {MIN_PASSWORD_LENGTH} characters long."
- )
- if any(ord(char) < 32 or ord(char) == 127 for char in password):
- raise ValueError("Password must not contain control characters.")
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index fb1d46d..bf7189f 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,7 +21,7 @@ def create_consumer() -> Huey:
"""
app = create_app()
- from . import notifications, receiver, retention, scanner, sender # noqa: F401
+ from . import notifications, receiver, retention, scanner
schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
HUEY.periodic_task(schedule)(scanner.scan_sources)
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
new file mode 100644
index 0000000..98f3fd5
--- /dev/null
+++ b/webmentions_ssg/tasks/discovery.py
@@ -0,0 +1,231 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Discover a target Webmention endpoint and submit a POST."""
+
+import re
+from urllib.parse import urljoin
+
+import httpx
+from bs4 import BeautifulSoup
+from flask import current_app
+
+from ..net import is_http_url, is_temporary_status, read_limited_body
+
+LINK_SPLIT = re.compile(r",\s*(?=<)")
+DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
+
+
+class SenderError(Exception):
+ """A Webmention cannot be sent."""
+
+
+class TemporarySenderError(SenderError):
+ """A Webmention send failed for a potentially temporary reason."""
+
+
+class PermanentSenderError(SenderError):
+ """A Webmention send failed permanently for this source revision."""
+
+
+def resolve_endpoint(response: httpx.Response, href: str) -> str:
+ """
+ Resolve and validate a discovered Webmention endpoint.
+
+ :param response: Response from which the endpoint was discovered.
+ :param href: Endpoint reference to resolve.
+ :return: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
+ HTTPS URL.
+ """
+ try:
+ endpoint = urljoin(str(response.url), href.strip())
+ except ValueError as exc:
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
+
+ if not is_http_url(endpoint):
+ raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
+
+ return endpoint
+
+
+def parse_link_value(value: str) -> tuple[str, set[str]] | None:
+ """
+ Parse a Link header value into its target and relations.
+
+ :param value: Link header value to parse.
+ :return: Link target and relation names, or ``None`` if the value is invalid.
+ """
+ value = value.strip()
+
+ if not value.startswith("<"):
+ return None
+
+ href, separator, parameters = value[1:].partition(">")
+
+ if not separator:
+ return None
+
+ relations: set[str] = set()
+
+ for parameter in parameters.split(";"):
+ name, separator, value = parameter.partition("=")
+
+ if separator and name.strip().lower() == "rel":
+ relations.update(
+ relation.lower() for relation in value.strip().strip("\"'").split()
+ )
+
+ return href, relations
+
+
+def endpoint_from_headers(response: httpx.Response) -> str | None:
+ """
+ Find a Webmention endpoint in the response Link headers.
+
+ :param response: HTTP response whose headers should be inspected.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ for header in response.headers.get_list("Link"):
+ for value in LINK_SPLIT.split(header):
+ if (link := parse_link_value(value)) is None:
+ continue
+
+ href, relations = link
+
+ if "webmention" in relations:
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
+ """
+ Find a Webmention endpoint in an HTML document.
+
+ ``link`` and ``a`` elements are inspected in document order.
+
+ :param response: HTTP response from which the document was retrieved.
+ :param body: HTML document body.
+ :return: First valid advertised Webmention endpoint, or ``None`` if none
+ is found. Invalid candidates are ignored.
+ """
+ document = BeautifulSoup(body, "html5lib")
+
+ for element in document.find_all(["link", "a"], href=True):
+ relations = element.get("rel")
+
+ if isinstance(relations, str):
+ relations = relations.split()
+
+ if not relations:
+ continue
+
+ if not any(
+ isinstance(relation, str) and relation.lower() == "webmention"
+ for relation in relations
+ ):
+ continue
+
+ href = element.get("href")
+
+ if not isinstance(href, str):
+ continue
+
+ try:
+ return resolve_endpoint(response, href)
+ except PermanentSenderError:
+ continue
+
+ return None
+
+
+def read_target_body(response: httpx.Response) -> bytes:
+ """Return a bounded target body, using the shared HTTP size guard."""
+ try:
+ return read_limited_body(
+ response, current_app.config["MAX_TARGET_BYTES"], label="Target"
+ )
+ except ValueError as exc:
+ raise PermanentSenderError(str(exc)) from exc
+
+
+def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
+ """
+ Discover the Webmention endpoint advertised by a target URL.
+
+ Endpoint discovery first checks the response to a HEAD request and then falls
+ back to a GET request, inspecting both HTTP Link headers and supported HTML
+ documents.
+
+ :param client: HTTP client to use for discovery requests.
+ :param target: Target URL whose Webmention endpoint should be discovered.
+ :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :raises TemporarySenderError: If the target returns a temporary HTTP failure.
+ :raises PermanentSenderError: If GET returns a permanent HTTP failure.
+ """
+ try:
+ head_response = client.head(target, headers=DISCOVERY_HEADERS)
+ except httpx.RequestError, TemporarySenderError:
+ # HEAD is only an optimization. Servers and intermediaries may reject
+ # or drop it even when fetching the document with GET would succeed.
+ pass
+ else:
+ if endpoint := endpoint_from_headers(head_response):
+ return endpoint
+
+ with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
+ if endpoint := endpoint_from_headers(response):
+ return endpoint
+
+ if is_temporary_status(response.status_code):
+ raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
+
+ if not response.is_success:
+ raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
+
+ media_type = (
+ response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
+ )
+
+ if media_type not in {"text/html", "application/xhtml+xml"}:
+ return None
+
+ return endpoint_from_html(response, read_target_body(response))
+
+
+def post_webmention(
+ client: httpx.Client, *, endpoint: str, source: str, target: str
+) -> tuple[int, str | None]:
+ """
+ Send a Webmention to a discovered endpoint.
+
+ :param client: HTTP client to use for the request.
+ :param endpoint: Webmention endpoint URL.
+ :param source: Source URL of the Webmention.
+ :param target: Target URL of the Webmention.
+ :return: HTTP response status code and optional status URL.
+ """
+ with client.stream(
+ "POST", endpoint, data={"source": source, "target": target}
+ ) as response:
+ status_url = None
+
+ if response.status_code == 201 and (
+ location := response.headers.get("Location")
+ ):
+ try:
+ candidate = urljoin(str(response.url), location)
+ except ValueError:
+ # A malformed Location must not fail an otherwise successful send.
+ pass
+ else:
+ if is_http_url(candidate):
+ status_url = candidate
+
+ return (response.status_code, status_url)
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index df6cb21..9f53f3b 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -93,54 +93,37 @@ class Huey:
)
return self._huey
+ def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]:
+ """Return a task wrapper that executes with the configured Flask app."""
+
+ @wraps(func)
+ def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
+ if self.app is None:
+ raise RuntimeError("Flask app is not available.")
+ with self.app.app_context():
+ return func(*args, **kwargs)
+
+ return wrapped
+
def task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's task decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's task decorator.
- :return: Decorator that registers the wrapped function as a Huey task.
- """
+ """Return a Huey task decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func))
return decorator
def periodic_task(
self, *task_args: Any, **task_kwargs: Any
) -> Callable[[Callable[P, object]], TaskWrapper]:
- """
- Create a periodic Huey task that runs within the Flask application context.
-
- :param task_args: Positional arguments forwarded to Huey's periodic task
- decorator.
- :param task_kwargs: Keyword arguments forwarded to Huey's periodic task
- decorator.
- :return: Decorator that registers the wrapped function as a periodic Huey
- task.
- """
+ """Return a periodic Huey decorator with Flask application context."""
def decorator(func: Callable[P, R]) -> TaskWrapper:
- @wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- if self.app is None:
- raise RuntimeError("Flask app is not available.")
-
- with self.app.app_context():
- return func(*args, **kwargs)
-
- return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper)
+ return self.huey.periodic_task(*task_args, **task_kwargs)(
+ self._in_app_context(func)
+ )
return decorator
diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py
index 473a544..7e51358 100644
--- a/webmentions_ssg/tasks/json_serializer.py
+++ b/webmentions_ssg/tasks/json_serializer.py
@@ -11,7 +11,7 @@ from typing import Any
from huey.registry import Message
from huey.serializer import Serializer
-from huey.utils import Error, SKIPPED
+from huey.utils import SKIPPED, Error
class JSONSerializer(Serializer):
@@ -39,66 +39,87 @@ class JSONSerializer(Serializer):
@classmethod
def _encode(cls, value: Any) -> Any:
- if value is None or type(value) in {str, int, float, bool}:
- return value
- if isinstance(value, uuid.UUID):
- return {"type": "uuid", "value": str(value)}
- if isinstance(value, datetime):
- return {"type": "datetime", "value": value.isoformat()}
- if isinstance(value, Message):
- return {"type": "message", "value": [cls._encode(v) for v in value]}
- if isinstance(value, Error):
- return {"type": "error", "value": cls._encode(value.metadata)}
- if value is SKIPPED:
- return {"type": "skipped", "value": None}
- if isinstance(value, tuple):
- return {"type": "tuple", "value": [cls._encode(v) for v in value]}
- if isinstance(value, list):
- return {"type": "list", "value": [cls._encode(v) for v in value]}
- if isinstance(value, dict) and all(type(k) is str for k in value):
- return {
- "type": "dict",
- "value": [[k, cls._encode(v)] for k, v in value.items()],
- }
- raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
+ match value:
+ case _ if value is None or type(value) in {str, int, float, bool}:
+ return value
+
+ case uuid.UUID():
+ return {"type": "uuid", "value": str(value)}
+
+ case datetime():
+ return {"type": "datetime", "value": value.isoformat()}
+
+ case Message():
+ return {"type": "message", "value": [cls._encode(v) for v in value]}
+
+ case Error():
+ return {"type": "error", "value": cls._encode(value.metadata)}
+
+ case _ if value is SKIPPED:
+ return {"type": "skipped", "value": None}
+
+ case tuple():
+ return {"type": "tuple", "value": [cls._encode(v) for v in value]}
+
+ case list():
+ return {"type": "list", "value": [cls._encode(v) for v in value]}
+
+ case dict() if all(isinstance(k, str) for k in value):
+ return {
+ "type": "dict",
+ "value": [[k, cls._encode(v)] for k, v in value.items()],
+ }
+
+ case _:
+ raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
@classmethod
def _decode(cls, value: Any) -> Any:
if value is None or type(value) in {str, int, float, bool}:
return value
+
if not isinstance(value, dict) or set(value) != {"type", "value"}:
raise ValueError("Invalid Huey JSON data")
- kind, payload = value["type"], value["value"]
- if kind == "uuid" and isinstance(payload, str):
- return uuid.UUID(payload)
- if kind == "datetime" and isinstance(payload, str):
- return datetime.fromisoformat(payload)
- if kind == "message" and isinstance(payload, list):
- if len(payload) != len(Message._fields):
- raise ValueError("Invalid Huey JSON message field count")
- return Message(*(cls._decode(v) for v in payload))
- if kind == "error":
- metadata = cls._decode(payload)
- if not isinstance(metadata, dict):
- raise ValueError("Invalid Huey JSON error metadata")
- return Error(metadata)
- if kind == "skipped" and payload is None:
- return SKIPPED
- if kind in {"tuple", "list"} and isinstance(payload, list):
- items = [cls._decode(v) for v in payload]
- return tuple(items) if kind == "tuple" else items
- if kind == "dict" and isinstance(payload, list):
- result: dict[str, Any] = {}
- for pair in payload:
- if (
- not isinstance(pair, list)
- or len(pair) != 2
- or type(pair[0]) is not str
- ):
- raise ValueError("Invalid Huey JSON dictionary entry")
- if pair[0] in result:
- raise ValueError("Duplicate Huey JSON dictionary key")
- result[pair[0]] = cls._decode(pair[1])
- return result
- raise ValueError("Unsupported Huey JSON type")
+ match value["type"], value["value"]:
+ case "uuid", str() as payload:
+ return uuid.UUID(payload)
+
+ case "datetime", str() as payload:
+ return datetime.fromisoformat(payload)
+
+ case "message", list() as payload:
+ if len(payload) != len(Message._fields):
+ raise ValueError("Invalid Huey JSON message field count")
+ return Message(*(cls._decode(v) for v in payload))
+
+ case "error", payload:
+ metadata = cls._decode(payload)
+ if not isinstance(metadata, dict):
+ raise TypeError("Invalid Huey JSON error metadata")
+ return Error(metadata)
+
+ case "skipped", None:
+ return SKIPPED
+
+ case ("tuple" | "list") as kind, list() as payload:
+ items = [cls._decode(v) for v in payload]
+ return tuple(items) if kind == "tuple" else items
+
+ case "dict", list() as payload:
+ result: dict[str, Any] = {}
+
+ for pair in payload:
+ match pair:
+ case list([str() as key, encoded]):
+ if key in result:
+ raise ValueError("Duplicate Huey JSON dictionary key")
+ result[key] = cls._decode(encoded)
+
+ case _:
+ raise ValueError("Invalid Huey JSON dictionary entry")
+
+ return result
+
+ case _:
+ raise ValueError("Unsupported Huey JSON type")
diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py
new file mode 100644
index 0000000..4e5a5ca
--- /dev/null
+++ b/webmentions_ssg/tasks/mentions.py
@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Pure HTML and text matching for received Webmention targets."""
+
+from urllib.parse import urljoin
+
+import rfc3987
+from bs4 import BeautifulSoup
+
+IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
+TRAILING_IRI_PUNCTUATION = ".,;:!?"
+
+HTML_URL_ATTRIBUTES = {
+ "href": {"a", "area", "link"},
+ "src": {
+ "audio",
+ "embed",
+ "iframe",
+ "img",
+ 'input[type="image" i]',
+ "script",
+ "audio source",
+ "video source",
+ "track",
+ "video",
+ },
+ "cite": {"blockquote", "del", "ins", "q"},
+}
+
+
+def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
+ """
+ Check whether an HTML document mentions the target URL.
+
+ URL references are resolved against the document's ``base`` element when
+ present, or against the source URL otherwise.
+
+ :param body: HTML document body.
+ :param source_url: URL from which the document was retrieved.
+ :param target_url: Exact URL to look for.
+ :return: Whether a supported HTML URL attribute references the target URL.
+ """
+
+ document = BeautifulSoup(body, "html.parser")
+ base_url = source_url
+
+ if (base_element := document.select_one("base[href]")) is not None and isinstance(
+ base_href := base_element.get("href"), str
+ ):
+ base_url = urljoin(source_url, base_href.strip())
+
+ for attribute, selectors in HTML_URL_ATTRIBUTES.items():
+ selector = ", ".join(
+ [f"{selector_string}[{attribute}]" for selector_string in selectors]
+ )
+
+ for element in document.select(selector):
+ if not isinstance(reference := element.get(attribute), str):
+ continue
+
+ if urljoin(base_url, reference.strip()) == target_url:
+ return True
+
+ return False
+
+
+def text_mentions_target(body: str, target_url: str) -> bool:
+ """
+ Check whether plain text contains the target IRI, ignoring trailing prose
+ punctuation but not extra URL path or query characters.
+
+ :param body: Plain-text document body.
+ :param target_url: Exact IRI to look for.
+ :return: Whether the target IRI occurs in the document.
+ """
+ for match in IRI_PATTERN.finditer(body):
+ candidate = match.group()
+
+ # A URL in prose can be followed by punctuation that the IRI pattern
+ # consumes as part of its match. Only remove trailing punctuation,
+ # leaving longer URLs (e.g. ``target/more``) unmatched.
+ if not candidate.startswith(target_url):
+ continue
+
+ while candidate != target_url:
+ if candidate[-1] in TRAILING_IRI_PUNCTUATION:
+ candidate = candidate[:-1]
+ elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
+ # Keep balanced parentheses that belong to the URL itself.
+ candidate = candidate[:-1]
+ else:
+ break
+
+ if candidate == target_url:
+ return True
+
+ return False
diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py
index 475fad1..889030c 100644
--- a/webmentions_ssg/tasks/notifications.py
+++ b/webmentions_ssg/tasks/notifications.py
@@ -10,10 +10,10 @@ import sqlalchemy as sa
from flask import current_app
from flask_mail import Message
-from .. import DATABASE as db
-from .. import HUEY as huey
-from .. import MAIL as mail
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..extensions import MAIL as mail
+from ..models import ReceivedStatus, ReceivedWebmention
@huey.lock_task("send-webmention-digest")
@@ -27,7 +27,7 @@ def send_webmention_digest() -> None:
cutoff = datetime.now(UTC)
conditions = (
- ReceivedWebmention.status == "verified",
+ ReceivedWebmention.status == ReceivedStatus.VERIFIED,
ReceivedWebmention.notification_sent_at.is_(None),
ReceivedWebmention.updated_at <= cutoff,
)
@@ -53,7 +53,6 @@ def send_webmention_digest() -> None:
),
)
- # If SMTP fails, the records remain pending for the next digest.
mail.send(message)
db.session.execute(
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index acd66a1..bed923c 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -2,27 +2,27 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Fetch and verify incoming Webmentions against their source pages."""
+"""Fetch and verify incoming Webmentions with transactional state changes."""
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
-import rfc3987
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..models import ReceivedWebmention
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError
-
-
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..models import ReceivedStatus, ReceivedWebmention
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_temporary_status,
+ read_limited_body,
+)
+from .mentions import html_mentions_target, text_mentions_target
class VerificationError(Exception):
@@ -37,95 +37,7 @@ class TemporaryFetchError(Exception):
"""Fetching the source may succeed when retried later."""
-IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
-TRAILING_IRI_PUNCTUATION = ".,;:!?"
-
-
-HTML_URL_ATTRIBUTES = {
- "href": {"a", "area", "link"},
- "src": {
- "audio",
- "embed",
- "iframe",
- "img",
- 'input[type="image" i]',
- "script",
- "audio source",
- "video source",
- "track",
- "video",
- },
- "cite": {"blockquote", "del", "ins", "q"},
-}
-
-
-def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """
- Check whether an HTML document mentions the target URL.
-
- URL references are resolved against the document's ``base`` element when
- present, or against the source URL otherwise.
-
- :param body: HTML document body.
- :param source_url: URL from which the document was retrieved.
- :param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
- """
-
- document = BeautifulSoup(body, "html.parser")
- base_url = source_url
-
- if (base_element := document.select_one("base[href]")) is not None and isinstance(
- base_href := base_element.get("href"), str
- ):
- base_url = urljoin(source_url, base_href.strip())
-
- for attribute, selectors in HTML_URL_ATTRIBUTES.items():
- selector = ", ".join(
- [f"{selector_string}[{attribute}]" for selector_string in selectors]
- )
-
- for element in document.select(selector):
- if not isinstance(reference := element.get(attribute), str):
- continue
-
- if urljoin(base_url, reference.strip()) == target_url:
- return True
-
- return False
-
-
-def text_mentions_target(body: str, target_url: str) -> bool:
- """
- Check whether plain text contains the target IRI, ignoring trailing prose
- punctuation but not extra URL path or query characters.
-
- :param body: Plain-text document body.
- :param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
- """
- for match in IRI_PATTERN.finditer(body):
- candidate = match.group()
-
- # A URL in prose can be followed by punctuation that the IRI pattern
- # consumes as part of its match. Only remove trailing punctuation,
- # leaving longer URLs (e.g. ``target/more``) unmatched.
- if not candidate.startswith(target_url):
- continue
-
- while candidate != target_url:
- if candidate[-1] in TRAILING_IRI_PUNCTUATION:
- candidate = candidate[:-1]
- elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
- # Keep balanced parentheses that belong to the URL itself.
- candidate = candidate[:-1]
- else:
- break
-
- if candidate == target_url:
- return True
-
- return False
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
@@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
try:
- return _fetch_source(source_url)
+ with (
+ build_client(
+ f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if is_temporary_status(status):
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
+
+ try:
+ return response, read_limited_body(
+ response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
+ )
+ except ValueError as exc:
+ raise VerificationError(str(exc)) from exc
+
except UnsafeAddressError as exc:
raise VerificationError("Source resolves to a non-public address") from exc
except AddressResolutionError as exc:
raise TemporaryFetchError("Source hostname could not be resolved") from exc
-def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- with (
- httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client,
- client.stream("GET", source_url) as response,
- ):
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
-
- max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
-
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_source_bytes:
- raise VerificationError("Source document is too large")
-
- return response, bytes(body)
-
-
def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
Fetch a source and check whether it mentions the target URL.
@@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- # Failed tasks may be retried by Huey. Completed and already-running tasks
- # should not start another concurrent verification.
- if webmention.status not in {"received", "failed"}:
+ if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
return
- # A reaper-exhausted row must not be revived by a late Huey retry.
+
if (
- webmention.status == "failed"
+ webmention.status == ReceivedStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return
@@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
ReceivedWebmention.status == previous_status,
ReceivedWebmention.updated_at == previous_updated_at,
)
- .values(status="verifying", failure_reason=None, updated_at=started_at)
+ .values(
+ status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at
+ )
)
+
if claimed.rowcount != 1:
db.session.rollback()
return
- db.session.commit()
- def finish(status: str, failure_reason: str | None) -> None:
- # A timeout or a newer verification may have superseded this attempt.
- # Do not let a late worker overwrite its state.
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == "verifying",
- ReceivedWebmention.updated_at == started_at,
- )
- .values(
- status=status,
- failure_reason=failure_reason,
- updated_at=datetime.now(UTC),
- )
- )
- db.session.commit()
+ db.session.commit()
+ status = None
+ failure_reason = None
try:
mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- finish("deleted", str(exc))
+ status = ReceivedStatus.DELETED
+ failure_reason = str(exc)
except VerificationError as exc:
- finish("failed", str(exc))
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc)
except (TemporaryFetchError, httpx.RequestError) as exc:
- finish("failed", str(exc) or "Source could not be fetched")
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or "Source could not be fetched"
raise
except Exception as exc:
# InvalidURL is not a RequestError; other unexpected source errors must
@@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", webmention_uuid
)
- finish("failed", str(exc) or type(exc).__name__)
+
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or type(exc).__name__
else:
if mentions_target:
- finish("verified", None)
+ status = ReceivedStatus.VERIFIED
else:
- finish("deleted", "Source does not mention target")
+ status = ReceivedStatus.DELETED
+ failure_reason = "Source does not mention target"
+ finally:
+ if status is not None:
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
def recover_stale_webmentions() -> int:
@@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int:
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+
stale = db.session.execute(
sa.select(
ReceivedWebmention.uuid,
@@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.updated_at,
ReceivedWebmention.recovery_attempts,
).where(
- ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ ),
ReceivedWebmention.updated_at <= cutoff,
)
).all()
@@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int:
recovered = 0
exhausted = 0
max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+
for identifier, status, updated_at, recovery_attempts in stale:
at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
@@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.recovery_attempts == recovery_attempts,
)
.values(
- status="failed" if at_limit else "received",
- failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
- updated_at=now,
- recovery_attempts=(
- recovery_attempts if at_limit else recovery_attempts + 1
- ),
+ **(
+ {
+ "status": ReceivedStatus.FAILED,
+ "failure_reason": RECOVERY_EXHAUSTED_REASON,
+ "updated_at": now,
+ "recovery_attempts": recovery_attempts,
+ }
+ if at_limit
+ else ReceivedWebmention.verification_reset_values(
+ at=now, recoveries=recovery_attempts + 1
+ )
+ )
)
)
+
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+
if at_limit:
exhausted += 1
continue
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 727b186..bf79512 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -11,8 +11,8 @@ import sqlalchemy as sa
from flask import current_app
from sqlalchemy.engine import CursorResult
-from .. import DATABASE as db
-from ..models import ReceivedWebmention
+from ..extensions import DATABASE as db
+from ..models import ReceivedStatus, ReceivedWebmention
def purge_old_received_webmentions() -> int:
@@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int:
cutoff = datetime.now(UTC) - timedelta(
days=current_app.config["RECEIVED_RETENTION_DAYS"]
)
+
result = cast(
CursorResult[Any],
db.session.execute(
sa.delete(ReceivedWebmention).where(
- ReceivedWebmention.status.in_(("failed", "deleted")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ ),
ReceivedWebmention.updated_at < cutoff,
)
),
)
db.session.commit()
+
removed = result.rowcount
if removed:
current_app.logger.info("Purged %d old failed/deleted Webmentions", removed)
+
return removed
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 666eba7..b2d6def 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -2,404 +2,22 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Scan generated site pages to track outgoing Webmention targets."""
+"""Reconcile scanned static-site sources with the database."""
-import json
-from collections.abc import Iterator
-from dataclasses import dataclass
from datetime import UTC, datetime
-from fnmatch import fnmatchcase
-from itertools import chain
from pathlib import Path
-from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
-import mf2py
import sqlalchemy as sa
-import xxhash
-from bs4 import BeautifulSoup, NavigableString, Tag
-from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..config import normalize_ignored_hostnames
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, Source
-from ..url_security import is_http_url
+from ..net import is_http_url
from .sender import send_webmention
-
-REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-
-CONTENT_BLOCK_TAGS = frozenset(
- {
- "address",
- "article",
- "aside",
- "blockquote",
- "br",
- "dd",
- "div",
- "dl",
- "dt",
- "figcaption",
- "figure",
- "footer",
- "h1",
- "h2",
- "h3",
- "h4",
- "h5",
- "h6",
- "header",
- "hr",
- "li",
- "main",
- "ol",
- "p",
- "pre",
- "section",
- "table",
- "td",
- "th",
- "tr",
- "ul",
- }
-)
-
-
-class SourceScanError(Exception):
- """A source document cannot safely be processed."""
-
-
-@dataclass(frozen=True)
-class ScannedSource:
- path: str
- url: str
- content_hash: str
- targets: frozenset[str]
-
-
-def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """
- Derive the public source URL from a relative filesystem path.
-
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
- """
- directory = relative_path.parent.as_posix()
-
- return urljoin(base_url, f"{quote(directory, safe='/')}/")
-
-
-def canonical_url(
- document: BeautifulSoup, *, relative_path: Path, base_url: str | None
-) -> str | None:
- """
- Return the canonical URL declared by a source document.
-
- Relative canonical URLs are resolved against the configured source base URL.
-
- :param document: Parsed HTML source document.
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
- """
- if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
- href = link.get("href")
-
- if not isinstance(href, str) or not (href := href.strip()):
- raise SourceScanError("Document contains an empty canonical URL")
-
- if is_http_url(href):
- return href
-
- if base_url is None:
- raise SourceScanError(
- "Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
- )
-
- resolved = urljoin(source_url_for_path(relative_path, base_url), href)
-
- if not is_http_url(resolved):
- raise SourceScanError(
- f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
- )
-
- return resolved
-
-
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """
- Yield URL values from a microformats property.
-
- :param entry: Parsed microformats entry.
- :param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
- """
- properties = entry.get("properties")
-
- if not isinstance(properties, dict):
- return
-
- values = properties.get(property_name)
-
- if not isinstance(values, list):
- return
-
- yield from (value for value in values if isinstance(value, str))
-
-
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """
- Parse and validate an h-entry with mf2py.
-
- :param element: HTML element containing the h-entry.
- :param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
- :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
- """
- parsed = mf2py.parse(doc=str(element), url=base_url)
-
- if not isinstance(parsed, dict):
- raise SourceScanError("Microformats parser did not return a document object")
-
- items = parsed.get("items")
-
- if not isinstance(items, list) or len(items) != 1:
- raise SourceScanError("Expected exactly one parsed h-entry")
-
- entry = items[0]
-
- if not isinstance(entry, dict):
- raise SourceScanError("Parsed h-entry is not an object")
-
- types = entry.get("type")
-
- if not isinstance(types, list) or "h-entry" not in types:
- raise SourceScanError("Parsed microformats item is not an h-entry")
-
- return entry
-
-
-def primary_entry(
- document: BeautifulSoup, source_url: str
-) -> tuple[Tag, dict[str, Any]]:
- """
- Return and validate the primary h-entry of a source document.
-
- The document must contain exactly one h-entry whose ``u-url`` matches the
- source URL.
-
- :param document: Parsed HTML source document.
- :param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
- """
- entries = document.find_all(class_="h-entry")
-
- if len(entries) != 1:
- raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
-
- entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
-
- urls = tuple(property_urls(mf2_entry, "url"))
-
- if source_url not in urls:
- raise SourceScanError(
- f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
- )
-
- return entry_element, mf2_entry
-
-
-def content_element(entry: Tag) -> Tag:
- """
- Return the e-content element of an h-entry.
-
- :param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
- """
- contents = entry.find_all(class_="e-content")
-
- if len(contents) != 1:
- raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
-
- return contents[0]
-
-
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """
- Resolve and validate a possible Webmention target URL.
-
- Empty values, non-HTTP URLs, and URLs referring to the source itself are
- discarded.
-
- :param value: URL reference to normalize.
- :param base_url: Base URL against which relative references are resolved.
- :param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
- """
- value = value.strip()
-
- if not value:
- return None
-
- target = urljoin(base_url, value)
-
- if not is_http_url(target):
- return None
-
- if urldefrag(target)[0] == urldefrag(source_url)[0]:
- return None
-
- return target
-
-
-def iter_targets(
- content: Tag,
- mf2_entry: dict[str, Any],
- *,
- base_url: str,
- source_url: str,
- ignored_hostnames: tuple[str, ...] = (),
-) -> Iterator[str]:
- """
- Yield outgoing Webmention targets from an h-entry.
-
- Targets are collected from links in the entry content and supported
- microformats reaction properties. Invalid, self-referencing, and ignored
- targets are excluded.
-
- :param content: e-content element of the h-entry.
- :param mf2_entry: Parsed microformats representation of the h-entry.
- :param base_url: Base URL against which relative targets are resolved.
- :param source_url: URL of the source document.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
- """
- hrefs = (
- href
- for element in content.find_all("a", href=True)
- if isinstance(href := element.get("href"), str)
- )
-
- reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
- )
-
- for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
- if target is not None:
- hostname = urlsplit(target).hostname or ""
- if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
- yield target
-
-
-def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
-
- def extract(node: PageElement) -> str:
- if isinstance(node, NavigableString):
- return str(node)
- if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
- return ""
-
- # A relative-time label may change on every site build even though its
- # machine-readable date stays the same.
- if node.name == "time" and isinstance(value := node.get("datetime"), str):
- return value.strip()
-
- # Image-only posts also have meaningful content, despite having no text.
- if node.name == "img":
- return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
-
- text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
-
- return " ".join(extract(content).split())
-
-
-def scan_source_file(
- path: Path,
- *,
- root: Path,
- base_url: str | None,
- ignored_hostnames: tuple[str, ...] = (),
-) -> ScannedSource:
- """
- Scan one generated source document for outgoing Webmentions.
-
- :param path: Path of the generated HTML document.
- :param root: Root directory containing generated source documents.
- :param base_url: Configured source base URL, if available.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
- :raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
- :raises OSError: If the source document cannot be read.
- """
- # Use the path as published for its URL, but read only files whose
- # resolved target stays inside the configured source directory. Resolving
- # before reading also protects against symlinked index.html files.
- resolved_root = root.resolve()
- resolved_path = path.resolve()
- if not resolved_path.is_relative_to(resolved_root):
- raise SourceScanError(f"Source document escapes source directory: {path}")
-
- relative_path = path.relative_to(root)
- document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
-
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
-
- if source_url is None:
- if base_url is None:
- raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
- )
-
- source_url = source_url_for_path(relative_path, base_url)
-
- entry_element, mf2_entry = primary_entry(document, source_url)
-
- content = content_element(entry_element)
-
- targets = frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- )
-
- # Fingerprint the meaningful post text and its outgoing targets rather than
- # the full h-entry markup. Relative timestamps, classes, and other volatile
- # metadata outside e-content must not cause repeat Webmentions.
- stable_content = normalized_content(content)
- fingerprint = json.dumps(
- {"content": stable_content, "targets": sorted(targets)},
- ensure_ascii=False,
- separators=(",", ":"),
- )
-
- return ScannedSource(
- path=relative_path.as_posix(),
- url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
- targets=targets,
- )
+from .source_parser import ScannedSource, SourceScanError, scan_source_file
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
@@ -466,10 +84,9 @@ def update_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.active = webmention.target in scanned.targets
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(
+ source.revision, webmention.target in scanned.targets
+ )
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -523,10 +140,6 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = normalize_ignored_hostnames(
- current_app.config["IGNORED_HOSTNAMES"]
- )
-
sources_by_path = {
source.path: source
for source in db.session.scalars(
@@ -555,7 +168,7 @@ def scan_sources_task() -> None:
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=ignored_hostnames,
+ ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -592,10 +205,6 @@ def scan_sources_task() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- # A static-site generator can temporarily empty its output directory while
- # rebuilding. With no successfully scanned pages, absence is not evidence
- # of deletion; do not generate removals or send pending Webmentions that
- # might be verified while the site is unavailable.
if scanned_count == 0:
current_app.logger.warning(
"No source documents successfully scanned; "
@@ -612,10 +221,7 @@ def scan_sources_task() -> None:
source.deleted_at = scan_time
for webmention in source.sent_webmentions:
- webmention.active = False
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 9330247..3c4c717 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -2,272 +2,33 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Discover endpoints and deliver outgoing Webmentions."""
+"""Claim, send, and persist outgoing Webmention attempts."""
-import re
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from huey.exceptions import TaskLockedException
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
-
-LINK_SPLIT = re.compile(r",\s*(?=<)")
-
-DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
-
-
-class SenderError(Exception):
- """A Webmention cannot be sent."""
-
-
-class TemporarySenderError(SenderError):
- """A Webmention send failed for a potentially temporary reason."""
-
-
-class PermanentSenderError(SenderError):
- """A Webmention send failed permanently for this source revision."""
-
-
-def temporary_http_status(status: int) -> bool:
- """
- Return whether an HTTP status indicates a temporary failure.
-
- :param status: HTTP response status code.
- :return: Whether the request should be retried.
- """
- return status in {408, 425, 429} or 500 <= status <= 599
-
-
-def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """
- Resolve and validate a discovered Webmention endpoint.
-
- :param response: Response from which the endpoint was discovered.
- :param href: Endpoint reference to resolve.
- :return: Absolute Webmention endpoint URL.
- :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
- HTTPS URL.
- """
- try:
- endpoint = urljoin(str(response.url), href.strip())
- except ValueError as exc:
- raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc
-
- if not is_http_url(endpoint):
- raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}")
-
- return endpoint
-
-
-def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """
- Parse a Link header value into its target and relations.
-
- :param value: Link header value to parse.
- :return: Link target and relation names, or ``None`` if the value is invalid.
- """
- value = value.strip()
-
- if not value.startswith("<"):
- return None
-
- href, separator, parameters = value[1:].partition(">")
-
- if not separator:
- return None
-
- relations: set[str] = set()
-
- for parameter in parameters.split(";"):
- name, separator, value = parameter.partition("=")
-
- if separator and name.strip().lower() == "rel":
- relations.update(
- relation.lower() for relation in value.strip().strip("\"'").split()
- )
-
- return href, relations
-
-
-def endpoint_from_headers(response: httpx.Response) -> str | None:
- """
- Find a Webmention endpoint in the response Link headers.
-
- :param response: HTTP response whose headers should be inspected.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- for header in response.headers.get_list("Link"):
- for value in LINK_SPLIT.split(header):
- if (link := parse_link_value(value)) is None:
- continue
-
- href, relations = link
-
- if "webmention" in relations:
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """
- Find a Webmention endpoint in an HTML document.
-
- ``link`` and ``a`` elements are inspected in document order.
-
- :param response: HTTP response from which the document was retrieved.
- :param body: HTML document body.
- :return: First valid advertised Webmention endpoint, or ``None`` if none
- is found. Invalid candidates are ignored.
- """
- document = BeautifulSoup(body, "html5lib")
-
- for element in document.find_all(["link", "a"], href=True):
- relations = element.get("rel")
-
- if isinstance(relations, str):
- relations = relations.split()
-
- if not relations:
- continue
-
- if not any(
- isinstance(relation, str) and relation.lower() == "webmention"
- for relation in relations
- ):
- continue
-
- href = element.get("href")
-
- if not isinstance(href, str):
- continue
-
- try:
- return resolve_endpoint(response, href)
- except PermanentSenderError:
- continue
-
- return None
-
-
-def read_target_body(response: httpx.Response) -> bytes:
- """
- Read a target document up to the configured size limit.
-
- :param response: Streaming HTTP response to read.
- :return: Response body.
- :raises PermanentSenderError: If the document exceeds the configured maximum
- size.
- """
- max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_bytes:
- raise PermanentSenderError("Target document is too large")
- except ValueError:
- pass
-
- body = bytearray()
-
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_bytes:
- raise PermanentSenderError("Target document is too large")
-
- return bytes(body)
-
-
-def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """
- Discover the Webmention endpoint advertised by a target URL.
-
- Endpoint discovery first checks the response to a HEAD request and then falls
- back to a GET request, inspecting both HTTP Link headers and supported HTML
- documents.
-
- :param client: HTTP client to use for discovery requests.
- :param target: Target URL whose Webmention endpoint should be discovered.
- :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
- :raises TemporarySenderError: If the target returns a temporary HTTP failure.
- :raises PermanentSenderError: If GET returns a permanent HTTP failure.
- """
- try:
- head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
- # HEAD is only an optimization. Servers and intermediaries may reject
- # or drop it even when fetching the document with GET would succeed.
- pass
- else:
- if endpoint := endpoint_from_headers(head_response):
- return endpoint
-
- with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response:
- if endpoint := endpoint_from_headers(response):
- return endpoint
-
- if temporary_http_status(response.status_code):
- raise TemporarySenderError(f"Target returned HTTP {response.status_code}")
-
- if not response.is_success:
- raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
-
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
-
- if media_type not in {"text/html", "application/xhtml+xml"}:
- return None
-
- return endpoint_from_html(response, read_target_body(response))
-
-
-def post_webmention(
- client: httpx.Client, *, endpoint: str, source: str, target: str
-) -> tuple[int, str | None]:
- """
- Send a Webmention to a discovered endpoint.
-
- :param client: HTTP client to use for the request.
- :param endpoint: Webmention endpoint URL.
- :param source: Source URL of the Webmention.
- :param target: Target URL of the Webmention.
- :return: HTTP response status code and optional status URL.
- """
- with client.stream(
- "POST", endpoint, data={"source": source, "target": target}
- ) as response:
- status_url = None
-
- if response.status_code == 201 and (
- location := response.headers.get("Location")
- ):
- try:
- candidate = urljoin(str(response.url), location)
- except ValueError:
- # A malformed Location must not fail an otherwise successful send.
- pass
- else:
- if is_http_url(candidate):
- status_url = candidate
-
- return (response.status_code, status_url)
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_http_url,
+ is_temporary_status,
+)
+from .discovery import (
+ PermanentSenderError,
+ TemporarySenderError,
+ discover_webmention_endpoint,
+ post_webmention,
+)
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
now = datetime.now(UTC)
deadline = webmention.next_retry_at
- # SQLite returns timezone-naive datetimes even for timezone-aware columns.
if deadline is not None:
aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
if aware_deadline > now:
@@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
endpoint: str | None = None
response_status: int | None = None
+ status_url: str | None = None
+ reason: str | None = None
+ terminal = True
+ temporary_failure = False
try:
if not is_http_url(target):
raise PermanentSenderError(f"Invalid target URL: {target!r}")
- with httpx.Client(
- headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")},
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client:
+ with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client:
endpoint = discover_webmention_endpoint(client, target)
if endpoint is None:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.UNSUPPORTED
- webmention.failure_reason = "No Webmention endpoint discovered"
- webmention.endpoint = None
- webmention.response_status = None
-
- db.session.commit()
- return
-
- (response_status, status_url) = post_webmention(
- client, endpoint=endpoint, source=source, target=target
- )
-
- if not (200 <= response_status <= 299):
- message = f"Webmention endpoint returned HTTP {response_status}"
-
- if temporary_http_status(response_status):
- raise TemporarySenderError(message)
-
- raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.UNSUPPORTED
+ reason = "No Webmention endpoint discovered"
+ else:
+ response_status, status_url = post_webmention(
+ client, endpoint=endpoint, source=source, target=target
+ )
+ if not 200 <= response_status <= 299:
+ message = f"Webmention endpoint returned HTTP {response_status}"
+ if is_temporary_status(response_status):
+ raise TemporarySenderError(message)
+ raise PermanentSenderError(message)
+ outcome = SentWebmentionStatus.SENT
except (
PermanentSenderError,
@@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
ValueError,
UnicodeError,
) as exc:
- # Malformed URLs and local request/protocol errors cannot be repaired
- # by retrying this source revision. InvalidURL is not a RequestError;
- # protocol errors are, so handle them before the retryable branch.
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or type(exc).__name__
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
- return
-
+ # Invalid URLs and local protocol errors cannot improve with retries.
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or type(exc).__name__
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
- if not attempt_is_current(webmention, revision):
- return
-
- webmention.status = SentWebmentionStatus.FAILED
- webmention.failure_reason = str(exc) or "Webmention request failed"
- if attempt >= max_attempts:
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- current_app.logger.warning(
- "Webmention %s exhausted %d send attempts: %s",
- webmention_uuid,
- attempt,
- webmention.failure_reason,
- )
- webmention.endpoint = endpoint
- webmention.response_status = response_status
-
- db.session.commit()
-
- # No Huey retries: only the scanner may requeue after next_retry_at.
- return
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or "Webmention request failed"
+ terminal = attempt >= max_attempts
+ temporary_failure = True
+ # A scanner may have reopened or superseded the revision during I/O.
+ # Refresh only once, and persist one outcome atomically with its metadata.
if not attempt_is_current(webmention, revision):
return
- webmention.processed_revision = revision
- webmention.next_retry_at = None
- webmention.sent_revision = revision
-
- webmention.status = SentWebmentionStatus.SENT
- webmention.failure_reason = None
-
+ webmention.status = outcome
+ webmention.failure_reason = reason
webmention.endpoint = endpoint
webmention.response_status = response_status
- webmention.status_url = (
- status_url if status_url is not None and is_http_url(status_url) else None
- )
- webmention.last_sent_at = datetime.now(UTC)
+ if terminal:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+
+ if outcome == SentWebmentionStatus.SENT:
+ webmention.sent_revision = revision
+ webmention.status_url = (
+ status_url if status_url is not None and is_http_url(status_url) else None
+ )
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
+
+ if temporary_failure and not terminal:
+ # Huey does not retry: scanner requeues after next_retry_at.
+ current_app.logger.info(
+ "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt
+ )
+ elif temporary_failure and terminal:
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ reason,
+ )
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
new file mode 100644
index 0000000..cb15ab6
--- /dev/null
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -0,0 +1,390 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Extract canonical source URLs, content fingerprints, and outgoing links."""
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from itertools import chain
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urldefrag, urljoin, urlsplit
+
+import mf2py
+import xxhash
+from bs4 import BeautifulSoup, Tag
+from bs4.element import NavigableString, PageElement
+
+from ..net import is_http_url
+
+REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
+
+class SourceScanError(Exception):
+ """A source document cannot safely be processed."""
+
+
+@dataclass(frozen=True)
+class ScannedSource:
+ path: str
+ url: str
+ content_hash: str
+ targets: frozenset[str]
+
+
+def source_url_for_path(relative_path: Path, base_url: str) -> str:
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
+ directory = relative_path.parent.as_posix()
+
+ return urljoin(base_url, f"{quote(directory, safe='/')}/")
+
+
+def canonical_url(
+ document: BeautifulSoup, *, relative_path: Path, base_url: str | None
+) -> str | None:
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
+ if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
+ href = link.get("href")
+
+ if not isinstance(href, str) or not (href := href.strip()):
+ raise SourceScanError("Document contains an empty canonical URL")
+
+ if is_http_url(href):
+ return href
+
+ if base_url is None:
+ raise SourceScanError(
+ "Document contains a relative canonical URL, but "
+ "SOURCE_BASE_URL is not configured"
+ )
+
+ resolved = urljoin(source_url_for_path(relative_path, base_url), href)
+
+ if not is_http_url(resolved):
+ raise SourceScanError(
+ f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
+ )
+
+ return resolved
+
+
+def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
+ properties = entry.get("properties")
+
+ if not isinstance(properties, dict):
+ return
+
+ values = properties.get(property_name)
+
+ if not isinstance(values, list):
+ return
+
+ yield from (value for value in values if isinstance(value, str))
+
+
+def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
+ parsed = mf2py.parse(doc=str(element), url=base_url)
+
+ if not isinstance(parsed, dict):
+ raise SourceScanError("Microformats parser did not return a document object")
+
+ items = parsed.get("items")
+
+ if not isinstance(items, list) or len(items) != 1:
+ raise SourceScanError("Expected exactly one parsed h-entry")
+
+ entry = items[0]
+
+ if not isinstance(entry, dict):
+ raise SourceScanError("Parsed h-entry is not an object")
+
+ types = entry.get("type")
+
+ if not isinstance(types, list) or "h-entry" not in types:
+ raise SourceScanError("Parsed microformats item is not an h-entry")
+
+ return entry
+
+
+def primary_entry(
+ document: BeautifulSoup, source_url: str
+) -> tuple[Tag, dict[str, Any]]:
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
+ entries = document.find_all(class_="h-entry")
+
+ if len(entries) != 1:
+ raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
+
+ entry_element = entries[0]
+ mf2_entry = parse_entry(entry_element, source_url)
+
+ urls = tuple(property_urls(mf2_entry, "url"))
+
+ if source_url not in urls:
+ raise SourceScanError(
+ f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
+ )
+
+ return entry_element, mf2_entry
+
+
+def content_element(entry: Tag) -> Tag:
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
+ contents = entry.find_all(class_="e-content")
+
+ if len(contents) != 1:
+ raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
+
+ return contents[0]
+
+
+def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
+ value = value.strip()
+
+ if not value:
+ return None
+
+ target = urljoin(base_url, value)
+
+ if not is_http_url(target):
+ return None
+
+ if urldefrag(target)[0] == urldefrag(source_url)[0]:
+ return None
+
+ return target
+
+
+def iter_targets(
+ content: Tag,
+ mf2_entry: dict[str, Any],
+ *,
+ base_url: str,
+ source_url: str,
+ ignored_hostnames: Iterable[str] = (),
+) -> Iterator[str]:
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
+ hrefs = (
+ href
+ for element in content.find_all("a", href=True)
+ if isinstance(href := element.get("href"), str)
+ )
+
+ reactions = chain.from_iterable(
+ property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ )
+
+ for value in chain(hrefs, reactions):
+ target = normalize_target(value, base_url=base_url, source_url=source_url)
+ if target is not None:
+ hostname = (urlsplit(target).hostname or "").rstrip(".")
+ if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
+ yield target
+
+
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
+def scan_source_file(
+ path: Path,
+ *,
+ root: Path,
+ base_url: str | None,
+ ignored_hostnames: Iterable[str] = (),
+) -> ScannedSource:
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
+
+ relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
+
+ source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+
+ if source_url is None:
+ if base_url is None:
+ raise SourceScanError(
+ "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ )
+
+ source_url = source_url_for_path(relative_path, base_url)
+
+ entry_element, mf2_entry = primary_entry(document, source_url)
+
+ content = content_element(entry_element)
+
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+ return ScannedSource(
+ path=relative_path.as_posix(),
+ url=source_url,
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
+ )
diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html
index 1ff3690..3dbb896 100644
--- a/webmentions_ssg/templates/received.html
+++ b/webmentions_ssg/templates/received.html
@@ -6,6 +6,33 @@ SPDX-License-Identifier: BSD-3-Clause
{% extends "base.html" %}
{% from "bootstrap5/pagination.html" import render_pagination %}
+{% set received_status_presentation = {
+ "received": {
+ "row_class": "table-info",
+ "badge_class": "text-bg-info",
+ "label": "Pending"
+ },
+ "verifying": {
+ "row_class": "table-warning",
+ "badge_class": "text-bg-warning",
+ "label": "Verifying"
+ },
+ "verified": {
+ "row_class": "table-success",
+ "badge_class": "text-bg-success",
+ "label": "Verified"
+ },
+ "failed": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Declined"
+ },
+ "deleted": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Declined"
+ }
+} %}
{% block content %}
<div class="d-flex justify-content-between align-items-center mb-3">
@@ -27,27 +54,9 @@ SPDX-License-Identifier: BSD-3-Clause
</thead>
<tbody>
{% for webmention in webmentions %}
- {% if webmention.status == "received" %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% set display_status = "Pending" %}
- {% elif webmention.status == "verifying" %}
- {% set row_class = "table-warning" %}
- {% set badge_class = "text-bg-warning" %}
- {% set display_status = "Verifying" %}
- {% elif webmention.status == "verified" %}
- {% set row_class = "table-success" %}
- {% set badge_class = "text-bg-success" %}
- {% set display_status = "Verified" %}
- {% else %}
- {% set row_class = "table-danger" %}
- {% set badge_class = "text-bg-danger" %}
- {% set display_status = "Declined" %}
- {% endif %}
- <tr class="{{ row_class }}">
- <td>
- <span class="badge {{ badge_class }}">{{ display_status }}</span>
- </td>
+ {% set style = received_status_presentation.get(webmention.status, {"row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined"}) %}
+ <tr class="{{ style.row_class }}">
+ <td><span class="badge {{ style.badge_class }}">{{ style.label }}</span></td>
<td class="text-break">
<a href="{{ webmention.source }}">{{ webmention.source }}</a>
</td>
diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html
index 689c651..974c7b3 100644
--- a/webmentions_ssg/templates/sent_source.html
+++ b/webmentions_ssg/templates/sent_source.html
@@ -6,6 +6,23 @@ SPDX-License-Identifier: BSD-3-Clause
{% extends "base.html" %}
{% from "bootstrap5/pagination.html" import render_pagination %}
+{% set sent_status_presentation = {
+ "sent": {
+ "row_class": "table-success",
+ "badge_class": "text-bg-success",
+ "label": "Sent"
+ },
+ "unsupported": {
+ "row_class": "table-warning",
+ "badge_class": "text-bg-warning",
+ "label": "Unsupported"
+ },
+ "failed": {
+ "row_class": "table-danger",
+ "badge_class": "text-bg-danger",
+ "label": "Failed"
+ }
+} %}
{% block content %}
<div class="mb-3">
@@ -85,32 +102,13 @@ SPDX-License-Identifier: BSD-3-Clause
</thead>
<tbody>
{% for webmention in webmentions %}
- {% if webmention.status is none %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% elif webmention.status.value == "sent" %}
- {% set row_class = "table-success" %}
- {% set badge_class = "text-bg-success" %}
- {% elif webmention.status.value == "unsupported" %}
- {% set row_class = "table-warning" %}
- {% set badge_class = "text-bg-warning" %}
- {% elif webmention.status.value == "failed" %}
- {% set row_class = "table-danger" %}
- {% set badge_class = "text-bg-danger" %}
- {% else %}
- {% set row_class = "table-info" %}
- {% set badge_class = "text-bg-info" %}
- {% endif %}
- <tr class="{{ row_class }}">
+ {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %}
+ <tr class="{{ style.row_class }}">
<td class="text-break">
<a href="{{ webmention.target }}">{{ webmention.target }}</a>
</td>
<td>
- {% if webmention.status is none %}
- <span class="badge {{ badge_class }}">Pending</span>
- {% else %}
- <span class="badge {{ badge_class }}">{{ webmention.status.value.title() }}</span>
- {% endif %}
+ <span class="badge {{ style.badge_class }}">{{ style.label }}</span>
{% if webmention.failure_reason is not none %}
<div class="small mt-2">{{ webmention.failure_reason }}</div>
{% endif %}
diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py
deleted file mode 100644
index 256d037..0000000
--- a/webmentions_ssg/url_digest.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Portable SHA-256 fingerprints for indexed Webmention URLs."""
-
-import hashlib
-import json
-
-
-def received_url_digest(source: str, target: str) -> str:
- """Fingerprint an unambiguous, ordered source/target pair."""
- encoded = json.dumps(
- [source, target], ensure_ascii=False, separators=(",", ":")
- ).encode("utf-8")
- return hashlib.sha256(encoded).hexdigest()
-
-
-def sent_target_digest(target: str) -> str:
- """Fingerprint an outgoing target (unique within each source ID)."""
- return hashlib.sha256(target.encode("utf-8")).hexdigest()
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
deleted file mode 100644
index c4324e0..0000000
--- a/webmentions_ssg/url_security.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Validate URL schemes and publicly routable network destinations."""
-
-import ipaddress
-from urllib.parse import urlsplit
-
-import dns.exception
-import dns.resolver
-
-
-class AddressResolutionError(Exception):
- """A hostname could not be resolved (potentially transient)."""
-
-
-class UnsafeAddressError(ValueError):
- """A hostname or IP literal identifies a non-public network address."""
-
-
-_NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
-_IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
-
-
-def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
- """Reject non-public addresses, including IPv4 hidden in IPv6 forms.
-
- ``is_global`` alone does not reliably exclude IPv6 transition addresses
- across Python versions or networks with NAT64/6to4 routing.
- """
- if isinstance(address, ipaddress.IPv6Address):
- # The well-known NAT64 prefix can translate an otherwise global IPv6
- # address into a private IPv4 destination. IPv4-compatible addresses
- # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
- if address in _NAT64_PREFIX or address in _IPV4_COMPATIBLE_PREFIX:
- return False
-
- # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
- # destination through ipaddress; validate that destination as well.
- for embedded in (address.sixtofour, address.ipv4_mapped):
- if embedded is not None and not embedded.is_global:
- return False
-
- return address.is_global
-
-
-def public_addresses(
- hostname: str, *, lifetime: float | None = None
-) -> tuple[str, ...]:
- """Resolve *once* and return only checked, globally routable IP addresses.
-
- A DNS name with a mixture of public and private answers is rejected in its
- entirety. IP literals are handled without DNS, including IPv6 literals.
- The returned strings must be used for the actual TCP connection; checking
- them and then connecting to ``hostname`` reintroduces DNS rebinding.
- """
- try:
- literal = ipaddress.ip_address(hostname)
- except ValueError:
- literal = None
-
- if literal is not None:
- addresses = (literal,)
- else:
- try:
- answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
- addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
- except (dns.exception.DNSException, ValueError) as exc:
- raise AddressResolutionError(
- f"Could not resolve hostname {hostname!r}"
- ) from exc
-
- if not addresses:
- raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
-
- if any(not is_public_address(address) for address in addresses):
- raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
-
- return tuple(str(address) for address in addresses)
-
-
-def is_public_url(url: str) -> bool:
- """Return whether the URL resolves entirely to public IP addresses.
-
- This is useful for diagnostics, but **not** sufficient to authorize a
- request: network connections must use the exact validated IP address.
- """
- hostname = urlsplit(url).hostname
-
- if hostname is None:
- raise ValueError("No hostname was specified")
-
- try:
- public_addresses(hostname)
- except UnsafeAddressError:
- return False
- return True
-
-
-def is_http_url(url: str) -> bool:
- """
- Check whether a URL is an absolute HTTP or HTTPS URL.
-
- :param url: URL to validate.
- :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
- """
- try:
- parsed = urlsplit(url)
- except ValueError:
- return False
-
- return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
-
-
-def is_safe_next_url(next_url: str) -> bool:
- """Allow redirects only to paths on this site."""
- # Check the original string too: urlsplit("///evil.example").path is
- # "/evil.example", even though browsers can treat the original as external.
- if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"):
- return False
-
- try:
- target = urlsplit(next_url)
- except ValueError:
- return False
-
- return (
- not target.scheme
- and not target.netloc
- and target.path.startswith("/")
- and not target.path.startswith("//")
- )
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
deleted file mode 100644
index 5b563f0..0000000
--- a/webmentions_ssg/views.py
+++ /dev/null
@@ -1,454 +0,0 @@
-# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-"""Provide routes for authentication and Webmention administration."""
-
-import hashlib
-import secrets
-import uuid
-from datetime import UTC, datetime, timedelta
-from typing import Any, cast
-
-import sqlalchemy as sa
-import sqlalchemy.exc as sa_exc
-from flask import (
- Blueprint,
- Response,
- abort,
- current_app,
- flash,
- redirect,
- render_template,
- request,
- url_for,
-)
-from flask.typing import ResponseReturnValue
-from flask_limiter.util import get_remote_address
-from flask_login import current_user, login_required, login_user, logout_user
-from sqlalchemy.engine import CursorResult
-from sqlalchemy.orm import selectinload
-
-from . import CSRF, LIMITER, forms
-from . import DATABASE as db
-from .models import ReceivedWebmention, SentWebmention, Source, User
-from .tasks.receiver import verify_webmention
-from .tasks.scanner import manual_scan_sources
-from .tasks.sender import send_webmention
-from .url_digest import received_url_digest
-from .url_security import is_safe_next_url
-
-root_page = Blueprint("root", __name__)
-
-DUMMY_USER = User(username=secrets.token_urlsafe())
-DUMMY_USER.password = secrets.token_urlsafe()
-
-
-def login_username_limit_key() -> str:
- """Group submitted usernames across IPs without storing raw account names."""
- username = request.form.get("username", "").strip().casefold()
- if not username:
- # A missing username must not put all clients into one global bucket.
- return f"missing-username:{get_remote_address()}"
- digest = hashlib.sha256(username.encode("utf-8")).hexdigest()
- return f"username:{digest}"
-
-
-@root_page.route("/")
-def index() -> ResponseReturnValue:
- """
- Redirect to the received Webmentions view.
-
- :return: Redirect response to the received Webmentions view.
- """
- return redirect(url_for("root.received"))
-
-
-@root_page.route("/login", methods=["GET", "POST"])
-@LIMITER.limit(
- lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"],
- key_func=login_username_limit_key,
- methods=["POST"],
- error_message="Too many login attempts. Please try again later.",
-)
-@LIMITER.limit(
- lambda: current_app.config["LOGIN_RATE_LIMIT"],
- methods=["POST"],
- error_message="Too many login attempts. Please try again later.",
-)
-def login() -> ResponseReturnValue:
- """
- Authenticate a user and start a login session.
-
- Authenticated users are redirected to the application index. After a
- successful login, the user is redirected to the requested local URL when
- provided.
-
- :return: Rendered login page or redirect response.
- """
- if current_user.is_authenticated:
- return redirect(url_for("root.index"))
-
- form = forms.LoginForm()
-
- if form.validate_on_submit():
- user = db.session.scalar(
- sa.select(User).where(User.username == form.username.data)
- )
-
- user_exists = user is not None
-
- if user is None:
- user = DUMMY_USER
-
- if (
- form.password.data is None
- or not user.check_password(form.password.data)
- or not user_exists
- ):
- flash("Invalid username or password", "danger")
- return redirect(url_for("root.login"))
- login_user(user)
-
- next_url = request.args.get("next")
- if not next_url or not is_safe_next_url(next_url):
- next_url = url_for("root.index")
- return redirect(next_url)
-
- return render_template("login.html", form=form)
-
-
-@root_page.route("/logout", methods=("POST",))
-@login_required
-def logout() -> ResponseReturnValue:
- """
- End the current user's login session.
-
- :return: Redirect response to the application index.
- """
- logout_user()
- return redirect(url_for("root.index"))
-
-
-@root_page.route("/received")
-@login_required
-def received() -> ResponseReturnValue:
- """
- Display received Webmentions.
-
- :return: Rendered page containing the paginated received Webmentions.
- """
- webmentions = db.paginate(
- sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()),
- per_page=25,
- )
-
- return render_template(
- "received.html",
- title="Received Webmentions",
- webmentions=webmentions,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/received/<uuid:identifier>/delete")
-@login_required
-def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Delete a received Webmention.
-
- :param identifier: Identifier of the Webmention to delete.
- :return: Redirect response to the received Webmentions view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- db.session.delete(webmention)
- db.session.commit()
-
- flash(f"Webmention {webmention.uuid} deleted.", "success")
-
- return redirect(
- url_for("root.received", page=request.args.get("page", 1, type=int))
- )
-
-
-@root_page.post("/received/<uuid:identifier>/reverify")
-@login_required
-def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Queue a received Webmention for reverification.
-
- The Webmention status is reset before a new verification task is queued.
-
- :param identifier: Identifier of the Webmention to reverify.
- :return: Redirect response to the received Webmentions view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- webmention.status = "received"
- webmention.failure_reason = None
- webmention.recovery_attempts = 0
-
- db.session.commit()
-
- verify_webmention(identifier)
-
- flash(f"Webmention {webmention.uuid} queued for reverification.", "success")
-
- return redirect(
- url_for("root.received", page=request.args.get("page", 1, type=int))
- )
-
-
-@root_page.route("/sent")
-@login_required
-def sent() -> ResponseReturnValue:
- """
- Display sources with sent Webmentions.
-
- :return: Rendered page containing the paginated source list.
- """
- sources = db.paginate(
- sa.select(Source)
- .options(selectinload(Source.sent_webmentions))
- .order_by(Source.last_seen_at.desc()),
- per_page=25,
- )
- return render_template(
- "sent.html",
- title="Sent Webmentions",
- sources=sources,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/sent/rescan")
-@login_required
-def rescan_sent_sources() -> ResponseReturnValue:
- """
- Queue a manual scan of sent Webmention sources.
-
- :return: Redirect response to the sent Webmentions view.
- """
- form = forms.AdminActionForm()
- if form.validate_on_submit():
- manual_scan_sources()
- flash("Source rescan queued.", "success")
- return redirect(url_for("root.sent"))
-
-
-@root_page.route("/sent/<uuid:source_identifier>")
-@login_required
-def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Display the sent Webmentions associated with a source.
-
- :param source_identifier: Identifier of the source to display.
- :return: Rendered source details page.
- """
- source = db.session.scalar(
- sa.select(Source).where(Source.uuid == source_identifier)
- )
-
- if source is None:
- return abort(404)
-
- webmentions = db.paginate(
- sa.select(SentWebmention)
- .where(SentWebmention.source_id == source.uuid)
- .order_by(SentWebmention.target, SentWebmention.uuid),
- per_page=25,
- )
-
- return render_template(
- "sent_source.html",
- title="Sent Webmentions",
- source=source,
- webmentions=webmentions,
- action_form=forms.AdminActionForm(),
- )
-
-
-@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend")
-@login_required
-def resend_sent_webmention(
- source_identifier: uuid.UUID, webmention_identifier: uuid.UUID
-) -> ResponseReturnValue:
- """
- Queue an outgoing Webmention to be sent again.
-
- The current desired source revision is reopened for processing so the
- sender task does not treat the Webmention as already completed. Historical
- successful-send fields are preserved, while result fields from the previous
- attempt are cleared.
-
- :param source_identifier: Identifier of the source owning the Webmention.
- :param webmention_identifier: Identifier of the sent Webmention to resend.
- :return: Redirect response to the source details view.
- """
- form = forms.AdminActionForm()
-
- if not form.validate_on_submit():
- return abort(400)
-
- webmention = db.session.get(SentWebmention, webmention_identifier)
-
- if webmention is None or webmention.source_id != source_identifier:
- return abort(404)
-
- # Reopen the existing desired revision instead of inventing a new source
- # revision. send_webmention() otherwise exits when this revision has already
- # been processed.
- webmention.processed_revision = None
- webmention.attempt_count = 0
- webmention.next_retry_at = None
- webmention.status = None
- webmention.failure_reason = None
- webmention.endpoint = None
- webmention.response_status = None
- webmention.status_url = None
-
- db.session.commit()
-
- send_webmention(webmention_identifier)
-
- flash(f"Webmention {webmention.uuid} queued for resend.", "success")
-
- return redirect(url_for("root.sent_source", source_identifier=source_identifier))
-
-
-@root_page.post("/endpoint")
-@LIMITER.limit(
- lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"],
- key_func=lambda: "all-clients",
-)
-@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"])
-@CSRF.exempt
-def endpoint() -> ResponseReturnValue:
- """
- Receive and queue a Webmention for verification.
-
- Existing Webmentions with the same source and target are reset for
- reverification. Concurrent insertion of the same Webmention is handled by
- retrieving and updating the row created by the competing request.
-
- :return: HTTP 201 response with the Webmention status URL in the
- ``Location`` header, or validation errors with HTTP 400.
- """
- form = forms.EndpointForm(meta={"csrf": False})
-
- if not form.validate_on_submit():
- return form.errors, 400
-
- source = cast(str, form.source.data)
- target = cast(str, form.target.data)
-
- url_digest = received_url_digest(source, target)
- webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
- ).scalar_one_or_none()
-
- # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
- if webmention is not None and (webmention.source, webmention.target) != (
- source,
- target,
- ):
- return {"error": "Webmention URL fingerprint conflict"}, 409
-
- should_verify = False
-
- if webmention is None:
- identifier = uuid.uuid7()
-
- webmention = ReceivedWebmention(uuid=identifier, source=source, target=target)
-
- db.session.add(webmention)
-
- try:
- db.session.commit()
- should_verify = True
- except sa_exc.IntegrityError:
- db.session.rollback()
-
- # Another request may have inserted the same source/target pair
- # after our SELECT but before our COMMIT.
- webmention = db.session.execute(
- sa.select(ReceivedWebmention).where(
- ReceivedWebmention.url_digest == url_digest
- )
- ).scalar_one_or_none()
- if webmention is None or (webmention.source, webmention.target) != (
- source,
- target,
- ):
- return {"error": "Webmention URL fingerprint conflict"}, 409
-
- else:
- now = datetime.now(UTC)
- cutoff = now - timedelta(
- seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
- )
-
- # Atomic conditional update: only one concurrent request can reopen an expired verification.
- result = cast(
- CursorResult[Any],
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention.uuid,
- ReceivedWebmention.updated_at <= cutoff,
- )
- .values(
- status="received",
- failure_reason=None,
- recovery_attempts=0,
- updated_at=now,
- )
- .execution_options(synchronize_session=False)
- ),
- )
- should_verify = result.rowcount == 1
- db.session.commit()
-
- identifier = webmention.uuid
-
- if should_verify:
- verify_webmention(identifier)
-
- status_url = url_for("root.status", identifier=str(identifier), _external=True)
-
- return Response(status=201, headers={"Location": status_url})
-
-
-@root_page.route("/status/<uuid:identifier>")
-def status(identifier: uuid.UUID) -> ResponseReturnValue:
- """
- Display the verification status of a received Webmention.
-
- :param identifier: Identifier of the Webmention to display.
- :return: Rendered Webmention status page.
- """
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- return abort(404)
-
- return render_template(
- "status.html", title="Webmention status", webmention=webmention
- )
diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py
new file mode 100644
index 0000000..91966d4
--- /dev/null
+++ b/webmentions_ssg/views/__init__.py
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Register all views under the historic root Blueprint."""
+
+from flask import Blueprint
+
+root_page = Blueprint("root", __name__)
+
+
+def register_routes() -> Blueprint:
+ """Import task-backed view handlers after Huey initialization and return the Blueprint."""
+ from . import admin, endpoint, session # noqa: F401
+
+ return root_page
diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py
new file mode 100644
index 0000000..46e3b18
--- /dev/null
+++ b/webmentions_ssg/views/admin.py
@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Authenticated incoming and outgoing Webmention management."""
+
+import uuid
+
+import sqlalchemy as sa
+from flask import abort, flash, redirect, render_template, request, url_for
+from flask.typing import ResponseReturnValue
+from flask_login import login_required
+from sqlalchemy.orm import selectinload
+
+from .. import forms
+from ..extensions import DATABASE as db
+from ..models import ReceivedWebmention, SentWebmention, Source
+from ..tasks.receiver import verify_webmention
+from ..tasks.scanner import manual_scan_sources
+from ..tasks.sender import send_webmention
+from . import root_page
+
+
+@root_page.route("/received")
+@login_required
+def received() -> ResponseReturnValue:
+ """
+ Display received Webmentions.
+
+ :return: Rendered page containing the paginated received Webmentions.
+ """
+ webmentions = db.paginate(
+ sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()),
+ per_page=25,
+ )
+
+ return render_template(
+ "received.html",
+ title="Received Webmentions",
+ webmentions=webmentions,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/received/<uuid:identifier>/delete")
+@login_required
+def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Delete a received Webmention.
+
+ :param identifier: Identifier of the Webmention to delete.
+ :return: Redirect response to the received Webmentions view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ db.session.delete(webmention)
+ db.session.commit()
+
+ flash(f"Webmention {webmention.uuid} deleted.", "success")
+
+ return redirect(
+ url_for("root.received", page=request.args.get("page", 1, type=int))
+ )
+
+
+@root_page.post("/received/<uuid:identifier>/reverify")
+@login_required
+def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Queue a received Webmention for reverification.
+
+ The Webmention status is reset before a new verification task is queued.
+
+ :param identifier: Identifier of the Webmention to reverify.
+ :return: Redirect response to the received Webmentions view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ webmention = webmention.reset_for_verification()
+
+ db.session.commit()
+
+ verify_webmention(identifier)
+
+ flash(f"Webmention {webmention.uuid} queued for reverification.", "success")
+
+ return redirect(
+ url_for("root.received", page=request.args.get("page", 1, type=int))
+ )
+
+
+@root_page.route("/sent")
+@login_required
+def sent() -> ResponseReturnValue:
+ """
+ Display sources with sent Webmentions.
+
+ :return: Rendered page containing the paginated source list.
+ """
+ sources = db.paginate(
+ sa.select(Source)
+ .options(selectinload(Source.sent_webmentions))
+ .order_by(Source.last_seen_at.desc()),
+ per_page=25,
+ )
+ return render_template(
+ "sent.html",
+ title="Sent Webmentions",
+ sources=sources,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/sent/rescan")
+@login_required
+def rescan_sent_sources() -> ResponseReturnValue:
+ """
+ Queue a manual scan of sent Webmention sources.
+
+ :return: Redirect response to the sent Webmentions view.
+ """
+ form = forms.AdminActionForm()
+ if form.validate_on_submit():
+ manual_scan_sources()
+ flash("Source rescan queued.", "success")
+ return redirect(url_for("root.sent"))
+
+
+@root_page.route("/sent/<uuid:source_identifier>")
+@login_required
+def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Display the sent Webmentions associated with a source.
+
+ :param source_identifier: Identifier of the source to display.
+ :return: Rendered source details page.
+ """
+ source = db.session.scalar(
+ sa.select(Source).where(Source.uuid == source_identifier)
+ )
+
+ if source is None:
+ return abort(404)
+
+ webmentions = db.paginate(
+ sa.select(SentWebmention)
+ .where(SentWebmention.source_id == source.uuid)
+ .order_by(SentWebmention.target, SentWebmention.uuid),
+ per_page=25,
+ )
+
+ return render_template(
+ "sent_source.html",
+ title="Sent Webmentions",
+ source=source,
+ webmentions=webmentions,
+ action_form=forms.AdminActionForm(),
+ )
+
+
+@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend")
+@login_required
+def resend_sent_webmention(
+ source_identifier: uuid.UUID, webmention_identifier: uuid.UUID
+) -> ResponseReturnValue:
+ """
+ Queue an outgoing Webmention to be sent again.
+
+ The current desired source revision is reopened for processing so the
+ sender task does not treat the Webmention as already completed. Historical
+ successful-send fields are preserved, while result fields from the previous
+ attempt are cleared.
+
+ :param source_identifier: Identifier of the source owning the Webmention.
+ :param webmention_identifier: Identifier of the sent Webmention to resend.
+ :return: Redirect response to the source details view.
+ """
+ form = forms.AdminActionForm()
+
+ if not form.validate_on_submit():
+ return abort(400)
+
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+
+ if webmention is None or webmention.source_id != source_identifier:
+ return abort(404)
+
+ # Reopen the existing desired revision instead of inventing a new source
+ # revision. send_webmention() otherwise exits when this revision has already
+ # been processed.
+ webmention = webmention.reopen(
+ webmention.desired_revision, webmention.active, resend=True
+ )
+
+ db.session.commit()
+
+ send_webmention(webmention_identifier)
+
+ flash(f"Webmention {webmention.uuid} queued for resend.", "success")
+
+ return redirect(url_for("root.sent_source", source_identifier=source_identifier))
diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py
new file mode 100644
index 0000000..1c2fe13
--- /dev/null
+++ b/webmentions_ssg/views/endpoint.py
@@ -0,0 +1,150 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Public Webmention endpoint and verification status view."""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+
+import sqlalchemy as sa
+import sqlalchemy.exc as sa_exc
+from flask import Response, abort, current_app, render_template, url_for
+from flask.typing import ResponseReturnValue
+from sqlalchemy.engine import CursorResult
+
+from .. import forms
+from ..extensions import CSRF, LIMITER
+from ..extensions import DATABASE as db
+from ..models import ReceivedWebmention, received_url_digest
+from ..tasks.receiver import verify_webmention
+from . import root_page
+
+
+def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None:
+ """Upsert a submission atomically and return (identifier, enqueue flag).
+
+ ``None`` means that the URL fingerprint matches a different source/target.
+ No background task is enqueued until the transaction is committed.
+ """
+ url_digest = received_url_digest(source, target)
+ webmention = db.session.execute(
+ sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest)
+ ).scalar_one_or_none()
+
+ # A SHA-256 collision must never cause an unrelated Webmention to be overwritten.
+ if webmention is not None and (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return None
+
+ should_verify = False
+
+ if webmention is None:
+ identifier = uuid.uuid7()
+ webmention = ReceivedWebmention(uuid=identifier, source=source, target=target)
+ db.session.add(webmention)
+
+ try:
+ db.session.commit()
+ should_verify = True
+ except sa_exc.IntegrityError:
+ db.session.rollback()
+
+ # Another request may have inserted the same source/target pair
+ # after our SELECT but before our COMMIT.
+ webmention = db.session.execute(
+ sa.select(ReceivedWebmention).where(
+ ReceivedWebmention.url_digest == url_digest
+ )
+ ).scalar_one_or_none()
+ if webmention is None or (webmention.source, webmention.target) != (
+ source,
+ target,
+ ):
+ return None
+
+ else:
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(
+ seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
+ )
+
+ # Atomic conditional update: only one concurrent request can reopen an expired verification.
+ result = cast(
+ CursorResult[Any],
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention.uuid,
+ ReceivedWebmention.updated_at <= cutoff,
+ )
+ .values(ReceivedWebmention.verification_reset_values(at=now))
+ .execution_options(synchronize_session=False)
+ ),
+ )
+ should_verify = result.rowcount == 1
+ db.session.commit()
+
+ return webmention.uuid, should_verify
+
+
+@root_page.post("/endpoint")
+@LIMITER.limit(
+ lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"],
+ key_func=lambda: "all-clients",
+)
+@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"])
+@CSRF.exempt
+def endpoint() -> ResponseReturnValue:
+ """
+ Receive and queue a Webmention for verification.
+
+ Existing Webmentions with the same source and target are reset for
+ reverification. Concurrent insertion of the same Webmention is handled by
+ retrieving and updating the row created by the competing request.
+
+ :return: HTTP 201 response with the Webmention status URL in the
+ ``Location`` header, or validation errors with HTTP 400.
+ """
+ form = forms.EndpointForm(meta={"csrf": False})
+
+ if not form.validate_on_submit():
+ return form.errors, 400
+
+ source = cast(str, form.source.data)
+ target = cast(str, form.target.data)
+
+ result = submit_received(source, target)
+
+ if result is None:
+ return {"error": "Webmention URL fingerprint conflict"}, 409
+
+ identifier, should_verify = result
+
+ if should_verify:
+ verify_webmention(identifier)
+
+ status_url = url_for("root.status", identifier=str(identifier), _external=True)
+
+ return Response(status=201, headers={"Location": status_url})
+
+
+@root_page.route("/status/<uuid:identifier>")
+def status(identifier: uuid.UUID) -> ResponseReturnValue:
+ """
+ Display the verification status of a received Webmention.
+
+ :param identifier: Identifier of the Webmention to display.
+ :return: Rendered Webmention status page.
+ """
+ webmention = db.session.get(ReceivedWebmention, identifier)
+
+ if webmention is None:
+ return abort(404)
+
+ return render_template(
+ "status.html", title="Webmention status", webmention=webmention
+ )
diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py
new file mode 100644
index 0000000..7bd8a10
--- /dev/null
+++ b/webmentions_ssg/views/session.py
@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Authentication and session routes."""
+
+import sqlalchemy as sa
+from flask import current_app, flash, redirect, render_template, request, url_for
+from flask.typing import ResponseReturnValue
+from flask_login import current_user, login_required, login_user, logout_user
+
+from .. import forms
+from ..auth import DUMMY_USER, is_safe_next_url, login_username_limit_key
+from ..extensions import DATABASE as db
+from ..extensions import LIMITER
+from ..models import User
+from . import root_page
+
+
+@root_page.route("/")
+def index() -> ResponseReturnValue:
+ """
+ Redirect to the received Webmentions view.
+
+ :return: Redirect response to the received Webmentions view.
+ """
+ return redirect(url_for("root.received"))
+
+
+@root_page.route("/login", methods=["GET", "POST"])
+@LIMITER.limit(
+ lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"],
+ key_func=login_username_limit_key,
+ methods=["POST"],
+ error_message="Too many login attempts. Please try again later.",
+)
+@LIMITER.limit(
+ lambda: current_app.config["LOGIN_RATE_LIMIT"],
+ methods=["POST"],
+ error_message="Too many login attempts. Please try again later.",
+)
+def login() -> ResponseReturnValue:
+ """
+ Authenticate a user and start a login session.
+
+ Authenticated users are redirected to the application index. After a
+ successful login, the user is redirected to the requested local URL when
+ provided.
+
+ :return: Rendered login page or redirect response.
+ """
+ if current_user.is_authenticated:
+ return redirect(url_for("root.index"))
+
+ form = forms.LoginForm()
+
+ if form.validate_on_submit():
+ user = db.session.scalar(
+ sa.select(User).where(User.username == form.username.data)
+ )
+
+ user_exists = user is not None
+
+ if user is None:
+ user = DUMMY_USER
+
+ if (
+ form.password.data is None
+ or not user.check_password(form.password.data)
+ or not user_exists
+ ):
+ flash("Invalid username or password", "danger")
+ return redirect(url_for("root.login"))
+ login_user(user)
+
+ next_url = request.args.get("next")
+ if not next_url or not is_safe_next_url(next_url):
+ next_url = url_for("root.index")
+ return redirect(next_url)
+
+ return render_template("login.html", form=form)
+
+
+@root_page.route("/logout", methods=("POST",))
+@login_required
+def logout() -> ResponseReturnValue:
+ """
+ End the current user's login session.
+
+ :return: Redirect response to the application index.
+ """
+ logout_user()
+ return redirect(url_for("root.index"))