diff options
| author | Dennis Fink | 2026-11-12 19:57:08 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-12 19:57:08 +0100 |
| commit | 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch) | |
| tree | f7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg | |
| parent | 7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff) | |
| download | webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip | |
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and
application extension setup so each module owns a focused concern.
Reorganize the test suite around the new boundaries while preserving
existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg')
37 files changed, 2086 insertions, 2113 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 8fed1a7..63e9f13 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -8,47 +8,20 @@ import logging import logging.handlers import os from pathlib import Path -from urllib.parse import urlsplit import coloredlogs from flask import Flask -from flask_bootstrap import Bootstrap5 -from flask_limiter import Limiter -from flask_limiter.util import get_remote_address -from flask_mail import Mail -from flask_migrate import Migrate -from flask_sqlalchemy import SQLAlchemy -from flask_wtf.csrf import CSRFProtect -from sqlalchemy.orm import DeclarativeBase from werkzeug.middleware.proxy_fix import ProxyFix -from webmentions_ssg.config import ( - DefaultConfig, - DevelopmentConfig, - ProductionConfig, - normalize_allowed_hostnames, - normalize_ignored_hostnames, -) - -from .security import validate_secret_key -from .tasks import Huey -from .url_security import is_http_url - APP_NAME = "webmentions-ssg" VERSION = "0.0.2" - -class Base(DeclarativeBase): - pass - - -CSRF = CSRFProtect() -BOOTSTRAP = Bootstrap5() -DATABASE = SQLAlchemy(model_class=Base) -MIGRATE = Migrate() -HUEY = Huey() -MAIL = Mail() -LIMITER = Limiter(key_func=get_remote_address) +from .auth import AUTH +from .cli import DB_CLI, MAIL_CLI, USERS_CLI +from .config import DefaultConfig, DevelopmentConfig, ProductionConfig, validate_config +from .extensions import BOOTSTRAP, CSRF, DATABASE, HUEY, LIMITER, MAIL, MIGRATE +from .net import is_http_url +from .views import register_routes def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: @@ -91,71 +64,15 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.config.from_prefixed_env(prefix="FLASK") app.config.from_prefixed_env(prefix="WEBMENTIONS_SSG") - # A fixed key is convenient only for local debugging. All other - # deployments must supply their own persistent, random signing key. - using_local_dev_key = ( - issubclass(config_class, DevelopmentConfig) - and app.debug - and app.config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY - ) - if not app.testing and not using_local_dev_key: - validate_secret_key(app.config.get("SECRET_KEY")) + app.config.update(validate_config(app.config, config_class=config_class)) - app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames( - app.config["ALLOWED_HOSTNAMES"] - ) - app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames( - app.config["IGNORED_HOSTNAMES"] - ) - proxy_hops = app.config["PROXY_FIX_TRUSTED_HOPS"] - if type(proxy_hops) is not int or proxy_hops < 0: - raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.") - if proxy_hops: - # Trust only the configured number of proxies; never trust forwarded - # host, port or URL prefix headers for rate limiting. + if proxy_hops := app.config["PROXY_FIX_TRUSTED_HOPS"]: + # Do not trust forwarded host, port, or URL prefixes. app.wsgi_app = ProxyFix(app.wsgi_app, x_for=proxy_hops, x_proto=proxy_hops) - for key in ( - "MAX_CONTENT_LENGTH", - "MAX_WEBMENTION_URL_BYTES", - "RECEIVED_RETENTION_DAYS", - ): - value = app.config[key] - if type(value) is not int or value <= 0: - raise ValueError(f"{key} must be a positive integer.") - if not app.testing and not app.debug and not app.config["ALLOWED_HOSTNAMES"]: - raise RuntimeError("Configure ALLOWED_HOSTNAMES before running in production.") - - if ( - not app.testing - and not app.debug - and app.config["RATELIMIT_STORAGE_URI"] == "memory://" - ): - raise RuntimeError( - "Configure RATELIMIT_STORAGE_URI with shared storage before running in production." - ) - - if not app.testing and not app.debug: - # HUEY_URL overrides the legacy HUEY.url option; absent either, Huey - # falls back to an in-process queue that cannot serve a separate worker. - huey_url = app.config.get( - "HUEY_URL", app.config.get("HUEY", {}).get("url", "memory://") - ) - if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in { - "memory", - "blackhole", - }: - raise RuntimeError( - "Configure HUEY_URL with a shared, persistent queue before " - "running in production (memory:// and blackhole:// are unsafe)." - ) - - # Local development and tests should start from a fresh checkout: their - # default log directory and development SQLite instance are generated. - if app.testing or app.debug: - Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True) - if app.debug: + if app.debug or app.testing: Path(app.instance_path).mkdir(parents=True, exist_ok=True) + Path(app.config["LOG_FILE"]).parent.mkdir(parents=True, exist_ok=True) default_logging_formatter = logging.Formatter(app.config["LOG_FORMAT"]) @@ -197,8 +114,6 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: LIMITER.init_app(app) MAIL.init_app(app) - from .auth import AUTH - AUTH.init_app(app) BOOTSTRAP.init_app(app) HUEY.init_app(app) @@ -214,20 +129,12 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.add_template_filter(is_http_url) - from .views import root_page - - app.register_blueprint(root_page) - - from .cli.users import USERS_CLI + app.register_blueprint(register_routes()) app.cli.add_command(USERS_CLI) - from .cli.mail import MAIL_CLI - app.cli.add_command(MAIL_CLI) - from .cli.database import DB_CLI - app.cli.add_command(DB_CLI) return app diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index 147220e..633aa82 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -4,17 +4,43 @@ """Configure user authentication and session loading.""" +import hashlib +import secrets from hmac import compare_digest +from urllib.parse import urlsplit +from flask import request +from flask_limiter.util import get_remote_address from flask_login import LoginManager -from . import DATABASE as db +from .extensions import DATABASE as db from .models import User AUTH = LoginManager() AUTH.login_view = "root.login" # pyright: ignore[reportAttributeAccessIssue] +MIN_PASSWORD_LENGTH = 8 + +DUMMY_USER = User(username=secrets.token_urlsafe()) +DUMMY_USER.password = secrets.token_urlsafe() + + +def validate_password(password: str) -> None: + """Validate newly chosen passwords without arbitrary composition rules. + + Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases, + but reject empty passwords and control characters. + """ + if not password: + raise ValueError("Password must not be empty.") + if len(password) < MIN_PASSWORD_LENGTH: + raise ValueError( + f"Password must be at least {MIN_PASSWORD_LENGTH} characters long." + ) + if any(ord(char) < 32 or ord(char) == 127 for char in password): + raise ValueError("Password must not contain control characters.") + @AUTH.user_loader def load_user(session_id: str) -> User | None: @@ -44,3 +70,28 @@ def load_user(session_id: str) -> User | None: if not compare_digest(session_id, user.get_id()): return None return user + + +def login_username_limit_key() -> str: + """Group usernames across IPs without exposing plaintext to rate-limit storage.""" + username = request.form.get("username", "").strip().casefold() + if not username: + return f"missing-username:{get_remote_address()}" + digest = hashlib.sha256(username.encode("utf-8")).hexdigest() + return f"username:{digest}" + + +def is_safe_next_url(next_url: str) -> bool: + """Allow redirects only to paths on this site.""" + if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"): + return False + try: + target = urlsplit(next_url) + except ValueError: + return False + return ( + not target.scheme + and not target.netloc + and target.path.startswith("/") + and not target.path.startswith("//") + ) diff --git a/webmentions_ssg/cli/users.py b/webmentions_ssg/cli.py index a48fc71..0387ba6 100644 --- a/webmentions_ssg/cli/users.py +++ b/webmentions_ssg/cli.py @@ -2,22 +2,26 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Provide commands for managing application user accounts.""" +"""Manage users, mail settings, and database migrations.""" import click import sqlalchemy as sa +from flask import current_app from flask.cli import AppGroup +from flask_mail import Message +from flask_migrate import upgrade as migrate_upgrade +from sqlalchemy import inspect from sqlalchemy.exc import IntegrityError -from .. import DATABASE -from ..security import validate_password +from .auth import validate_password +from .extensions import DATABASE, MAIL +from .models import User USERS_CLI = AppGroup("users", help="Manage users.") def _require_user(username: str): """Return a user by username or report a CLI-friendly error.""" - from ..models import User user = DATABASE.session.scalar(sa.select(User).where(User.username == username)) if user is None: @@ -36,7 +40,6 @@ def create_user(username: str, password: str) -> None: :param password: Plain-text password to hash and store. :raises click.ClickException: If the credentials are invalid or the user exists. """ - from ..models import User username = username.strip() if not username: @@ -63,7 +66,6 @@ def create_user(username: str, password: str) -> None: @USERS_CLI.command("list", help="List registered users.") def list_users() -> None: """List user IDs and usernames in alphabetical order.""" - from ..models import User users = DATABASE.session.scalars(sa.select(User).order_by(User.username)).all() if not users: @@ -115,3 +117,83 @@ def delete_user(username: str, yes: bool) -> None: DATABASE.session.delete(user) DATABASE.session.commit() click.echo(f"Deleted user {username!r}.") + + +MAIL_CLI = AppGroup("mail", help="Handle mail") + + +@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") +@click.option( + "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)." +) +def send_test_mail(recipient: str | None) -> None: + """ + Send a test email using the configured SMTP server. + + :param recipient: Optional recipient overriding NOTIFICATION_EMAIL. + :raises click.ClickException: If configuration is missing or sending fails. + """ + recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL") + if not recipient: + raise click.ClickException( + "No recipient specified. Set NOTIFICATION_EMAIL or pass --to." + ) + + sender = current_app.config.get("MAIL_DEFAULT_SENDER") + if not sender: + raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.") + + message = Message( + subject="webmentions-ssg: test email", + sender=sender, + recipients=[recipient], + body=( + "This is a test email from webmentions-ssg.\n\n" + "If you received this message, the outgoing mail configuration works.\n" + ), + ) + + try: + MAIL.send(message) + except Exception as exc: + raise click.ClickException(f"Could not send test email: {exc}") from exc + + click.echo(f"Test email sent to {recipient}.") + + +DB_CLI = AppGroup("db", help="Handle database") + + +def _table_names() -> set[str]: + """Return existing table names without modifying the database schema.""" + with DATABASE.engine.connect() as connection: + return set(inspect(connection).get_table_names()) + + +@DB_CLI.command("create-db", help="Create a new database using all migrations") +def create_db() -> None: + """Initialize an empty database and record its Alembic revision.""" + if _table_names(): + raise click.ClickException( + "The database already contains tables. " + "Use 'flask db upgrade' for an existing migrated database." + ) + + # Applying migrations (rather than create_all()) keeps alembic_version in sync. + migrate_upgrade() + click.echo("Database created at the latest migration revision.") + + +@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision") +def upgrade_db() -> None: + """Apply pending migrations to a previously initialized database.""" + if "alembic_version" not in _table_names(): + raise click.ClickException( + "No Flask-Migrate revision table found. " + "For a new, empty database run 'flask db create-db'. " + "For a legacy database, inspect its schema and use " + "'flask migrate stamp' only if appropriate." + ) + + migrate_upgrade() + click.echo("Database upgraded to the latest migration revision.") diff --git a/webmentions_ssg/cli/__init__.py b/webmentions_ssg/cli/__init__.py deleted file mode 100644 index accaaa3..0000000 --- a/webmentions_ssg/cli/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Group the application command-line management modules.""" diff --git a/webmentions_ssg/cli/database.py b/webmentions_ssg/cli/database.py deleted file mode 100644 index e918600..0000000 --- a/webmentions_ssg/cli/database.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Simple database commands built on Flask-Migrate.""" - -import click -from flask.cli import AppGroup -from flask_migrate import upgrade as migrate_upgrade -from sqlalchemy import inspect - -from .. import DATABASE - -DB_CLI = AppGroup("db", help="Handle database") - - -def _table_names() -> set[str]: - """Return existing table names without modifying the database schema.""" - with DATABASE.engine.connect() as connection: - return set(inspect(connection).get_table_names()) - - -@DB_CLI.command("create-db", help="Create a new database using all migrations") -def create_db() -> None: - """Initialize an empty database and record its Alembic revision.""" - if _table_names(): - raise click.ClickException( - "The database already contains tables. " - "Use 'flask db upgrade' for an existing migrated database." - ) - - # Applying migrations (rather than create_all()) keeps alembic_version in sync. - migrate_upgrade() - click.echo("Database created at the latest migration revision.") - - -@DB_CLI.command("upgrade", help="Upgrade an existing database to the latest revision") -def upgrade_db() -> None: - """Apply pending migrations to a previously initialized database.""" - if "alembic_version" not in _table_names(): - raise click.ClickException( - "No Flask-Migrate revision table found. " - "For a new, empty database run 'flask db create-db'. " - "For a legacy database, inspect its schema and use " - "'flask migrate stamp' only if appropriate." - ) - - migrate_upgrade() - click.echo("Database upgraded to the latest migration revision.") diff --git a/webmentions_ssg/cli/mail.py b/webmentions_ssg/cli/mail.py deleted file mode 100644 index 47687e2..0000000 --- a/webmentions_ssg/cli/mail.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Provide a command for testing outgoing email configuration.""" - -import click -from flask import current_app -from flask.cli import AppGroup -from flask_mail import Message - -from .. import MAIL - -MAIL_CLI = AppGroup("mail", help="Handle mail") - - -@MAIL_CLI.command("test", help="Send an email to test the SMTP configuration") -@click.option( - "--to", "recipient", help="Recipient address (defaults to NOTIFICATION_EMAIL)." -) -def send_test_mail(recipient: str | None) -> None: - """ - Send a test email using the configured SMTP server. - - :param recipient: Optional recipient overriding NOTIFICATION_EMAIL. - :raises click.ClickException: If configuration is missing or sending fails. - """ - recipient = recipient or current_app.config.get("NOTIFICATION_EMAIL") - if not recipient: - raise click.ClickException( - "No recipient specified. Set NOTIFICATION_EMAIL or pass --to." - ) - - sender = current_app.config.get("MAIL_DEFAULT_SENDER") - if not sender: - raise click.ClickException("MAIL_DEFAULT_SENDER is not configured.") - - message = Message( - subject="webmentions-ssg: test email", - sender=sender, - recipients=[recipient], - body=( - "This is a test email from webmentions-ssg.\n\n" - "If you received this message, the outgoing mail configuration works.\n" - ), - ) - - try: - MAIL.send(message) - except Exception as exc: - raise click.ClickException(f"Could not send test email: {exc}") from exc - - click.echo(f"Test email sent to {recipient}.") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 8a3e58d..8251da3 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -5,39 +5,11 @@ """Define application settings for production, development, and testing.""" import logging +from urllib.parse import urlsplit +from flask.config import Config -def normalize_allowed_hostnames(value: object) -> set[str]: - """Validate and normalize the configured collection of allowed hostnames.""" - if not isinstance(value, (list, tuple, set, frozenset)): - raise TypeError( - "ALLOWED_HOSTNAMES must be a list, tuple, or set of hostnames, not a string." - ) - - hostnames: set[str] = set() - for hostname in value: - if not isinstance(hostname, str) or not hostname.strip(): - raise ValueError("ALLOWED_HOSTNAMES must contain non-empty strings only.") - hostnames.add(hostname.strip().lower()) - return hostnames - - -def normalize_ignored_hostnames(value: object) -> tuple[str, ...]: - """Validate and normalize hostname patterns ignored by the source scanner.""" - if not isinstance(value, (list, tuple, set, frozenset)): - raise TypeError( - "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, " - "not a string." - ) - - patterns: set[str] = set() - for pattern in value: - if not isinstance(pattern, str) or not ( - normalized := pattern.strip().lower().rstrip(".") - ): - raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.") - patterns.add(normalized) - return tuple(sorted(patterns)) +MIN_SECRET_KEY_LENGTH = 32 class DefaultConfig: @@ -69,6 +41,7 @@ class DefaultConfig: RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 + MAX_TARGET_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 # Absolute wall-clock budget for each verification or delivery attempt. REQUEST_TOTAL_TIMEOUT = 30.0 @@ -144,3 +117,107 @@ class TestingConfig(DefaultConfig): HUEY_URL = "memory://" ALLOWED_HOSTNAMES = {"dennisfink.me"} + + +def normalize_hostnames(value: object) -> set[str]: + """Validate and normalize a collection of hostnames.""" + if not isinstance(value, (list, tuple, set, frozenset)): + raise TypeError("Expected a list, tuple, or set of hostnames.") + + hostnames: set[str] = set() + for hostname in value: + if not isinstance(hostname, str) or not ( + normalized := hostname.strip().lower().rstrip(".") + ): + raise ValueError("Hostnames must be non-empty strings.") + + hostnames.add(normalized) + + return hostnames + + +def validate_secret_key(value: object) -> str | bytes: + """Require a non-placeholder, sufficiently long deployment signing key.""" + if not isinstance(value, (str, bytes)): + raise TypeError( + "Configure a persistent SECRET_KEY of at least 32 random bytes " + "before starting." + ) + + # Flask accepts either text or bytes as its signing key. Reject common + # example/development values even when their lengths look sufficient. + key = value.encode("utf-8") if isinstance(value, str) else value + if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip(): + raise RuntimeError( + "Configure a persistent SECRET_KEY of at least 32 random bytes " + "before starting." + ) + + normalized = key.strip().lower() + if normalized.startswith( + (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_") + ): + raise RuntimeError( + "SECRET_KEY is an example/development placeholder; " + "generate and configure a real random key." + ) + + return value + + +def validate_config( + config: Config, *, config_class: type[DefaultConfig] +) -> dict[str, object]: + """Validate runtime configuration and return normalized values. + + The caller applies the returned values once; validation does not mutate + the Flask application or install extensions. + """ + normalized: dict[str, object] = {} + for key in ("ALLOWED_HOSTNAMES", "IGNORED_HOSTNAMES"): + try: + normalized[key] = normalize_hostnames(config[key]) + except (TypeError, ValueError) as exc: + raise type(exc)(f"{key}: {exc}") from exc + + proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] + if type(proxy_hops) is not int or proxy_hops < 0: + raise ValueError("PROXY_FIX_TRUSTED_HOPS must be a non-negative integer.") + + for key in ( + "MAX_CONTENT_LENGTH", + "MAX_WEBMENTION_URL_BYTES", + "RECEIVED_RETENTION_DAYS", + ): + value = config[key] + if type(value) is not int or value <= 0: + raise ValueError(f"{key} must be a positive integer.") + + if not config["TESTING"] and not ( + issubclass(config_class, DevelopmentConfig) + and config["DEBUG"] + and config.get("SECRET_KEY") == DevelopmentConfig.SECRET_KEY + ): + normalized["SECRET_KEY"] = validate_secret_key(config.get("SECRET_KEY")) + + if not config["DEBUG"] and not config["TESTING"]: + if not normalized["ALLOWED_HOSTNAMES"]: + raise RuntimeError( + "Configure ALLOWED_HOSTNAMES before running in production." + ) + if config["RATELIMIT_STORAGE_URI"] == "memory://": + raise RuntimeError( + "Configure RATELIMIT_STORAGE_URI with shared storage before running in production." + ) + huey_url = config.get( + "HUEY_URL", config.get("HUEY", {}).get("url", "memory://") + ) + if isinstance(huey_url, str) and urlsplit(huey_url).scheme.lower() in { + "memory", + "blackhole", + }: + raise RuntimeError( + "Configure HUEY_URL with a shared, persistent queue before " + "running in production (memory:// and blackhole:// are unsafe)." + ) + return normalized diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py new file mode 100644 index 0000000..0afec9c --- /dev/null +++ b/webmentions_ssg/extensions.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unbound extensions shared across Flask application instances.""" + +from flask_bootstrap import Bootstrap5 +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address +from flask_mail import Mail +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect +from sqlalchemy.orm import DeclarativeBase + +from .tasks.extension import Huey + + +class Base(DeclarativeBase): + pass + + +CSRF = CSRFProtect() +BOOTSTRAP = Bootstrap5() +DATABASE = SQLAlchemy(model_class=Base) +MIGRATE = Migrate() +HUEY = Huey() +MAIL = Mail() +LIMITER = Limiter(key_func=get_remote_address) diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms.py index 384c7a1..a721dc6 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms.py @@ -2,19 +2,21 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Provide validators for URL safety and form field constraints.""" +"""Webmention forms and URL validation.""" import ipaddress +import re import unicodedata from urllib.parse import urlsplit import httpx from flask import current_app -from wtforms import Field, ValidationError -from wtforms.validators import StopValidation +from flask_wtf import FlaskForm +from wtforms import Field, PasswordField, StringField, SubmitField, ValidationError from wtforms.form import BaseForm +from wtforms.validators import URL, InputRequired, Regexp, StopValidation -from ..url_security import AddressResolutionError, is_public_address, is_public_url +from .net import is_public_address class URLMaxBytes: @@ -143,7 +145,7 @@ class AllowedHostname: :raises ValidationError: If the URL hostname is not configured as allowed. """ try: - hostname = urlsplit(field.data).hostname + hostname = (urlsplit(field.data).hostname or "").rstrip(".") except ValueError, TypeError: hostname = None if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]: @@ -151,42 +153,6 @@ class AllowedHostname: raise ValidationError(self.message or field.gettext("Invalid input.")) -class PublicURL: - """ - Validate that a URL resolves exclusively to public IP addresses. - - :param message: Optional validation error message. - """ - - def __init__(self, message: str | None = None) -> None: - """ - Initialize the validator. - - :param message: Optional validation error message. - """ - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - """ - Validate that a URL resolves exclusively to public IP addresses. - - :param form: Form containing the field. - :param field: Field containing the URL to validate. - :raises ValidationError: If the URL cannot be resolved, has no hostname, or - resolves to a non-public address. - """ - message = self.message - - if message is None: - message = field.gettext("URL must resolve to a public address.") - - try: - if not is_public_url(field.data): - raise ValidationError(message) - except (AddressResolutionError, ValueError) as exc: - raise ValidationError(message) from exc - - class PublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" @@ -206,3 +172,62 @@ class PublicIPLiteral: if not is_public_address(address): raise ValidationError(field.gettext("URL must use a public address.")) + + +class LoginForm(FlaskForm): + """ + Form for authenticating a user. + """ + + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") + + +class EndpointForm(FlaskForm): + """ + Form for receiving a Webmention. + + The source must be a public HTTP or HTTPS URL distinct from the target. + The target must be an HTTP or HTTPS URL using an allowed hostname. + """ + + source = StringField( + "source", + validators=[ + InputRequired(), + URLMaxBytes(), + HTTPXURL(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="source must begin with http or https", + ), + NotEqualTo("target"), + PublicIPLiteral(), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + URLMaxBytes(), + HTTPXURL(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="target must begin with http or https", + ), + AllowedHostname(), + ], + ) + + +class AdminActionForm(FlaskForm): + """ + Form for confirming an administrative action. + """ + + submit = SubmitField("Confirm") diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py deleted file mode 100644 index 589cbd9..0000000 --- a/webmentions_ssg/forms/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Define forms for login, Webmention submission, and administration.""" - -import re - -from flask_wtf import FlaskForm -from wtforms import PasswordField, StringField, SubmitField -from wtforms.validators import URL, InputRequired, Regexp - -from .validators import ( - AllowedHostname, - HTTPXURL, - NotEqualTo, - PublicIPLiteral, - URLMaxBytes, -) - - -class LoginForm(FlaskForm): - """ - Form for authenticating a user. - """ - - username = StringField("Username", validators=[InputRequired()]) - password = PasswordField("Password", validators=[InputRequired()]) - submit = SubmitField("Sign In") - - -class EndpointForm(FlaskForm): - """ - Form for receiving a Webmention. - - The source must be a public HTTP or HTTPS URL distinct from the target. - The target must be an HTTP or HTTPS URL using an allowed hostname. - """ - - source = StringField( - "source", - validators=[ - InputRequired(), - URLMaxBytes(), - HTTPXURL(), - URL(), - Regexp( - "^https?://.*", - flags=re.IGNORECASE, - message="source must begin with http or https", - ), - NotEqualTo("target"), - PublicIPLiteral(), - ], - ) - target = StringField( - "target", - validators=[ - InputRequired(), - URLMaxBytes(), - HTTPXURL(), - URL(), - Regexp( - "^https?://.*", - flags=re.IGNORECASE, - message="target must begin with http or https", - ), - AllowedHostname(), - ], - ) - - -class AdminActionForm(FlaskForm): - """ - Form for confirming an administrative action. - """ - - submit = SubmitField("Confirm") diff --git a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py index 905b2c2..a263a2f 100644 --- a/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py +++ b/webmentions_ssg/migrations/versions/408fc2eca9ce_track_webmention_digest_notifications.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Track webmention digest notifications Revision ID: 408fc2eca9ce diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py index c4eac52..ee6ad09 100644 --- a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Add Webmention retry backoff Revision ID: 85f2b226ec99 diff --git a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py index cbb40ef..08dd45f 100644 --- a/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py +++ b/webmentions_ssg/migrations/versions/c0272f25ce17_index_received_verification_queue.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + """Index received webmentions for expired verification recovery. Revision ID: c0272f25ce17 diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 0157df3..41c5376 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,7 +11,7 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.url_digest import received_url_digest, sent_target_digest +from webmentions_ssg.models import received_url_digest, sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 31516d5..9442309 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import uuid from datetime import UTC, datetime from enum import StrEnum @@ -28,8 +29,17 @@ from sqlalchemy import ( from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from werkzeug.security import check_password_hash, generate_password_hash -from . import Base -from .url_digest import received_url_digest, sent_target_digest +from .extensions import Base + + +class ReceivedStatus(StrEnum): + """Persisted lifecycle state of an incoming Webmention.""" + + RECEIVED = "received" + VERIFYING = "verifying" + VERIFIED = "verified" + FAILED = "failed" + DELETED = "deleted" class SentWebmentionStatus(StrEnum): @@ -120,7 +130,9 @@ class ReceivedWebmention(Base): self.url_digest = received_url_digest(source, target) return value - status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + status: Mapped[str] = mapped_column( + Text(), nullable=False, default=ReceivedStatus.RECEIVED + ) # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh # explicit submission resets this counter. recovery_attempts: Mapped[int] = mapped_column( @@ -154,6 +166,24 @@ class ReceivedWebmention(Base): """ return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" + @staticmethod + def verification_reset_values( + *, at: datetime, recoveries: int = 0 + ) -> dict[str, object]: + """Return the columns to reset when verification is explicitly reopened.""" + return { + "status": ReceivedStatus.RECEIVED, + "failure_reason": None, + "recovery_attempts": recoveries, + "updated_at": at, + } + + def reset_for_verification(self) -> ReceivedWebmention: + """Reopen an existing row for manual verification and return it.""" + for key, value in self.verification_reset_values(at=datetime.now(UTC)).items(): + setattr(self, key, value) + return self + @property def verified(self) -> bool: """ @@ -161,7 +191,7 @@ class ReceivedWebmention(Base): :return: Whether the Webmention status is ``verified``. """ - return self.status == "verified" + return self.status == ReceivedStatus.VERIFIED @property def created_at(self) -> datetime: @@ -308,6 +338,23 @@ class SentWebmention(Base): """ return f"<SentWebmention {self.target!r}>" + def reopen( + self, revision: int, active: bool, *, resend: bool = False + ) -> SentWebmention: + """Reopen delivery for a revision while preserving send history.""" + self.desired_revision = revision + self.active = active + self.attempt_count = 0 + self.next_retry_at = None + if resend: + self.processed_revision = None + self.status = None + self.failure_reason = None + self.endpoint = None + self.response_status = None + self.status_url = None + return self + @property def pending(self) -> bool: """ @@ -338,3 +385,16 @@ def uuid7_to_datetime(identifier: uuid.UUID) -> datetime: :return: Timestamp as a timezone-aware UTC datetime. """ return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) + + +def received_url_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair.""" + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def sent_target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID).""" + return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/net.py index 131cf55..01a9a80 100644 --- a/webmentions_ssg/public_transport.py +++ b/webmentions_ssg/net.py @@ -2,23 +2,31 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""HTTP transport that connects only to validated DNS answers. - -httpcore receives the original host for Host and TLS SNI/certificate checks, -but the socket is opened to an IP from a single, bounded DNS lookup. This -protects the receiver and sender, including redirects and advertised URLs. -""" +"""Public-only HTTP transport, URL checks, and bounded response reads.""" import ipaddress import socket import time from collections.abc import Iterable +from urllib.parse import urlsplit +import dns.exception +import dns.resolver import httpcore import httpx -from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation +from flask import current_app +from httpcore._backends.sync import SyncStream + +NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") +IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") + + +class AddressResolutionError(Exception): + """A hostname could not be resolved (potentially transient).""" -from .url_security import public_addresses + +class UnsafeAddressError(ValueError): + """A hostname or IP literal identifies a non-public network address.""" class NetworkDeadline: @@ -142,3 +150,114 @@ class PublicHTTPTransport(httpx.HTTPTransport): ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)), ) + + +def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client: + """Return the hardened HTTP client shared by receiver and sender.""" + headers = {"User-Agent": user_agent} + if accept is not None: + headers["Accept"] = accept + return httpx.Client( + headers=headers, + timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]), + follow_redirects=True, + max_redirects=current_app.config["MAX_REDIRECTS"], + trust_env=False, + transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]), + ) + + +def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """Reject non-public addresses, including IPv4 hidden in IPv6 forms. + + ``is_global`` alone does not reliably exclude IPv6 transition addresses + across Python versions or networks with NAT64/6to4 routing. + """ + if isinstance(address, ipaddress.IPv6Address): + # The well-known NAT64 prefix can translate an otherwise global IPv6 + # address into a private IPv4 destination. IPv4-compatible addresses + # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. + if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX: + return False + + # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 + # destination through ipaddress; validate that destination as well. + for embedded in (address.sixtofour, address.ipv4_mapped): + if embedded is not None and not embedded.is_global: + return False + + return address.is_global + + +def public_addresses( + hostname: str, *, lifetime: float | None = None +) -> tuple[str, ...]: + """Resolve *once* and return only checked, globally routable IP addresses. + + A DNS name with a mixture of public and private answers is rejected in its + entirety. IP literals are handled without DNS, including IPv6 literals. + The returned strings must be used for the actual TCP connection; checking + them and then connecting to ``hostname`` reintroduces DNS rebinding. + """ + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + + if literal is not None: + addresses = (literal,) + else: + try: + answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) + addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) + except (dns.exception.DNSException, ValueError) as exc: + raise AddressResolutionError( + f"Could not resolve hostname {hostname!r}" + ) from exc + + if not addresses: + raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") + + if any(not is_public_address(address) for address in addresses): + raise UnsafeAddressError(f"Non-public network address for {hostname!r}") + + return tuple(str(address) for address in addresses) + + +def is_http_url(url: str) -> bool: + """ + Check whether a URL is an absolute HTTP or HTTPS URL. + + :param url: URL to validate. + :return: Whether the URL uses HTTP or HTTPS and contains a hostname. + """ + try: + parsed = urlsplit(url) + except ValueError: + return False + + return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None + + +def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes: + """Read a streaming response, rejecting oversized declared or actual bodies.""" + if (content_length := response.headers.get("Content-Length")) is not None: + try: + declared_length = int(content_length) + except ValueError: + pass + else: + if declared_length > max_bytes: + raise ValueError(f"{label} document is too large") + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) + if len(body) > max_bytes: + raise ValueError(f"{label} document is too large") + return bytes(body) + + +def is_temporary_status(status: int) -> bool: + """Return whether an HTTP status is normally retryable.""" + return status in {408, 425, 429} or 500 <= status <= 599 diff --git a/webmentions_ssg/security.py b/webmentions_ssg/security.py deleted file mode 100644 index f4841a4..0000000 --- a/webmentions_ssg/security.py +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Validate persistent signing keys and administrator passwords.""" - -MIN_PASSWORD_LENGTH = 8 -MIN_SECRET_KEY_LENGTH = 32 - - -def validate_secret_key(value: object) -> None: - """Require a non-placeholder, sufficiently long deployment signing key.""" - if not isinstance(value, (str, bytes)): - raise TypeError( - "Configure a persistent SECRET_KEY of at least 32 random bytes " - "before starting." - ) - - # Flask accepts either text or bytes as its signing key. Reject common - # example/development values even when their lengths look sufficient. - key = value.encode("utf-8") if isinstance(value, str) else value - if len(key) < MIN_SECRET_KEY_LENGTH or not key.strip(): - raise RuntimeError( - "Configure a persistent SECRET_KEY of at least 32 random bytes " - "before starting." - ) - - normalized = key.strip().lower() - if normalized.startswith( - (b"replace_with", b"changeme", b"change_me", b"your_", b"insecure_") - ): - raise RuntimeError( - "SECRET_KEY is an example/development placeholder; " - "generate and configure a real random key." - ) - - -def validate_password(password: str) -> None: - """Validate newly chosen passwords without arbitrary composition rules. - - Allow Unicode, whitespace-only passwords, and arbitrarily long passphrases, - but reject empty passwords and control characters. - """ - if not password: - raise ValueError("Password must not be empty.") - if len(password) < MIN_PASSWORD_LENGTH: - raise ValueError( - f"Password must be at least {MIN_PASSWORD_LENGTH} characters long." - ) - if any(ord(char) < 32 or ord(char) == 127 for char in password): - raise ValueError("Password must not contain control characters.") diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index fb1d46d..bf7189f 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,7 +21,7 @@ def create_consumer() -> Huey: """ app = create_app() - from . import notifications, receiver, retention, scanner, sender # noqa: F401 + from . import notifications, receiver, retention, scanner schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) HUEY.periodic_task(schedule)(scanner.scan_sources) diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py new file mode 100644 index 0000000..98f3fd5 --- /dev/null +++ b/webmentions_ssg/tasks/discovery.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Discover a target Webmention endpoint and submit a POST.""" + +import re +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup +from flask import current_app + +from ..net import is_http_url, is_temporary_status, read_limited_body + +LINK_SPLIT = re.compile(r",\s*(?=<)") +DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} + + +class SenderError(Exception): + """A Webmention cannot be sent.""" + + +class TemporarySenderError(SenderError): + """A Webmention send failed for a potentially temporary reason.""" + + +class PermanentSenderError(SenderError): + """A Webmention send failed permanently for this source revision.""" + + +def resolve_endpoint(response: httpx.Response, href: str) -> str: + """ + Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :return: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or + HTTPS URL. + """ + try: + endpoint = urljoin(str(response.url), href.strip()) + except ValueError as exc: + raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc + + if not is_http_url(endpoint): + raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") + + return endpoint + + +def parse_link_value(value: str) -> tuple[str, set[str]] | None: + """ + Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :return: Link target and relation names, or ``None`` if the value is invalid. + """ + value = value.strip() + + if not value.startswith("<"): + return None + + href, separator, parameters = value[1:].partition(">") + + if not separator: + return None + + relations: set[str] = set() + + for parameter in parameters.split(";"): + name, separator, value = parameter.partition("=") + + if separator and name.strip().lower() == "rel": + relations.update( + relation.lower() for relation in value.strip().strip("\"'").split() + ) + + return href, relations + + +def endpoint_from_headers(response: httpx.Response) -> str | None: + """ + Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + for header in response.headers.get_list("Link"): + for value in LINK_SPLIT.split(header): + if (link := parse_link_value(value)) is None: + continue + + href, relations = link + + if "webmention" in relations: + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: + """ + Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. + """ + document = BeautifulSoup(body, "html5lib") + + for element in document.find_all(["link", "a"], href=True): + relations = element.get("rel") + + if isinstance(relations, str): + relations = relations.split() + + if not relations: + continue + + if not any( + isinstance(relation, str) and relation.lower() == "webmention" + for relation in relations + ): + continue + + href = element.get("href") + + if not isinstance(href, str): + continue + + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue + + return None + + +def read_target_body(response: httpx.Response) -> bytes: + """Return a bounded target body, using the shared HTTP size guard.""" + try: + return read_limited_body( + response, current_app.config["MAX_TARGET_BYTES"], label="Target" + ) + except ValueError as exc: + raise PermanentSenderError(str(exc)) from exc + + +def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: + """ + Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises TemporarySenderError: If the target returns a temporary HTTP failure. + :raises PermanentSenderError: If GET returns a permanent HTTP failure. + """ + try: + head_response = client.head(target, headers=DISCOVERY_HEADERS) + except httpx.RequestError, TemporarySenderError: + # HEAD is only an optimization. Servers and intermediaries may reject + # or drop it even when fetching the document with GET would succeed. + pass + else: + if endpoint := endpoint_from_headers(head_response): + return endpoint + + with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: + if endpoint := endpoint_from_headers(response): + return endpoint + + if is_temporary_status(response.status_code): + raise TemporarySenderError(f"Target returned HTTP {response.status_code}") + + if not response.is_success: + raise PermanentSenderError(f"Target returned HTTP {response.status_code}") + + media_type = ( + response.headers.get("Content-Type", "").partition(";")[0].strip().lower() + ) + + if media_type not in {"text/html", "application/xhtml+xml"}: + return None + + return endpoint_from_html(response, read_target_body(response)) + + +def post_webmention( + client: httpx.Client, *, endpoint: str, source: str, target: str +) -> tuple[int, str | None]: + """ + Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :return: HTTP response status code and optional status URL. + """ + with client.stream( + "POST", endpoint, data={"source": source, "target": target} + ) as response: + status_url = None + + if response.status_code == 201 and ( + location := response.headers.get("Location") + ): + try: + candidate = urljoin(str(response.url), location) + except ValueError: + # A malformed Location must not fail an otherwise successful send. + pass + else: + if is_http_url(candidate): + status_url = candidate + + return (response.status_code, status_url) diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index df6cb21..9f53f3b 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -93,54 +93,37 @@ class Huey: ) return self._huey + def _in_app_context(self, func: Callable[P, R]) -> Callable[P, R]: + """Return a task wrapper that executes with the configured Flask app.""" + + @wraps(func) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + if self.app is None: + raise RuntimeError("Flask app is not available.") + with self.app.app_context(): + return func(*args, **kwargs) + + return wrapped + def task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's task decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's task decorator. - :return: Decorator that registers the wrapped function as a Huey task. - """ + """Return a Huey task decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.task(*task_args, **task_kwargs)(wrapper) + return self.huey.task(*task_args, **task_kwargs)(self._in_app_context(func)) return decorator def periodic_task( self, *task_args: Any, **task_kwargs: Any ) -> Callable[[Callable[P, object]], TaskWrapper]: - """ - Create a periodic Huey task that runs within the Flask application context. - - :param task_args: Positional arguments forwarded to Huey's periodic task - decorator. - :param task_kwargs: Keyword arguments forwarded to Huey's periodic task - decorator. - :return: Decorator that registers the wrapped function as a periodic Huey - task. - """ + """Return a periodic Huey decorator with Flask application context.""" def decorator(func: Callable[P, R]) -> TaskWrapper: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if self.app is None: - raise RuntimeError("Flask app is not available.") - - with self.app.app_context(): - return func(*args, **kwargs) - - return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper) + return self.huey.periodic_task(*task_args, **task_kwargs)( + self._in_app_context(func) + ) return decorator diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py index 473a544..7e51358 100644 --- a/webmentions_ssg/tasks/json_serializer.py +++ b/webmentions_ssg/tasks/json_serializer.py @@ -11,7 +11,7 @@ from typing import Any from huey.registry import Message from huey.serializer import Serializer -from huey.utils import Error, SKIPPED +from huey.utils import SKIPPED, Error class JSONSerializer(Serializer): @@ -39,66 +39,87 @@ class JSONSerializer(Serializer): @classmethod def _encode(cls, value: Any) -> Any: - if value is None or type(value) in {str, int, float, bool}: - return value - if isinstance(value, uuid.UUID): - return {"type": "uuid", "value": str(value)} - if isinstance(value, datetime): - return {"type": "datetime", "value": value.isoformat()} - if isinstance(value, Message): - return {"type": "message", "value": [cls._encode(v) for v in value]} - if isinstance(value, Error): - return {"type": "error", "value": cls._encode(value.metadata)} - if value is SKIPPED: - return {"type": "skipped", "value": None} - if isinstance(value, tuple): - return {"type": "tuple", "value": [cls._encode(v) for v in value]} - if isinstance(value, list): - return {"type": "list", "value": [cls._encode(v) for v in value]} - if isinstance(value, dict) and all(type(k) is str for k in value): - return { - "type": "dict", - "value": [[k, cls._encode(v)] for k, v in value.items()], - } - raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") + match value: + case _ if value is None or type(value) in {str, int, float, bool}: + return value + + case uuid.UUID(): + return {"type": "uuid", "value": str(value)} + + case datetime(): + return {"type": "datetime", "value": value.isoformat()} + + case Message(): + return {"type": "message", "value": [cls._encode(v) for v in value]} + + case Error(): + return {"type": "error", "value": cls._encode(value.metadata)} + + case _ if value is SKIPPED: + return {"type": "skipped", "value": None} + + case tuple(): + return {"type": "tuple", "value": [cls._encode(v) for v in value]} + + case list(): + return {"type": "list", "value": [cls._encode(v) for v in value]} + + case dict() if all(isinstance(k, str) for k in value): + return { + "type": "dict", + "value": [[k, cls._encode(v)] for k, v in value.items()], + } + + case _: + raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") @classmethod def _decode(cls, value: Any) -> Any: if value is None or type(value) in {str, int, float, bool}: return value + if not isinstance(value, dict) or set(value) != {"type", "value"}: raise ValueError("Invalid Huey JSON data") - kind, payload = value["type"], value["value"] - if kind == "uuid" and isinstance(payload, str): - return uuid.UUID(payload) - if kind == "datetime" and isinstance(payload, str): - return datetime.fromisoformat(payload) - if kind == "message" and isinstance(payload, list): - if len(payload) != len(Message._fields): - raise ValueError("Invalid Huey JSON message field count") - return Message(*(cls._decode(v) for v in payload)) - if kind == "error": - metadata = cls._decode(payload) - if not isinstance(metadata, dict): - raise ValueError("Invalid Huey JSON error metadata") - return Error(metadata) - if kind == "skipped" and payload is None: - return SKIPPED - if kind in {"tuple", "list"} and isinstance(payload, list): - items = [cls._decode(v) for v in payload] - return tuple(items) if kind == "tuple" else items - if kind == "dict" and isinstance(payload, list): - result: dict[str, Any] = {} - for pair in payload: - if ( - not isinstance(pair, list) - or len(pair) != 2 - or type(pair[0]) is not str - ): - raise ValueError("Invalid Huey JSON dictionary entry") - if pair[0] in result: - raise ValueError("Duplicate Huey JSON dictionary key") - result[pair[0]] = cls._decode(pair[1]) - return result - raise ValueError("Unsupported Huey JSON type") + match value["type"], value["value"]: + case "uuid", str() as payload: + return uuid.UUID(payload) + + case "datetime", str() as payload: + return datetime.fromisoformat(payload) + + case "message", list() as payload: + if len(payload) != len(Message._fields): + raise ValueError("Invalid Huey JSON message field count") + return Message(*(cls._decode(v) for v in payload)) + + case "error", payload: + metadata = cls._decode(payload) + if not isinstance(metadata, dict): + raise TypeError("Invalid Huey JSON error metadata") + return Error(metadata) + + case "skipped", None: + return SKIPPED + + case ("tuple" | "list") as kind, list() as payload: + items = [cls._decode(v) for v in payload] + return tuple(items) if kind == "tuple" else items + + case "dict", list() as payload: + result: dict[str, Any] = {} + + for pair in payload: + match pair: + case list([str() as key, encoded]): + if key in result: + raise ValueError("Duplicate Huey JSON dictionary key") + result[key] = cls._decode(encoded) + + case _: + raise ValueError("Invalid Huey JSON dictionary entry") + + return result + + case _: + raise ValueError("Unsupported Huey JSON type") diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py new file mode 100644 index 0000000..4e5a5ca --- /dev/null +++ b/webmentions_ssg/tasks/mentions.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure HTML and text matching for received Webmention targets.""" + +from urllib.parse import urljoin + +import rfc3987 +from bs4 import BeautifulSoup + +IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") +TRAILING_IRI_PUNCTUATION = ".,;:!?" + +HTML_URL_ATTRIBUTES = { + "href": {"a", "area", "link"}, + "src": { + "audio", + "embed", + "iframe", + "img", + 'input[type="image" i]', + "script", + "audio source", + "video source", + "track", + "video", + }, + "cite": {"blockquote", "del", "ins", "q"}, +} + + +def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: + """ + Check whether an HTML document mentions the target URL. + + URL references are resolved against the document's ``base`` element when + present, or against the source URL otherwise. + + :param body: HTML document body. + :param source_url: URL from which the document was retrieved. + :param target_url: Exact URL to look for. + :return: Whether a supported HTML URL attribute references the target URL. + """ + + document = BeautifulSoup(body, "html.parser") + base_url = source_url + + if (base_element := document.select_one("base[href]")) is not None and isinstance( + base_href := base_element.get("href"), str + ): + base_url = urljoin(source_url, base_href.strip()) + + for attribute, selectors in HTML_URL_ATTRIBUTES.items(): + selector = ", ".join( + [f"{selector_string}[{attribute}]" for selector_string in selectors] + ) + + for element in document.select(selector): + if not isinstance(reference := element.get(attribute), str): + continue + + if urljoin(base_url, reference.strip()) == target_url: + return True + + return False + + +def text_mentions_target(body: str, target_url: str) -> bool: + """ + Check whether plain text contains the target IRI, ignoring trailing prose + punctuation but not extra URL path or query characters. + + :param body: Plain-text document body. + :param target_url: Exact IRI to look for. + :return: Whether the target IRI occurs in the document. + """ + for match in IRI_PATTERN.finditer(body): + candidate = match.group() + + # A URL in prose can be followed by punctuation that the IRI pattern + # consumes as part of its match. Only remove trailing punctuation, + # leaving longer URLs (e.g. ``target/more``) unmatched. + if not candidate.startswith(target_url): + continue + + while candidate != target_url: + if candidate[-1] in TRAILING_IRI_PUNCTUATION: + candidate = candidate[:-1] + elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): + # Keep balanced parentheses that belong to the URL itself. + candidate = candidate[:-1] + else: + break + + if candidate == target_url: + return True + + return False diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index 475fad1..889030c 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -10,10 +10,10 @@ import sqlalchemy as sa from flask import current_app from flask_mail import Message -from .. import DATABASE as db -from .. import HUEY as huey -from .. import MAIL as mail -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..extensions import MAIL as mail +from ..models import ReceivedStatus, ReceivedWebmention @huey.lock_task("send-webmention-digest") @@ -27,7 +27,7 @@ def send_webmention_digest() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == "verified", + ReceivedWebmention.status == ReceivedStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) @@ -53,7 +53,6 @@ def send_webmention_digest() -> None: ), ) - # If SMTP fails, the records remain pending for the next digest. mail.send(message) db.session.execute( diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index acd66a1..bed923c 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -2,27 +2,27 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Fetch and verify incoming Webmentions against their source pages.""" +"""Fetch and verify incoming Webmentions with transactional state changes.""" import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx -import rfc3987 import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey -from ..models import ReceivedWebmention -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey +from ..models import ReceivedStatus, ReceivedWebmention +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_temporary_status, + read_limited_body, +) +from .mentions import html_mentions_target, text_mentions_target class VerificationError(Exception): @@ -37,95 +37,7 @@ class TemporaryFetchError(Exception): """Fetching the source may succeed when retried later.""" -IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") -TRAILING_IRI_PUNCTUATION = ".,;:!?" - - -HTML_URL_ATTRIBUTES = { - "href": {"a", "area", "link"}, - "src": { - "audio", - "embed", - "iframe", - "img", - 'input[type="image" i]', - "script", - "audio source", - "video source", - "track", - "video", - }, - "cite": {"blockquote", "del", "ins", "q"}, -} - - -def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """ - Check whether an HTML document mentions the target URL. - - URL references are resolved against the document's ``base`` element when - present, or against the source URL otherwise. - - :param body: HTML document body. - :param source_url: URL from which the document was retrieved. - :param target_url: Exact URL to look for. - :return: Whether a supported HTML URL attribute references the target URL. - """ - - document = BeautifulSoup(body, "html.parser") - base_url = source_url - - if (base_element := document.select_one("base[href]")) is not None and isinstance( - base_href := base_element.get("href"), str - ): - base_url = urljoin(source_url, base_href.strip()) - - for attribute, selectors in HTML_URL_ATTRIBUTES.items(): - selector = ", ".join( - [f"{selector_string}[{attribute}]" for selector_string in selectors] - ) - - for element in document.select(selector): - if not isinstance(reference := element.get(attribute), str): - continue - - if urljoin(base_url, reference.strip()) == target_url: - return True - - return False - - -def text_mentions_target(body: str, target_url: str) -> bool: - """ - Check whether plain text contains the target IRI, ignoring trailing prose - punctuation but not extra URL path or query characters. - - :param body: Plain-text document body. - :param target_url: Exact IRI to look for. - :return: Whether the target IRI occurs in the document. - """ - for match in IRI_PATTERN.finditer(body): - candidate = match.group() - - # A URL in prose can be followed by punctuation that the IRI pattern - # consumes as part of its match. Only remove trailing punctuation, - # leaving longer URLs (e.g. ``target/more``) unmatched. - if not candidate.startswith(target_url): - continue - - while candidate != target_url: - if candidate[-1] in TRAILING_IRI_PUNCTUATION: - candidate = candidate[:-1] - elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("): - # Keep balanced parentheses that belong to the URL itself. - candidate = candidate[:-1] - else: - break - - if candidate == target_url: - return True - - return False +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: @@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ try: - return _fetch_source(source_url) + with ( + build_client( + f"{APP_NAME}/{VERSION} ReceivedWebmention", + accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if is_temporary_status(status): + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") + + try: + return response, read_limited_body( + response, current_app.config["MAX_SOURCE_BYTES"], label="Source" + ) + except ValueError as exc: + raise VerificationError(str(exc)) from exc + except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc -def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - with ( - httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client, - client.stream("GET", source_url) as response, - ): - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") - - max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass - - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_source_bytes: - raise VerificationError("Source document is too large") - - return response, bytes(body) - - def source_mentions_target(source_url: str, target_url: str) -> bool: """ Fetch a source and check whether it mentions the target URL. @@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) return - # Failed tasks may be retried by Huey. Completed and already-running tasks - # should not start another concurrent verification. - if webmention.status not in {"received", "failed"}: + if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: return - # A reaper-exhausted row must not be revived by a late Huey retry. + if ( - webmention.status == "failed" + webmention.status == ReceivedStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return @@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ReceivedWebmention.status == previous_status, ReceivedWebmention.updated_at == previous_updated_at, ) - .values(status="verifying", failure_reason=None, updated_at=started_at) + .values( + status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at + ) ) + if claimed.rowcount != 1: db.session.rollback() return - db.session.commit() - def finish(status: str, failure_reason: str | None) -> None: - # A timeout or a newer verification may have superseded this attempt. - # Do not let a late worker overwrite its state. - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == "verifying", - ReceivedWebmention.updated_at == started_at, - ) - .values( - status=status, - failure_reason=failure_reason, - updated_at=datetime.now(UTC), - ) - ) - db.session.commit() + db.session.commit() + status = None + failure_reason = None try: mentions_target = source_mentions_target(source, target) except SourceGoneError as exc: - finish("deleted", str(exc)) + status = ReceivedStatus.DELETED + failure_reason = str(exc) except VerificationError as exc: - finish("failed", str(exc)) + status = ReceivedStatus.FAILED + failure_reason = str(exc) except (TemporaryFetchError, httpx.RequestError) as exc: - finish("failed", str(exc) or "Source could not be fetched") + status = ReceivedStatus.FAILED + failure_reason = str(exc) or "Source could not be fetched" raise except Exception as exc: # InvalidURL is not a RequestError; other unexpected source errors must @@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", webmention_uuid ) - finish("failed", str(exc) or type(exc).__name__) + + status = ReceivedStatus.FAILED + failure_reason = str(exc) or type(exc).__name__ else: if mentions_target: - finish("verified", None) + status = ReceivedStatus.VERIFIED else: - finish("deleted", "Source does not mention target") + status = ReceivedStatus.DELETED + failure_reason = "Source does not mention target" + finally: + if status is not None: + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.updated_at == started_at, + ) + .values( + status=status, + failure_reason=failure_reason, + updated_at=datetime.now(UTC), + ) + ) + db.session.commit() def recover_stale_webmentions() -> int: @@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int: """ now = datetime.now(UTC) cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + stale = db.session.execute( sa.select( ReceivedWebmention.uuid, @@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.updated_at, ReceivedWebmention.recovery_attempts, ).where( - ReceivedWebmention.status.in_(("verifying", "received")), + ReceivedWebmention.status.in_( + (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + ), ReceivedWebmention.updated_at <= cutoff, ) ).all() @@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int: recovered = 0 exhausted = 0 max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( @@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.recovery_attempts == recovery_attempts, ) .values( - status="failed" if at_limit else "received", - failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, - updated_at=now, - recovery_attempts=( - recovery_attempts if at_limit else recovery_attempts + 1 - ), + **( + { + "status": ReceivedStatus.FAILED, + "failure_reason": RECOVERY_EXHAUSTED_REASON, + "updated_at": now, + "recovery_attempts": recovery_attempts, + } + if at_limit + else ReceivedWebmention.verification_reset_values( + at=now, recoveries=recovery_attempts + 1 + ) + ) ) ) + if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: exhausted += 1 continue diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 727b186..bf79512 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,8 +11,8 @@ import sqlalchemy as sa from flask import current_app from sqlalchemy.engine import CursorResult -from .. import DATABASE as db -from ..models import ReceivedWebmention +from ..extensions import DATABASE as db +from ..models import ReceivedStatus, ReceivedWebmention def purge_old_received_webmentions() -> int: @@ -25,17 +25,22 @@ def purge_old_received_webmentions() -> int: cutoff = datetime.now(UTC) - timedelta( days=current_app.config["RECEIVED_RETENTION_DAYS"] ) + result = cast( CursorResult[Any], db.session.execute( sa.delete(ReceivedWebmention).where( - ReceivedWebmention.status.in_(("failed", "deleted")), + ReceivedWebmention.status.in_( + (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ), ReceivedWebmention.updated_at < cutoff, ) ), ) db.session.commit() + removed = result.rowcount if removed: current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + return removed diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 666eba7..b2d6def 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -2,404 +2,22 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Scan generated site pages to track outgoing Webmention targets.""" +"""Reconcile scanned static-site sources with the database.""" -import json -from collections.abc import Iterator -from dataclasses import dataclass from datetime import UTC, datetime -from fnmatch import fnmatchcase -from itertools import chain from pathlib import Path -from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit -import mf2py import sqlalchemy as sa -import xxhash -from bs4 import BeautifulSoup, NavigableString, Tag -from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload -from .. import DATABASE as db -from .. import HUEY as huey -from ..config import normalize_ignored_hostnames +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, Source -from ..url_security import is_http_url +from ..net import is_http_url from .sender import send_webmention - -REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") - -CONTENT_BLOCK_TAGS = frozenset( - { - "address", - "article", - "aside", - "blockquote", - "br", - "dd", - "div", - "dl", - "dt", - "figcaption", - "figure", - "footer", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "header", - "hr", - "li", - "main", - "ol", - "p", - "pre", - "section", - "table", - "td", - "th", - "tr", - "ul", - } -) - - -class SourceScanError(Exception): - """A source document cannot safely be processed.""" - - -@dataclass(frozen=True) -class ScannedSource: - path: str - url: str - content_hash: str - targets: frozenset[str] - - -def source_url_for_path(relative_path: Path, base_url: str) -> str: - """ - Derive the public source URL from a relative filesystem path. - - :param relative_path: Path of the source document relative to the source root. - :param base_url: Base URL under which source documents are published. - :return: Public URL corresponding to the source document. - """ - directory = relative_path.parent.as_posix() - - return urljoin(base_url, f"{quote(directory, safe='/')}/") - - -def canonical_url( - document: BeautifulSoup, *, relative_path: Path, base_url: str | None -) -> str | None: - """ - Return the canonical URL declared by a source document. - - Relative canonical URLs are resolved against the configured source base URL. - - :param document: Parsed HTML source document. - :param relative_path: Path of the source document relative to the source root. - :param base_url: Configured source base URL, if available. - :return: Canonical URL, or ``None`` if none is declared. - :raises SourceScanError: If the canonical URL is empty, cannot be resolved, - or does not resolve to an HTTP or HTTPS URL. - """ - if (link := document.select_one('link[rel~="canonical"][href]')) is not None: - href = link.get("href") - - if not isinstance(href, str) or not (href := href.strip()): - raise SourceScanError("Document contains an empty canonical URL") - - if is_http_url(href): - return href - - if base_url is None: - raise SourceScanError( - "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" - ) - - resolved = urljoin(source_url_for_path(relative_path, base_url), href) - - if not is_http_url(resolved): - raise SourceScanError( - f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" - ) - - return resolved - - -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """ - Yield URL values from a microformats property. - - :param entry: Parsed microformats entry. - :param property_name: Name of the property to inspect. - :return: Iterator over string values of the property. - """ - properties = entry.get("properties") - - if not isinstance(properties, dict): - return - - values = properties.get(property_name) - - if not isinstance(values, list): - return - - yield from (value for value in values if isinstance(value, str)) - - -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """ - Parse and validate an h-entry with mf2py. - - :param element: HTML element containing the h-entry. - :param base_url: Base URL used when parsing the microformats data. - :return: Parsed h-entry. - :raises SourceScanError: If parsing does not produce exactly one valid h-entry. - """ - parsed = mf2py.parse(doc=str(element), url=base_url) - - if not isinstance(parsed, dict): - raise SourceScanError("Microformats parser did not return a document object") - - items = parsed.get("items") - - if not isinstance(items, list) or len(items) != 1: - raise SourceScanError("Expected exactly one parsed h-entry") - - entry = items[0] - - if not isinstance(entry, dict): - raise SourceScanError("Parsed h-entry is not an object") - - types = entry.get("type") - - if not isinstance(types, list) or "h-entry" not in types: - raise SourceScanError("Parsed microformats item is not an h-entry") - - return entry - - -def primary_entry( - document: BeautifulSoup, source_url: str -) -> tuple[Tag, dict[str, Any]]: - """ - Return and validate the primary h-entry of a source document. - - The document must contain exactly one h-entry whose ``u-url`` matches the - source URL. - - :param document: Parsed HTML source document. - :param source_url: Public URL of the source document. - :return: h-entry element and its parsed microformats representation. - :raises SourceScanError: If the document does not contain exactly one h-entry - or its ``u-url`` does not match the source URL. - """ - entries = document.find_all(class_="h-entry") - - if len(entries) != 1: - raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") - - entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) - - urls = tuple(property_urls(mf2_entry, "url")) - - if source_url not in urls: - raise SourceScanError( - f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" - ) - - return entry_element, mf2_entry - - -def content_element(entry: Tag) -> Tag: - """ - Return the e-content element of an h-entry. - - :param entry: HTML element containing the h-entry. - :return: The h-entry's e-content element. - :raises SourceScanError: If the h-entry does not contain exactly one - e-content element. - """ - contents = entry.find_all(class_="e-content") - - if len(contents) != 1: - raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") - - return contents[0] - - -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """ - Resolve and validate a possible Webmention target URL. - - Empty values, non-HTTP URLs, and URLs referring to the source itself are - discarded. - - :param value: URL reference to normalize. - :param base_url: Base URL against which relative references are resolved. - :param source_url: URL of the source document. - :return: Normalized target URL, or ``None`` if the value is not a valid - Webmention target. - """ - value = value.strip() - - if not value: - return None - - target = urljoin(base_url, value) - - if not is_http_url(target): - return None - - if urldefrag(target)[0] == urldefrag(source_url)[0]: - return None - - return target - - -def iter_targets( - content: Tag, - mf2_entry: dict[str, Any], - *, - base_url: str, - source_url: str, - ignored_hostnames: tuple[str, ...] = (), -) -> Iterator[str]: - """ - Yield outgoing Webmention targets from an h-entry. - - Targets are collected from links in the entry content and supported - microformats reaction properties. Invalid, self-referencing, and ignored - targets are excluded. - - :param content: e-content element of the h-entry. - :param mf2_entry: Parsed microformats representation of the h-entry. - :param base_url: Base URL against which relative targets are resolved. - :param source_url: URL of the source document. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Iterator over outgoing Webmention target URLs. - """ - hrefs = ( - href - for element in content.find_all("a", href=True) - if isinstance(href := element.get("href"), str) - ) - - reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES - ) - - for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) - if target is not None: - hostname = urlsplit(target).hostname or "" - if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): - yield target - - -def normalized_content(content: Tag) -> str: - """Extract stable reader-facing content without generated HTML attributes.""" - - def extract(node: PageElement) -> str: - if isinstance(node, NavigableString): - return str(node) - if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: - return "" - - # A relative-time label may change on every site build even though its - # machine-readable date stays the same. - if node.name == "time" and isinstance(value := node.get("datetime"), str): - return value.strip() - - # Image-only posts also have meaningful content, despite having no text. - if node.name == "img": - return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " - - text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text - - return " ".join(extract(content).split()) - - -def scan_source_file( - path: Path, - *, - root: Path, - base_url: str | None, - ignored_hostnames: tuple[str, ...] = (), -) -> ScannedSource: - """ - Scan one generated source document for outgoing Webmentions. - - :param path: Path of the generated HTML document. - :param root: Root directory containing generated source documents. - :param base_url: Configured source base URL, if available. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Scanned source metadata and discovered targets. - :raises SourceScanError: If the document cannot be safely interpreted as a - Webmention source. - :raises OSError: If the source document cannot be read. - """ - # Use the path as published for its URL, but read only files whose - # resolved target stays inside the configured source directory. Resolving - # before reading also protects against symlinked index.html files. - resolved_root = root.resolve() - resolved_path = path.resolve() - if not resolved_path.is_relative_to(resolved_root): - raise SourceScanError(f"Source document escapes source directory: {path}") - - relative_path = path.relative_to(root) - document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) - - if source_url is None: - if base_url is None: - raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" - ) - - source_url = source_url_for_path(relative_path, base_url) - - entry_element, mf2_entry = primary_entry(document, source_url) - - content = content_element(entry_element) - - targets = frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ) - - # Fingerprint the meaningful post text and its outgoing targets rather than - # the full h-entry markup. Relative timestamps, classes, and other volatile - # metadata outside e-content must not cause repeat Webmentions. - stable_content = normalized_content(content) - fingerprint = json.dumps( - {"content": stable_content, "targets": sorted(targets)}, - ensure_ascii=False, - separators=(",", ":"), - ) - - return ScannedSource( - path=relative_path.as_posix(), - url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), - targets=targets, - ) +from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: @@ -466,10 +84,9 @@ def update_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.active = webmention.target in scanned.targets - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen( + source.revision, webmention.target in scanned.targets + ) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -523,10 +140,6 @@ def scan_sources_task() -> None: raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" - ignored_hostnames = normalize_ignored_hostnames( - current_app.config["IGNORED_HOSTNAMES"] - ) - sources_by_path = { source.path: source for source in db.session.scalars( @@ -555,7 +168,7 @@ def scan_sources_task() -> None: index_path, root=root, base_url=base_url, - ignored_hostnames=ignored_hostnames, + ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -592,10 +205,6 @@ def scan_sources_task() -> None: except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - # A static-site generator can temporarily empty its output directory while - # rebuilding. With no successfully scanned pages, absence is not evidence - # of deletion; do not generate removals or send pending Webmentions that - # might be verified while the site is unavailable. if scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " @@ -612,10 +221,7 @@ def scan_sources_task() -> None: source.deleted_at = scan_time for webmention in source.sent_webmentions: - webmention.active = False - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 9330247..3c4c717 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -2,272 +2,33 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Discover endpoints and deliver outgoing Webmentions.""" +"""Claim, send, and persist outgoing Webmention attempts.""" -import re import uuid from datetime import UTC, datetime, timedelta -from urllib.parse import urljoin import httpx import sqlalchemy as sa -from bs4 import BeautifulSoup from flask import current_app from huey.exceptions import TaskLockedException from .. import APP_NAME, VERSION -from .. import DATABASE as db -from .. import HUEY as huey +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, SentWebmentionStatus -from ..public_transport import PublicHTTPTransport -from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url - -LINK_SPLIT = re.compile(r",\s*(?=<)") - -DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} - - -class SenderError(Exception): - """A Webmention cannot be sent.""" - - -class TemporarySenderError(SenderError): - """A Webmention send failed for a potentially temporary reason.""" - - -class PermanentSenderError(SenderError): - """A Webmention send failed permanently for this source revision.""" - - -def temporary_http_status(status: int) -> bool: - """ - Return whether an HTTP status indicates a temporary failure. - - :param status: HTTP response status code. - :return: Whether the request should be retried. - """ - return status in {408, 425, 429} or 500 <= status <= 599 - - -def resolve_endpoint(response: httpx.Response, href: str) -> str: - """ - Resolve and validate a discovered Webmention endpoint. - - :param response: Response from which the endpoint was discovered. - :param href: Endpoint reference to resolve. - :return: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or - HTTPS URL. - """ - try: - endpoint = urljoin(str(response.url), href.strip()) - except ValueError as exc: - raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc - - if not is_http_url(endpoint): - raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") - - return endpoint - - -def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """ - Parse a Link header value into its target and relations. - - :param value: Link header value to parse. - :return: Link target and relation names, or ``None`` if the value is invalid. - """ - value = value.strip() - - if not value.startswith("<"): - return None - - href, separator, parameters = value[1:].partition(">") - - if not separator: - return None - - relations: set[str] = set() - - for parameter in parameters.split(";"): - name, separator, value = parameter.partition("=") - - if separator and name.strip().lower() == "rel": - relations.update( - relation.lower() for relation in value.strip().strip("\"'").split() - ) - - return href, relations - - -def endpoint_from_headers(response: httpx.Response) -> str | None: - """ - Find a Webmention endpoint in the response Link headers. - - :param response: HTTP response whose headers should be inspected. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - for header in response.headers.get_list("Link"): - for value in LINK_SPLIT.split(header): - if (link := parse_link_value(value)) is None: - continue - - href, relations = link - - if "webmention" in relations: - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """ - Find a Webmention endpoint in an HTML document. - - ``link`` and ``a`` elements are inspected in document order. - - :param response: HTTP response from which the document was retrieved. - :param body: HTML document body. - :return: First valid advertised Webmention endpoint, or ``None`` if none - is found. Invalid candidates are ignored. - """ - document = BeautifulSoup(body, "html5lib") - - for element in document.find_all(["link", "a"], href=True): - relations = element.get("rel") - - if isinstance(relations, str): - relations = relations.split() - - if not relations: - continue - - if not any( - isinstance(relation, str) and relation.lower() == "webmention" - for relation in relations - ): - continue - - href = element.get("href") - - if not isinstance(href, str): - continue - - try: - return resolve_endpoint(response, href) - except PermanentSenderError: - continue - - return None - - -def read_target_body(response: httpx.Response) -> bytes: - """ - Read a target document up to the configured size limit. - - :param response: Streaming HTTP response to read. - :return: Response body. - :raises PermanentSenderError: If the document exceeds the configured maximum - size. - """ - max_bytes = current_app.config.get("MAX_TARGET_BYTES", 1_000_000) - - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_bytes: - raise PermanentSenderError("Target document is too large") - except ValueError: - pass - - body = bytearray() - - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) - - if len(body) > max_bytes: - raise PermanentSenderError("Target document is too large") - - return bytes(body) - - -def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """ - Discover the Webmention endpoint advertised by a target URL. - - Endpoint discovery first checks the response to a HEAD request and then falls - back to a GET request, inspecting both HTTP Link headers and supported HTML - documents. - - :param client: HTTP client to use for discovery requests. - :param target: Target URL whose Webmention endpoint should be discovered. - :return: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If GET returns a permanent HTTP failure. - """ - try: - head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError, TemporarySenderError: - # HEAD is only an optimization. Servers and intermediaries may reject - # or drop it even when fetching the document with GET would succeed. - pass - else: - if endpoint := endpoint_from_headers(head_response): - return endpoint - - with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := endpoint_from_headers(response): - return endpoint - - if temporary_http_status(response.status_code): - raise TemporarySenderError(f"Target returned HTTP {response.status_code}") - - if not response.is_success: - raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) - - if media_type not in {"text/html", "application/xhtml+xml"}: - return None - - return endpoint_from_html(response, read_target_body(response)) - - -def post_webmention( - client: httpx.Client, *, endpoint: str, source: str, target: str -) -> tuple[int, str | None]: - """ - Send a Webmention to a discovered endpoint. - - :param client: HTTP client to use for the request. - :param endpoint: Webmention endpoint URL. - :param source: Source URL of the Webmention. - :param target: Target URL of the Webmention. - :return: HTTP response status code and optional status URL. - """ - with client.stream( - "POST", endpoint, data={"source": source, "target": target} - ) as response: - status_url = None - - if response.status_code == 201 and ( - location := response.headers.get("Location") - ): - try: - candidate = urljoin(str(response.url), location) - except ValueError: - # A malformed Location must not fail an otherwise successful send. - pass - else: - if is_http_url(candidate): - status_url = candidate - - return (response.status_code, status_url) +from ..net import ( + AddressResolutionError, + UnsafeAddressError, + build_client, + is_http_url, + is_temporary_status, +) +from .discovery import ( + PermanentSenderError, + TemporarySenderError, + discover_webmention_endpoint, + post_webmention, +) def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @@ -342,7 +103,6 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: now = datetime.now(UTC) deadline = webmention.next_retry_at - # SQLite returns timezone-naive datetimes even for timezone-aware columns. if deadline is not None: aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) if aware_deadline > now: @@ -420,48 +180,31 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: endpoint: str | None = None response_status: int | None = None + status_url: str | None = None + reason: str | None = None + terminal = True + temporary_failure = False try: if not is_http_url(target): raise PermanentSenderError(f"Invalid target URL: {target!r}") - with httpx.Client( - headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")}, - timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)), - follow_redirects=True, - max_redirects=current_app.config.get("MAX_REDIRECTS", 20), - trust_env=False, - transport=PublicHTTPTransport( - current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) - ), - ) as client: + with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.UNSUPPORTED - webmention.failure_reason = "No Webmention endpoint discovered" - webmention.endpoint = None - webmention.response_status = None - - db.session.commit() - return - - (response_status, status_url) = post_webmention( - client, endpoint=endpoint, source=source, target=target - ) - - if not (200 <= response_status <= 299): - message = f"Webmention endpoint returned HTTP {response_status}" - - if temporary_http_status(response_status): - raise TemporarySenderError(message) - - raise PermanentSenderError(message) + outcome = SentWebmentionStatus.UNSUPPORTED + reason = "No Webmention endpoint discovered" + else: + response_status, status_url = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + if not 200 <= response_status <= 299: + message = f"Webmention endpoint returned HTTP {response_status}" + if is_temporary_status(response_status): + raise TemporarySenderError(message) + raise PermanentSenderError(message) + outcome = SentWebmentionStatus.SENT except ( PermanentSenderError, @@ -472,61 +215,47 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: ValueError, UnicodeError, ) as exc: - # Malformed URLs and local request/protocol errors cannot be repaired - # by retrying this source revision. InvalidURL is not a RequestError; - # protocol errors are, so handle them before the retryable branch. - if not attempt_is_current(webmention, revision): - return - - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or type(exc).__name__ - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - return - + # Invalid URLs and local protocol errors cannot improve with retries. + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or type(exc).__name__ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: - if not attempt_is_current(webmention, revision): - return - - webmention.status = SentWebmentionStatus.FAILED - webmention.failure_reason = str(exc) or "Webmention request failed" - if attempt >= max_attempts: - webmention.processed_revision = revision - webmention.next_retry_at = None - current_app.logger.warning( - "Webmention %s exhausted %d send attempts: %s", - webmention_uuid, - attempt, - webmention.failure_reason, - ) - webmention.endpoint = endpoint - webmention.response_status = response_status - - db.session.commit() - - # No Huey retries: only the scanner may requeue after next_retry_at. - return + outcome = SentWebmentionStatus.FAILED + reason = str(exc) or "Webmention request failed" + terminal = attempt >= max_attempts + temporary_failure = True + # A scanner may have reopened or superseded the revision during I/O. + # Refresh only once, and persist one outcome atomically with its metadata. if not attempt_is_current(webmention, revision): return - webmention.processed_revision = revision - webmention.next_retry_at = None - webmention.sent_revision = revision - - webmention.status = SentWebmentionStatus.SENT - webmention.failure_reason = None - + webmention.status = outcome + webmention.failure_reason = reason webmention.endpoint = endpoint webmention.response_status = response_status - webmention.status_url = ( - status_url if status_url is not None and is_http_url(status_url) else None - ) - webmention.last_sent_at = datetime.now(UTC) + if terminal: + webmention.processed_revision = revision + webmention.next_retry_at = None + + if outcome == SentWebmentionStatus.SENT: + webmention.sent_revision = revision + webmention.status_url = ( + status_url if status_url is not None and is_http_url(status_url) else None + ) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() + + if temporary_failure and not terminal: + # Huey does not retry: scanner requeues after next_retry_at. + current_app.logger.info( + "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt + ) + elif temporary_failure and terminal: + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + reason, + ) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py new file mode 100644 index 0000000..cb15ab6 --- /dev/null +++ b/webmentions_ssg/tasks/source_parser.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Extract canonical source URLs, content fingerprints, and outgoing links.""" + +import json +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from fnmatch import fnmatchcase +from itertools import chain +from pathlib import Path +from typing import Any +from urllib.parse import quote, urldefrag, urljoin, urlsplit + +import mf2py +import xxhash +from bs4 import BeautifulSoup, Tag +from bs4.element import NavigableString, PageElement + +from ..net import is_http_url + +REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") + +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + + +class SourceScanError(Exception): + """A source document cannot safely be processed.""" + + +@dataclass(frozen=True) +class ScannedSource: + path: str + url: str + content_hash: str + targets: frozenset[str] + + +def source_url_for_path(relative_path: Path, base_url: str) -> str: + """ + Derive the public source URL from a relative filesystem path. + + :param relative_path: Path of the source document relative to the source root. + :param base_url: Base URL under which source documents are published. + :return: Public URL corresponding to the source document. + """ + directory = relative_path.parent.as_posix() + + return urljoin(base_url, f"{quote(directory, safe='/')}/") + + +def canonical_url( + document: BeautifulSoup, *, relative_path: Path, base_url: str | None +) -> str | None: + """ + Return the canonical URL declared by a source document. + + Relative canonical URLs are resolved against the configured source base URL. + + :param document: Parsed HTML source document. + :param relative_path: Path of the source document relative to the source root. + :param base_url: Configured source base URL, if available. + :return: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, + or does not resolve to an HTTP or HTTPS URL. + """ + if (link := document.select_one('link[rel~="canonical"][href]')) is not None: + href = link.get("href") + + if not isinstance(href, str) or not (href := href.strip()): + raise SourceScanError("Document contains an empty canonical URL") + + if is_http_url(href): + return href + + if base_url is None: + raise SourceScanError( + "Document contains a relative canonical URL, but " + "SOURCE_BASE_URL is not configured" + ) + + resolved = urljoin(source_url_for_path(relative_path, base_url), href) + + if not is_http_url(resolved): + raise SourceScanError( + f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" + ) + + return resolved + + +def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: + """ + Yield URL values from a microformats property. + + :param entry: Parsed microformats entry. + :param property_name: Name of the property to inspect. + :return: Iterator over string values of the property. + """ + properties = entry.get("properties") + + if not isinstance(properties, dict): + return + + values = properties.get(property_name) + + if not isinstance(values, list): + return + + yield from (value for value in values if isinstance(value, str)) + + +def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: + """ + Parse and validate an h-entry with mf2py. + + :param element: HTML element containing the h-entry. + :param base_url: Base URL used when parsing the microformats data. + :return: Parsed h-entry. + :raises SourceScanError: If parsing does not produce exactly one valid h-entry. + """ + parsed = mf2py.parse(doc=str(element), url=base_url) + + if not isinstance(parsed, dict): + raise SourceScanError("Microformats parser did not return a document object") + + items = parsed.get("items") + + if not isinstance(items, list) or len(items) != 1: + raise SourceScanError("Expected exactly one parsed h-entry") + + entry = items[0] + + if not isinstance(entry, dict): + raise SourceScanError("Parsed h-entry is not an object") + + types = entry.get("type") + + if not isinstance(types, list) or "h-entry" not in types: + raise SourceScanError("Parsed microformats item is not an h-entry") + + return entry + + +def primary_entry( + document: BeautifulSoup, source_url: str +) -> tuple[Tag, dict[str, Any]]: + """ + Return and validate the primary h-entry of a source document. + + The document must contain exactly one h-entry whose ``u-url`` matches the + source URL. + + :param document: Parsed HTML source document. + :param source_url: Public URL of the source document. + :return: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry + or its ``u-url`` does not match the source URL. + """ + entries = document.find_all(class_="h-entry") + + if len(entries) != 1: + raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") + + entry_element = entries[0] + mf2_entry = parse_entry(entry_element, source_url) + + urls = tuple(property_urls(mf2_entry, "url")) + + if source_url not in urls: + raise SourceScanError( + f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" + ) + + return entry_element, mf2_entry + + +def content_element(entry: Tag) -> Tag: + """ + Return the e-content element of an h-entry. + + :param entry: HTML element containing the h-entry. + :return: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one + e-content element. + """ + contents = entry.find_all(class_="e-content") + + if len(contents) != 1: + raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") + + return contents[0] + + +def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: + """ + Resolve and validate a possible Webmention target URL. + + Empty values, non-HTTP URLs, and URLs referring to the source itself are + discarded. + + :param value: URL reference to normalize. + :param base_url: Base URL against which relative references are resolved. + :param source_url: URL of the source document. + :return: Normalized target URL, or ``None`` if the value is not a valid + Webmention target. + """ + value = value.strip() + + if not value: + return None + + target = urljoin(base_url, value) + + if not is_http_url(target): + return None + + if urldefrag(target)[0] == urldefrag(source_url)[0]: + return None + + return target + + +def iter_targets( + content: Tag, + mf2_entry: dict[str, Any], + *, + base_url: str, + source_url: str, + ignored_hostnames: Iterable[str] = (), +) -> Iterator[str]: + """ + Yield outgoing Webmention targets from an h-entry. + + Targets are collected from links in the entry content and supported + microformats reaction properties. Invalid, self-referencing, and ignored + targets are excluded. + + :param content: e-content element of the h-entry. + :param mf2_entry: Parsed microformats representation of the h-entry. + :param base_url: Base URL against which relative targets are resolved. + :param source_url: URL of the source document. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Iterator over outgoing Webmention target URLs. + """ + hrefs = ( + href + for element in content.find_all("a", href=True) + if isinstance(href := element.get("href"), str) + ) + + reactions = chain.from_iterable( + property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + ) + + for value in chain(hrefs, reactions): + target = normalize_target(value, base_url=base_url, source_url=source_url) + if target is not None: + hostname = (urlsplit(target).hostname or "").rstrip(".") + if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): + yield target + + +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + +def scan_source_file( + path: Path, + *, + root: Path, + base_url: str | None, + ignored_hostnames: Iterable[str] = (), +) -> ScannedSource: + """ + Scan one generated source document for outgoing Webmentions. + + :param path: Path of the generated HTML document. + :param root: Root directory containing generated source documents. + :param base_url: Configured source base URL, if available. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Scanned source metadata and discovered targets. + :raises SourceScanError: If the document cannot be safely interpreted as a + Webmention source. + :raises OSError: If the source document cannot be read. + """ + # Use the path as published for its URL, but read only files whose + # resolved target stays inside the configured source directory. Resolving + # before reading also protects against symlinked index.html files. + resolved_root = root.resolve() + resolved_path = path.resolve() + if not resolved_path.is_relative_to(resolved_root): + raise SourceScanError(f"Source document escapes source directory: {path}") + + relative_path = path.relative_to(root) + document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") + + source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + + if source_url is None: + if base_url is None: + raise SourceScanError( + "Document has no canonical URL and SOURCE_BASE_URL is not configured" + ) + + source_url = source_url_for_path(relative_path, base_url) + + entry_element, mf2_entry = primary_entry(document, source_url) + + content = content_element(entry_element) + + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + + return ScannedSource( + path=relative_path.as_posix(), + url=source_url, + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, + ) diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index 1ff3690..3dbb896 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -6,6 +6,33 @@ SPDX-License-Identifier: BSD-3-Clause {% extends "base.html" %} {% from "bootstrap5/pagination.html" import render_pagination %} +{% set received_status_presentation = { + "received": { + "row_class": "table-info", + "badge_class": "text-bg-info", + "label": "Pending" + }, + "verifying": { + "row_class": "table-warning", + "badge_class": "text-bg-warning", + "label": "Verifying" + }, + "verified": { + "row_class": "table-success", + "badge_class": "text-bg-success", + "label": "Verified" + }, + "failed": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Declined" + }, + "deleted": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Declined" + } +} %} {% block content %} <div class="d-flex justify-content-between align-items-center mb-3"> @@ -27,27 +54,9 @@ SPDX-License-Identifier: BSD-3-Clause </thead> <tbody> {% for webmention in webmentions %} - {% if webmention.status == "received" %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% set display_status = "Pending" %} - {% elif webmention.status == "verifying" %} - {% set row_class = "table-warning" %} - {% set badge_class = "text-bg-warning" %} - {% set display_status = "Verifying" %} - {% elif webmention.status == "verified" %} - {% set row_class = "table-success" %} - {% set badge_class = "text-bg-success" %} - {% set display_status = "Verified" %} - {% else %} - {% set row_class = "table-danger" %} - {% set badge_class = "text-bg-danger" %} - {% set display_status = "Declined" %} - {% endif %} - <tr class="{{ row_class }}"> - <td> - <span class="badge {{ badge_class }}">{{ display_status }}</span> - </td> + {% set style = received_status_presentation.get(webmention.status, {"row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined"}) %} + <tr class="{{ style.row_class }}"> + <td><span class="badge {{ style.badge_class }}">{{ style.label }}</span></td> <td class="text-break"> <a href="{{ webmention.source }}">{{ webmention.source }}</a> </td> diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 689c651..974c7b3 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -6,6 +6,23 @@ SPDX-License-Identifier: BSD-3-Clause {% extends "base.html" %} {% from "bootstrap5/pagination.html" import render_pagination %} +{% set sent_status_presentation = { + "sent": { + "row_class": "table-success", + "badge_class": "text-bg-success", + "label": "Sent" + }, + "unsupported": { + "row_class": "table-warning", + "badge_class": "text-bg-warning", + "label": "Unsupported" + }, + "failed": { + "row_class": "table-danger", + "badge_class": "text-bg-danger", + "label": "Failed" + } +} %} {% block content %} <div class="mb-3"> @@ -85,32 +102,13 @@ SPDX-License-Identifier: BSD-3-Clause </thead> <tbody> {% for webmention in webmentions %} - {% if webmention.status is none %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% elif webmention.status.value == "sent" %} - {% set row_class = "table-success" %} - {% set badge_class = "text-bg-success" %} - {% elif webmention.status.value == "unsupported" %} - {% set row_class = "table-warning" %} - {% set badge_class = "text-bg-warning" %} - {% elif webmention.status.value == "failed" %} - {% set row_class = "table-danger" %} - {% set badge_class = "text-bg-danger" %} - {% else %} - {% set row_class = "table-info" %} - {% set badge_class = "text-bg-info" %} - {% endif %} - <tr class="{{ row_class }}"> + {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %} + <tr class="{{ style.row_class }}"> <td class="text-break"> <a href="{{ webmention.target }}">{{ webmention.target }}</a> </td> <td> - {% if webmention.status is none %} - <span class="badge {{ badge_class }}">Pending</span> - {% else %} - <span class="badge {{ badge_class }}">{{ webmention.status.value.title() }}</span> - {% endif %} + <span class="badge {{ style.badge_class }}">{{ style.label }}</span> {% if webmention.failure_reason is not none %} <div class="small mt-2">{{ webmention.failure_reason }}</div> {% endif %} diff --git a/webmentions_ssg/url_digest.py b/webmentions_ssg/url_digest.py deleted file mode 100644 index 256d037..0000000 --- a/webmentions_ssg/url_digest.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Portable SHA-256 fingerprints for indexed Webmention URLs.""" - -import hashlib -import json - - -def received_url_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair.""" - encoded = json.dumps( - [source, target], ensure_ascii=False, separators=(",", ":") - ).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def sent_target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID).""" - return hashlib.sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py deleted file mode 100644 index c4324e0..0000000 --- a/webmentions_ssg/url_security.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Validate URL schemes and publicly routable network destinations.""" - -import ipaddress -from urllib.parse import urlsplit - -import dns.exception -import dns.resolver - - -class AddressResolutionError(Exception): - """A hostname could not be resolved (potentially transient).""" - - -class UnsafeAddressError(ValueError): - """A hostname or IP literal identifies a non-public network address.""" - - -_NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") -_IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") - - -def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: - """Reject non-public addresses, including IPv4 hidden in IPv6 forms. - - ``is_global`` alone does not reliably exclude IPv6 transition addresses - across Python versions or networks with NAT64/6to4 routing. - """ - if isinstance(address, ipaddress.IPv6Address): - # The well-known NAT64 prefix can translate an otherwise global IPv6 - # address into a private IPv4 destination. IPv4-compatible addresses - # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. - if address in _NAT64_PREFIX or address in _IPV4_COMPATIBLE_PREFIX: - return False - - # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 - # destination through ipaddress; validate that destination as well. - for embedded in (address.sixtofour, address.ipv4_mapped): - if embedded is not None and not embedded.is_global: - return False - - return address.is_global - - -def public_addresses( - hostname: str, *, lifetime: float | None = None -) -> tuple[str, ...]: - """Resolve *once* and return only checked, globally routable IP addresses. - - A DNS name with a mixture of public and private answers is rejected in its - entirety. IP literals are handled without DNS, including IPv6 literals. - The returned strings must be used for the actual TCP connection; checking - them and then connecting to ``hostname`` reintroduces DNS rebinding. - """ - try: - literal = ipaddress.ip_address(hostname) - except ValueError: - literal = None - - if literal is not None: - addresses = (literal,) - else: - try: - answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) - addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) - except (dns.exception.DNSException, ValueError) as exc: - raise AddressResolutionError( - f"Could not resolve hostname {hostname!r}" - ) from exc - - if not addresses: - raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") - - if any(not is_public_address(address) for address in addresses): - raise UnsafeAddressError(f"Non-public network address for {hostname!r}") - - return tuple(str(address) for address in addresses) - - -def is_public_url(url: str) -> bool: - """Return whether the URL resolves entirely to public IP addresses. - - This is useful for diagnostics, but **not** sufficient to authorize a - request: network connections must use the exact validated IP address. - """ - hostname = urlsplit(url).hostname - - if hostname is None: - raise ValueError("No hostname was specified") - - try: - public_addresses(hostname) - except UnsafeAddressError: - return False - return True - - -def is_http_url(url: str) -> bool: - """ - Check whether a URL is an absolute HTTP or HTTPS URL. - - :param url: URL to validate. - :return: Whether the URL uses HTTP or HTTPS and contains a hostname. - """ - try: - parsed = urlsplit(url) - except ValueError: - return False - - return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None - - -def is_safe_next_url(next_url: str) -> bool: - """Allow redirects only to paths on this site.""" - # Check the original string too: urlsplit("///evil.example").path is - # "/evil.example", even though browsers can treat the original as external. - if "\\" in next_url or not next_url.startswith("/") or next_url.startswith("//"): - return False - - try: - target = urlsplit(next_url) - except ValueError: - return False - - return ( - not target.scheme - and not target.netloc - and target.path.startswith("/") - and not target.path.startswith("//") - ) diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py deleted file mode 100644 index 5b563f0..0000000 --- a/webmentions_ssg/views.py +++ /dev/null @@ -1,454 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Provide routes for authentication and Webmention administration.""" - -import hashlib -import secrets -import uuid -from datetime import UTC, datetime, timedelta -from typing import Any, cast - -import sqlalchemy as sa -import sqlalchemy.exc as sa_exc -from flask import ( - Blueprint, - Response, - abort, - current_app, - flash, - redirect, - render_template, - request, - url_for, -) -from flask.typing import ResponseReturnValue -from flask_limiter.util import get_remote_address -from flask_login import current_user, login_required, login_user, logout_user -from sqlalchemy.engine import CursorResult -from sqlalchemy.orm import selectinload - -from . import CSRF, LIMITER, forms -from . import DATABASE as db -from .models import ReceivedWebmention, SentWebmention, Source, User -from .tasks.receiver import verify_webmention -from .tasks.scanner import manual_scan_sources -from .tasks.sender import send_webmention -from .url_digest import received_url_digest -from .url_security import is_safe_next_url - -root_page = Blueprint("root", __name__) - -DUMMY_USER = User(username=secrets.token_urlsafe()) -DUMMY_USER.password = secrets.token_urlsafe() - - -def login_username_limit_key() -> str: - """Group submitted usernames across IPs without storing raw account names.""" - username = request.form.get("username", "").strip().casefold() - if not username: - # A missing username must not put all clients into one global bucket. - return f"missing-username:{get_remote_address()}" - digest = hashlib.sha256(username.encode("utf-8")).hexdigest() - return f"username:{digest}" - - -@root_page.route("/") -def index() -> ResponseReturnValue: - """ - Redirect to the received Webmentions view. - - :return: Redirect response to the received Webmentions view. - """ - return redirect(url_for("root.received")) - - -@root_page.route("/login", methods=["GET", "POST"]) -@LIMITER.limit( - lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"], - key_func=login_username_limit_key, - methods=["POST"], - error_message="Too many login attempts. Please try again later.", -) -@LIMITER.limit( - lambda: current_app.config["LOGIN_RATE_LIMIT"], - methods=["POST"], - error_message="Too many login attempts. Please try again later.", -) -def login() -> ResponseReturnValue: - """ - Authenticate a user and start a login session. - - Authenticated users are redirected to the application index. After a - successful login, the user is redirected to the requested local URL when - provided. - - :return: Rendered login page or redirect response. - """ - if current_user.is_authenticated: - return redirect(url_for("root.index")) - - form = forms.LoginForm() - - if form.validate_on_submit(): - user = db.session.scalar( - sa.select(User).where(User.username == form.username.data) - ) - - user_exists = user is not None - - if user is None: - user = DUMMY_USER - - if ( - form.password.data is None - or not user.check_password(form.password.data) - or not user_exists - ): - flash("Invalid username or password", "danger") - return redirect(url_for("root.login")) - login_user(user) - - next_url = request.args.get("next") - if not next_url or not is_safe_next_url(next_url): - next_url = url_for("root.index") - return redirect(next_url) - - return render_template("login.html", form=form) - - -@root_page.route("/logout", methods=("POST",)) -@login_required -def logout() -> ResponseReturnValue: - """ - End the current user's login session. - - :return: Redirect response to the application index. - """ - logout_user() - return redirect(url_for("root.index")) - - -@root_page.route("/received") -@login_required -def received() -> ResponseReturnValue: - """ - Display received Webmentions. - - :return: Rendered page containing the paginated received Webmentions. - """ - webmentions = db.paginate( - sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), - per_page=25, - ) - - return render_template( - "received.html", - title="Received Webmentions", - webmentions=webmentions, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/received/<uuid:identifier>/delete") -@login_required -def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Delete a received Webmention. - - :param identifier: Identifier of the Webmention to delete. - :return: Redirect response to the received Webmentions view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - db.session.delete(webmention) - db.session.commit() - - flash(f"Webmention {webmention.uuid} deleted.", "success") - - return redirect( - url_for("root.received", page=request.args.get("page", 1, type=int)) - ) - - -@root_page.post("/received/<uuid:identifier>/reverify") -@login_required -def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Queue a received Webmention for reverification. - - The Webmention status is reset before a new verification task is queued. - - :param identifier: Identifier of the Webmention to reverify. - :return: Redirect response to the received Webmentions view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - webmention.status = "received" - webmention.failure_reason = None - webmention.recovery_attempts = 0 - - db.session.commit() - - verify_webmention(identifier) - - flash(f"Webmention {webmention.uuid} queued for reverification.", "success") - - return redirect( - url_for("root.received", page=request.args.get("page", 1, type=int)) - ) - - -@root_page.route("/sent") -@login_required -def sent() -> ResponseReturnValue: - """ - Display sources with sent Webmentions. - - :return: Rendered page containing the paginated source list. - """ - sources = db.paginate( - sa.select(Source) - .options(selectinload(Source.sent_webmentions)) - .order_by(Source.last_seen_at.desc()), - per_page=25, - ) - return render_template( - "sent.html", - title="Sent Webmentions", - sources=sources, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/sent/rescan") -@login_required -def rescan_sent_sources() -> ResponseReturnValue: - """ - Queue a manual scan of sent Webmention sources. - - :return: Redirect response to the sent Webmentions view. - """ - form = forms.AdminActionForm() - if form.validate_on_submit(): - manual_scan_sources() - flash("Source rescan queued.", "success") - return redirect(url_for("root.sent")) - - -@root_page.route("/sent/<uuid:source_identifier>") -@login_required -def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: - """ - Display the sent Webmentions associated with a source. - - :param source_identifier: Identifier of the source to display. - :return: Rendered source details page. - """ - source = db.session.scalar( - sa.select(Source).where(Source.uuid == source_identifier) - ) - - if source is None: - return abort(404) - - webmentions = db.paginate( - sa.select(SentWebmention) - .where(SentWebmention.source_id == source.uuid) - .order_by(SentWebmention.target, SentWebmention.uuid), - per_page=25, - ) - - return render_template( - "sent_source.html", - title="Sent Webmentions", - source=source, - webmentions=webmentions, - action_form=forms.AdminActionForm(), - ) - - -@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend") -@login_required -def resend_sent_webmention( - source_identifier: uuid.UUID, webmention_identifier: uuid.UUID -) -> ResponseReturnValue: - """ - Queue an outgoing Webmention to be sent again. - - The current desired source revision is reopened for processing so the - sender task does not treat the Webmention as already completed. Historical - successful-send fields are preserved, while result fields from the previous - attempt are cleared. - - :param source_identifier: Identifier of the source owning the Webmention. - :param webmention_identifier: Identifier of the sent Webmention to resend. - :return: Redirect response to the source details view. - """ - form = forms.AdminActionForm() - - if not form.validate_on_submit(): - return abort(400) - - webmention = db.session.get(SentWebmention, webmention_identifier) - - if webmention is None or webmention.source_id != source_identifier: - return abort(404) - - # Reopen the existing desired revision instead of inventing a new source - # revision. send_webmention() otherwise exits when this revision has already - # been processed. - webmention.processed_revision = None - webmention.attempt_count = 0 - webmention.next_retry_at = None - webmention.status = None - webmention.failure_reason = None - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None - - db.session.commit() - - send_webmention(webmention_identifier) - - flash(f"Webmention {webmention.uuid} queued for resend.", "success") - - return redirect(url_for("root.sent_source", source_identifier=source_identifier)) - - -@root_page.post("/endpoint") -@LIMITER.limit( - lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"], - key_func=lambda: "all-clients", -) -@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"]) -@CSRF.exempt -def endpoint() -> ResponseReturnValue: - """ - Receive and queue a Webmention for verification. - - Existing Webmentions with the same source and target are reset for - reverification. Concurrent insertion of the same Webmention is handled by - retrieving and updating the row created by the competing request. - - :return: HTTP 201 response with the Webmention status URL in the - ``Location`` header, or validation errors with HTTP 400. - """ - form = forms.EndpointForm(meta={"csrf": False}) - - if not form.validate_on_submit(): - return form.errors, 400 - - source = cast(str, form.source.data) - target = cast(str, form.target.data) - - url_digest = received_url_digest(source, target) - webmention = db.session.execute( - sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) - ).scalar_one_or_none() - - # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. - if webmention is not None and (webmention.source, webmention.target) != ( - source, - target, - ): - return {"error": "Webmention URL fingerprint conflict"}, 409 - - should_verify = False - - if webmention is None: - identifier = uuid.uuid7() - - webmention = ReceivedWebmention(uuid=identifier, source=source, target=target) - - db.session.add(webmention) - - try: - db.session.commit() - should_verify = True - except sa_exc.IntegrityError: - db.session.rollback() - - # Another request may have inserted the same source/target pair - # after our SELECT but before our COMMIT. - webmention = db.session.execute( - sa.select(ReceivedWebmention).where( - ReceivedWebmention.url_digest == url_digest - ) - ).scalar_one_or_none() - if webmention is None or (webmention.source, webmention.target) != ( - source, - target, - ): - return {"error": "Webmention URL fingerprint conflict"}, 409 - - else: - now = datetime.now(UTC) - cutoff = now - timedelta( - seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] - ) - - # Atomic conditional update: only one concurrent request can reopen an expired verification. - result = cast( - CursorResult[Any], - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention.uuid, - ReceivedWebmention.updated_at <= cutoff, - ) - .values( - status="received", - failure_reason=None, - recovery_attempts=0, - updated_at=now, - ) - .execution_options(synchronize_session=False) - ), - ) - should_verify = result.rowcount == 1 - db.session.commit() - - identifier = webmention.uuid - - if should_verify: - verify_webmention(identifier) - - status_url = url_for("root.status", identifier=str(identifier), _external=True) - - return Response(status=201, headers={"Location": status_url}) - - -@root_page.route("/status/<uuid:identifier>") -def status(identifier: uuid.UUID) -> ResponseReturnValue: - """ - Display the verification status of a received Webmention. - - :param identifier: Identifier of the Webmention to display. - :return: Rendered Webmention status page. - """ - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - return abort(404) - - return render_template( - "status.html", title="Webmention status", webmention=webmention - ) diff --git a/webmentions_ssg/views/__init__.py b/webmentions_ssg/views/__init__.py new file mode 100644 index 0000000..91966d4 --- /dev/null +++ b/webmentions_ssg/views/__init__.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Register all views under the historic root Blueprint.""" + +from flask import Blueprint + +root_page = Blueprint("root", __name__) + + +def register_routes() -> Blueprint: + """Import task-backed view handlers after Huey initialization and return the Blueprint.""" + from . import admin, endpoint, session # noqa: F401 + + return root_page diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py new file mode 100644 index 0000000..46e3b18 --- /dev/null +++ b/webmentions_ssg/views/admin.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authenticated incoming and outgoing Webmention management.""" + +import uuid + +import sqlalchemy as sa +from flask import abort, flash, redirect, render_template, request, url_for +from flask.typing import ResponseReturnValue +from flask_login import login_required +from sqlalchemy.orm import selectinload + +from .. import forms +from ..extensions import DATABASE as db +from ..models import ReceivedWebmention, SentWebmention, Source +from ..tasks.receiver import verify_webmention +from ..tasks.scanner import manual_scan_sources +from ..tasks.sender import send_webmention +from . import root_page + + +@root_page.route("/received") +@login_required +def received() -> ResponseReturnValue: + """ + Display received Webmentions. + + :return: Rendered page containing the paginated received Webmentions. + """ + webmentions = db.paginate( + sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), + per_page=25, + ) + + return render_template( + "received.html", + title="Received Webmentions", + webmentions=webmentions, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/received/<uuid:identifier>/delete") +@login_required +def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Delete a received Webmention. + + :param identifier: Identifier of the Webmention to delete. + :return: Redirect response to the received Webmentions view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + db.session.delete(webmention) + db.session.commit() + + flash(f"Webmention {webmention.uuid} deleted.", "success") + + return redirect( + url_for("root.received", page=request.args.get("page", 1, type=int)) + ) + + +@root_page.post("/received/<uuid:identifier>/reverify") +@login_required +def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Queue a received Webmention for reverification. + + The Webmention status is reset before a new verification task is queued. + + :param identifier: Identifier of the Webmention to reverify. + :return: Redirect response to the received Webmentions view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + webmention = webmention.reset_for_verification() + + db.session.commit() + + verify_webmention(identifier) + + flash(f"Webmention {webmention.uuid} queued for reverification.", "success") + + return redirect( + url_for("root.received", page=request.args.get("page", 1, type=int)) + ) + + +@root_page.route("/sent") +@login_required +def sent() -> ResponseReturnValue: + """ + Display sources with sent Webmentions. + + :return: Rendered page containing the paginated source list. + """ + sources = db.paginate( + sa.select(Source) + .options(selectinload(Source.sent_webmentions)) + .order_by(Source.last_seen_at.desc()), + per_page=25, + ) + return render_template( + "sent.html", + title="Sent Webmentions", + sources=sources, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/sent/rescan") +@login_required +def rescan_sent_sources() -> ResponseReturnValue: + """ + Queue a manual scan of sent Webmention sources. + + :return: Redirect response to the sent Webmentions view. + """ + form = forms.AdminActionForm() + if form.validate_on_submit(): + manual_scan_sources() + flash("Source rescan queued.", "success") + return redirect(url_for("root.sent")) + + +@root_page.route("/sent/<uuid:source_identifier>") +@login_required +def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: + """ + Display the sent Webmentions associated with a source. + + :param source_identifier: Identifier of the source to display. + :return: Rendered source details page. + """ + source = db.session.scalar( + sa.select(Source).where(Source.uuid == source_identifier) + ) + + if source is None: + return abort(404) + + webmentions = db.paginate( + sa.select(SentWebmention) + .where(SentWebmention.source_id == source.uuid) + .order_by(SentWebmention.target, SentWebmention.uuid), + per_page=25, + ) + + return render_template( + "sent_source.html", + title="Sent Webmentions", + source=source, + webmentions=webmentions, + action_form=forms.AdminActionForm(), + ) + + +@root_page.post("/sent/<uuid:source_identifier>/<uuid:webmention_identifier>/resend") +@login_required +def resend_sent_webmention( + source_identifier: uuid.UUID, webmention_identifier: uuid.UUID +) -> ResponseReturnValue: + """ + Queue an outgoing Webmention to be sent again. + + The current desired source revision is reopened for processing so the + sender task does not treat the Webmention as already completed. Historical + successful-send fields are preserved, while result fields from the previous + attempt are cleared. + + :param source_identifier: Identifier of the source owning the Webmention. + :param webmention_identifier: Identifier of the sent Webmention to resend. + :return: Redirect response to the source details view. + """ + form = forms.AdminActionForm() + + if not form.validate_on_submit(): + return abort(400) + + webmention = db.session.get(SentWebmention, webmention_identifier) + + if webmention is None or webmention.source_id != source_identifier: + return abort(404) + + # Reopen the existing desired revision instead of inventing a new source + # revision. send_webmention() otherwise exits when this revision has already + # been processed. + webmention = webmention.reopen( + webmention.desired_revision, webmention.active, resend=True + ) + + db.session.commit() + + send_webmention(webmention_identifier) + + flash(f"Webmention {webmention.uuid} queued for resend.", "success") + + return redirect(url_for("root.sent_source", source_identifier=source_identifier)) diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py new file mode 100644 index 0000000..1c2fe13 --- /dev/null +++ b/webmentions_ssg/views/endpoint.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Public Webmention endpoint and verification status view.""" + +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +import sqlalchemy as sa +import sqlalchemy.exc as sa_exc +from flask import Response, abort, current_app, render_template, url_for +from flask.typing import ResponseReturnValue +from sqlalchemy.engine import CursorResult + +from .. import forms +from ..extensions import CSRF, LIMITER +from ..extensions import DATABASE as db +from ..models import ReceivedWebmention, received_url_digest +from ..tasks.receiver import verify_webmention +from . import root_page + + +def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None: + """Upsert a submission atomically and return (identifier, enqueue flag). + + ``None`` means that the URL fingerprint matches a different source/target. + No background task is enqueued until the transaction is committed. + """ + url_digest = received_url_digest(source, target) + webmention = db.session.execute( + sa.select(ReceivedWebmention).where(ReceivedWebmention.url_digest == url_digest) + ).scalar_one_or_none() + + # A SHA-256 collision must never cause an unrelated Webmention to be overwritten. + if webmention is not None and (webmention.source, webmention.target) != ( + source, + target, + ): + return None + + should_verify = False + + if webmention is None: + identifier = uuid.uuid7() + webmention = ReceivedWebmention(uuid=identifier, source=source, target=target) + db.session.add(webmention) + + try: + db.session.commit() + should_verify = True + except sa_exc.IntegrityError: + db.session.rollback() + + # Another request may have inserted the same source/target pair + # after our SELECT but before our COMMIT. + webmention = db.session.execute( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.url_digest == url_digest + ) + ).scalar_one_or_none() + if webmention is None or (webmention.source, webmention.target) != ( + source, + target, + ): + return None + + else: + now = datetime.now(UTC) + cutoff = now - timedelta( + seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] + ) + + # Atomic conditional update: only one concurrent request can reopen an expired verification. + result = cast( + CursorResult[Any], + db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention.uuid, + ReceivedWebmention.updated_at <= cutoff, + ) + .values(ReceivedWebmention.verification_reset_values(at=now)) + .execution_options(synchronize_session=False) + ), + ) + should_verify = result.rowcount == 1 + db.session.commit() + + return webmention.uuid, should_verify + + +@root_page.post("/endpoint") +@LIMITER.limit( + lambda: current_app.config["ENDPOINT_GLOBAL_RATE_LIMIT"], + key_func=lambda: "all-clients", +) +@LIMITER.limit(lambda: current_app.config["ENDPOINT_RATE_LIMIT"]) +@CSRF.exempt +def endpoint() -> ResponseReturnValue: + """ + Receive and queue a Webmention for verification. + + Existing Webmentions with the same source and target are reset for + reverification. Concurrent insertion of the same Webmention is handled by + retrieving and updating the row created by the competing request. + + :return: HTTP 201 response with the Webmention status URL in the + ``Location`` header, or validation errors with HTTP 400. + """ + form = forms.EndpointForm(meta={"csrf": False}) + + if not form.validate_on_submit(): + return form.errors, 400 + + source = cast(str, form.source.data) + target = cast(str, form.target.data) + + result = submit_received(source, target) + + if result is None: + return {"error": "Webmention URL fingerprint conflict"}, 409 + + identifier, should_verify = result + + if should_verify: + verify_webmention(identifier) + + status_url = url_for("root.status", identifier=str(identifier), _external=True) + + return Response(status=201, headers={"Location": status_url}) + + +@root_page.route("/status/<uuid:identifier>") +def status(identifier: uuid.UUID) -> ResponseReturnValue: + """ + Display the verification status of a received Webmention. + + :param identifier: Identifier of the Webmention to display. + :return: Rendered Webmention status page. + """ + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + return render_template( + "status.html", title="Webmention status", webmention=webmention + ) diff --git a/webmentions_ssg/views/session.py b/webmentions_ssg/views/session.py new file mode 100644 index 0000000..7bd8a10 --- /dev/null +++ b/webmentions_ssg/views/session.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Authentication and session routes.""" + +import sqlalchemy as sa +from flask import current_app, flash, redirect, render_template, request, url_for +from flask.typing import ResponseReturnValue +from flask_login import current_user, login_required, login_user, logout_user + +from .. import forms +from ..auth import DUMMY_USER, is_safe_next_url, login_username_limit_key +from ..extensions import DATABASE as db +from ..extensions import LIMITER +from ..models import User +from . import root_page + + +@root_page.route("/") +def index() -> ResponseReturnValue: + """ + Redirect to the received Webmentions view. + + :return: Redirect response to the received Webmentions view. + """ + return redirect(url_for("root.received")) + + +@root_page.route("/login", methods=["GET", "POST"]) +@LIMITER.limit( + lambda: current_app.config["LOGIN_USERNAME_RATE_LIMIT"], + key_func=login_username_limit_key, + methods=["POST"], + error_message="Too many login attempts. Please try again later.", +) +@LIMITER.limit( + lambda: current_app.config["LOGIN_RATE_LIMIT"], + methods=["POST"], + error_message="Too many login attempts. Please try again later.", +) +def login() -> ResponseReturnValue: + """ + Authenticate a user and start a login session. + + Authenticated users are redirected to the application index. After a + successful login, the user is redirected to the requested local URL when + provided. + + :return: Rendered login page or redirect response. + """ + if current_user.is_authenticated: + return redirect(url_for("root.index")) + + form = forms.LoginForm() + + if form.validate_on_submit(): + user = db.session.scalar( + sa.select(User).where(User.username == form.username.data) + ) + + user_exists = user is not None + + if user is None: + user = DUMMY_USER + + if ( + form.password.data is None + or not user.check_password(form.password.data) + or not user_exists + ): + flash("Invalid username or password", "danger") + return redirect(url_for("root.login")) + login_user(user) + + next_url = request.args.get("next") + if not next_url or not is_safe_next_url(next_url): + next_url = url_for("root.index") + return redirect(next_url) + + return render_template("login.html", form=form) + + +@root_page.route("/logout", methods=("POST",)) +@login_required +def logout() -> ResponseReturnValue: + """ + End the current user's login session. + + :return: Redirect response to the application index. + """ + logout_user() + return redirect(url_for("root.index")) |
