From 6d6d079a3a99d3d3a922a640b6925eb2adf044de Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Tue, 17 Nov 2026 17:09:59 +0100 Subject: refactor(core)!: split forms and models by feature Replace monolithic form and model modules with focused packages for receiver, moderation, source, user, and shared validation concerns. Rename source policy concepts to domain moderation, including routes, models, tables, configuration, templates, and tests. BREAKING CHANGE: form and model import paths, moderation routes, and `SOURCE_ALLOWLIST_ENABLED` have changed; use `ALLOWLIST_ENABLED` and the new feature-specific modules. --- webmentions_ssg/auth.py | 2 +- webmentions_ssg/cli.py | 2 +- webmentions_ssg/config.py | 10 +- webmentions_ssg/forms.py | 268 ---------- webmentions_ssg/forms/__init__.py | 5 + webmentions_ssg/forms/admin.py | 14 + webmentions_ssg/forms/endpoint.py | 52 ++ webmentions_ssg/forms/moderation.py | 25 + webmentions_ssg/forms/received.py | 70 +++ webmentions_ssg/forms/user.py | 17 + webmentions_ssg/forms/validators.py | 135 +++++ ...1c2e08d7_rename_source_lists_to_domain_lists.py | 28 + .../d845aa7fbc19_portable_url_fingerprints.py | 4 +- webmentions_ssg/models.py | 420 --------------- webmentions_ssg/models/__init__.py | 5 + webmentions_ssg/models/moderation.py | 28 + webmentions_ssg/models/received.py | 131 +++++ webmentions_ssg/models/sources.py | 208 ++++++++ webmentions_ssg/models/user.py | 66 +++ webmentions_ssg/models/utils.py | 41 ++ webmentions_ssg/moderation_policy.py | 62 +++ webmentions_ssg/source_policy.py | 62 --- webmentions_ssg/tasks/notifications.py | 2 +- webmentions_ssg/tasks/retention.py | 2 +- webmentions_ssg/tasks/scanner.py | 2 +- webmentions_ssg/tasks/sender.py | 2 +- webmentions_ssg/tasks/verification.py | 6 +- webmentions_ssg/templates/base.html | 4 +- webmentions_ssg/templates/moderation.html | 69 +++ webmentions_ssg/templates/source_domains.html | 68 --- webmentions_ssg/views/__init__.py | 4 +- webmentions_ssg/views/admin.py | 585 --------------------- webmentions_ssg/views/endpoint.py | 11 +- webmentions_ssg/views/moderation.py | 105 ++++ webmentions_ssg/views/received.py | 391 ++++++++++++++ webmentions_ssg/views/session.py | 6 +- webmentions_ssg/views/sources.py | 122 +++++ 37 files changed, 1603 insertions(+), 1431 deletions(-) delete mode 100644 webmentions_ssg/forms.py create mode 100644 webmentions_ssg/forms/__init__.py create mode 100644 webmentions_ssg/forms/admin.py create mode 100644 webmentions_ssg/forms/endpoint.py create mode 100644 webmentions_ssg/forms/moderation.py create mode 100644 webmentions_ssg/forms/received.py create mode 100644 webmentions_ssg/forms/user.py create mode 100644 webmentions_ssg/forms/validators.py create mode 100644 webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py delete mode 100644 webmentions_ssg/models.py create mode 100644 webmentions_ssg/models/__init__.py create mode 100644 webmentions_ssg/models/moderation.py create mode 100644 webmentions_ssg/models/received.py create mode 100644 webmentions_ssg/models/sources.py create mode 100644 webmentions_ssg/models/user.py create mode 100644 webmentions_ssg/models/utils.py create mode 100644 webmentions_ssg/moderation_policy.py delete mode 100644 webmentions_ssg/source_policy.py create mode 100644 webmentions_ssg/templates/moderation.html delete mode 100644 webmentions_ssg/templates/source_domains.html delete mode 100644 webmentions_ssg/views/admin.py create mode 100644 webmentions_ssg/views/moderation.py create mode 100644 webmentions_ssg/views/received.py create mode 100644 webmentions_ssg/views/sources.py (limited to 'webmentions_ssg') diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index ce4960d..ac1f4e7 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -16,7 +16,7 @@ from flask_login import LoginManager from werkzeug.security import generate_password_hash from .extensions import db -from .models import User +from .models.user import User from .validation import is_control_character login_manager = LoginManager() diff --git a/webmentions_ssg/cli.py b/webmentions_ssg/cli.py index b26b35b..707402d 100644 --- a/webmentions_ssg/cli.py +++ b/webmentions_ssg/cli.py @@ -15,7 +15,7 @@ from sqlalchemy.exc import IntegrityError from .auth import validate_new_password from .extensions import db, mail -from .models import User +from .models.user import User USERS_CLI = AppGroup("users", help="Manage users.") MAIL_CLI = AppGroup("mail", help="Handle mail") diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 576f376..28ab43d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -54,9 +54,9 @@ class DefaultConfig: # Receiver # Empty by default: no target hostnames are accepted until configured. ALLOWED_TARGET_HOSTNAMES = frozenset() - # When enabled, only source domains listed in AllowedSource are accepted. - # Entries in BlockedSource always take precedence. - SOURCE_ALLOWLIST_ENABLED = False + # When enabled, only source domains listed in AllowedDomain are accepted. + # Entries in BlockedDomain always take precedence. + ALLOWLIST_ENABLED = False # Cap incoming HTTP request bodies (also applies to other Flask routes). MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. @@ -213,8 +213,8 @@ def validated_config_overrides( except (TypeError, ValueError) as exc: raise type(exc)(f"{key}: {exc}") from exc - if type(config["SOURCE_ALLOWLIST_ENABLED"]) is not bool: - raise ValueError("SOURCE_ALLOWLIST_ENABLED must be a boolean.") + if type(config["ALLOWLIST_ENABLED"]) is not bool: + raise ValueError("ALLOWLIST_ENABLED must be a boolean.") proxy_hops = config["PROXY_FIX_TRUSTED_HOPS"] if type(proxy_hops) is not int or proxy_hops < 0: diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py deleted file mode 100644 index 3b61a8a..0000000 --- a/webmentions_ssg/forms.py +++ /dev/null @@ -1,268 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Webmention forms and URL validation.""" - -import ipaddress - -import httpx -from flask import current_app -from flask_wtf import FlaskForm -from wtforms import ( - DateField, - Field, - HiddenField, - IntegerField, - PasswordField, - SearchField, - SelectField, - StringField, - SubmitField, - ValidationError, -) -from wtforms.form import BaseForm -from wtforms.validators import ( - AnyOf, - InputRequired, - NumberRange, - Optional, - StopValidation, -) - -from .hostname import normalize_hostname -from .net import is_public_address -from .validation import is_control_character - -RECEIVED_STATUS_CHOICES = { - "": "All statuses", - "pending": "Pending", - "verifying": "Verifying", - "verified": "Verified", - "declined": "Declined", -} - - -class MaxURLBytes: - """Check the UTF-8 byte length of a submitted URL using the current config.""" - - def __init__( - self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None - ) -> None: - self.config_key = config_key - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - if field.data is None: - return - - max_bytes = current_app.config[self.config_key] - - try: - length = len(field.data.encode("utf-8")) - except UnicodeEncodeError as exc: - raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc - - if length > max_bytes: - raise ValidationError( - self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") - ) - - -class StrictHTTPURL: - """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. - - httpx automatically percent-encodes some whitespace. Reject it instead of - silently changing the submitted URL (which may be used in exact URL matches). - """ - - def __call__(self, form: BaseForm, field: Field) -> None: - value = field.data - if not isinstance(value, str): - raise StopValidation(field.gettext("Invalid URL.")) - - if any( - character.isspace() or is_control_character(character) - for character in value - ): - raise StopValidation(field.gettext("Invalid URL.")) - - try: - parsed = httpx.URL(value) - except (httpx.InvalidURL, ValueError, UnicodeError) as exc: - raise StopValidation(field.gettext("Invalid URL.")) from exc - - if not parsed.host: - raise StopValidation(field.gettext("Invalid URL.")) - - if parsed.scheme not in ("http", "https"): - raise StopValidation( - field.gettext(f"{field.name} must begin with http or https") - ) - - -class AllowedTargetHostname: - """Validate that a URL uses an allowed hostname. - - :param message: Optional validation error message. - """ - - def __init__(self, message: str | None = None) -> None: - """Initialize the validator. - - :param message: Optional validation error message. - """ - self.message = message - - def __call__(self, form: BaseForm, field: Field) -> None: - """Validate that a URL uses an allowed hostname. - - :param form: Form containing the field. - :param field: Field containing the URL to validate. - :raises ValidationError: If the URL hostname is not configured as allowed. - """ - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError: - hostname = None - - if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: - return - raise ValidationError(self.message or field.gettext("Invalid input.")) - - -class RejectNonPublicIPLiteral: - """Reject non-public IP literals without performing DNS in the web worker.""" - - def __call__(self, form: BaseForm, field: Field) -> None: - try: - hostname = normalize_hostname(field.data, from_url=True) - except ValueError as exc: - raise ValidationError(field.gettext("Invalid input.")) from exc - - try: - address = ipaddress.ip_address(hostname) - except ValueError: - return # Named hosts are resolved only during the background fetch. - - if not is_public_address(address): - raise ValidationError(field.gettext("URL must use a public address.")) - - -class LoginForm(FlaskForm): - """Form for authenticating a user.""" - - username = StringField("Username", validators=[InputRequired()]) - password = PasswordField("Password", validators=[InputRequired()]) - submit = SubmitField("Sign In") - - -class WebmentionSubmissionForm(FlaskForm): - """Form for receiving a Webmention. - - The source must be a public HTTP or HTTPS URL distinct from the target. - The target must be an HTTP or HTTPS URL using an allowed hostname. - """ - - source = StringField( - "source", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - RejectNonPublicIPLiteral(), - ], - ) - target = StringField( - "target", - validators=[ - InputRequired(), - MaxURLBytes(), - StrictHTTPURL(), - AllowedTargetHostname(), - ], - ) - - def validate_source(self, field: Field) -> None: - """Reject a source URL that equals the target URL. - - :param field: Source URL field being validated. - :raises ValidationError: If the source URL equals the target URL. - """ - if field.data == self.target.data: - raise ValidationError(field.gettext("Field must not be equal to target.")) - - -class CanonicalDate: - """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" - - def __call__(self, form: BaseForm, field: DateField) -> None: - if ( - field.data is not None - and field.raw_data - and field.raw_data[0] != field.data.isoformat() - ): - raise ValidationError(field.gettext("Invalid date.")) - - -def default_received_page_size() -> int: - """Use 25 rows unless the configured maximum is lower.""" - return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) - - -class ReceivedWebmentionFilterForm(FlaskForm): - """GET filters for the received Webmention list.""" - - status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") - q = SearchField("Source or target") - date_from = DateField( - "Updated from (UTC)", validators=[Optional(), CanonicalDate()] - ) - date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) - sort = HiddenField( - default="received", - validators=[AnyOf(("received", "updated", "status", "source", "target"))], - ) - order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) - per_page = IntegerField( - "Rows per page", - default=default_received_page_size, - validators=[NumberRange(min=1)], - ) - - def validate_per_page(self, field: IntegerField) -> None: - """Enforce the deployment's maximum received-list page size.""" - if ( - field.data is not None - and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] - ): - raise ValidationError("Rows per page exceeds the configured maximum.") - - def validate_date_to(self, field: DateField) -> None: - """Require the end date to be on or after the start date.""" - if ( - self.date_from.data is not None - and field.data is not None - and field.data < self.date_from.data - ): - raise ValidationError("End date must be on or after start date.") - - -class ConfirmActionForm(FlaskForm): - """Form for confirming an administrative action.""" - - submit = SubmitField("Confirm") - - -class SourceDomainForm(FlaskForm): - """Add a domain to the source blocklist or allowlist.""" - - domain = StringField("Source domain", validators=[InputRequired()]) - submit = SubmitField("Add domain") - - def validate_domain(self, field: Field) -> None: - """Require a bare hostname and persist its canonical form.""" - try: - field.data = normalize_hostname(field.data) - except ValueError as exc: - raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py new file mode 100644 index 0000000..5f61c0e --- /dev/null +++ b/webmentions_ssg/forms/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for webmention submission, authentication, filtering and moderation.""" diff --git a/webmentions_ssg/forms/admin.py b/webmentions_ssg/forms/admin.py new file mode 100644 index 0000000..97d523d --- /dev/null +++ b/webmentions_ssg/forms/admin.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Administrative action confirmation forms.""" + +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class ConfirmActionForm(FlaskForm): + """Form for confirming an administrative action.""" + + submit = SubmitField("Confirm") diff --git a/webmentions_ssg/forms/endpoint.py b/webmentions_ssg/forms/endpoint.py new file mode 100644 index 0000000..e063e73 --- /dev/null +++ b/webmentions_ssg/forms/endpoint.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Webmention submission form.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, ValidationError +from wtforms.validators import InputRequired + +from .validators import ( + AllowedTargetHostname, + MaxURLBytes, + RejectNonPublicIPLiteral, + StrictHTTPURL, +) + + +class WebmentionSubmissionForm(FlaskForm): + """Form for receiving a Webmention. + + The source must be a public HTTP or HTTPS URL distinct from the target. + The target must be an HTTP or HTTPS URL using an allowed hostname. + """ + + source = StringField( + "source", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + RejectNonPublicIPLiteral(), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + AllowedTargetHostname(), + ], + ) + + def validate_source(self, field: Field) -> None: + """Reject a source URL that equals the target URL. + + :param field: Source URL field being validated. + :raises ValidationError: If the source URL equals the target URL. + """ + if field.data == self.target.data: + raise ValidationError(field.gettext("Field must not be equal to target.")) diff --git a/webmentions_ssg/forms/moderation.py b/webmentions_ssg/forms/moderation.py new file mode 100644 index 0000000..5987d14 --- /dev/null +++ b/webmentions_ssg/forms/moderation.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Forms for managing domain allowlists and blocklists.""" + +from flask_wtf import FlaskForm +from wtforms import Field, StringField, SubmitField, ValidationError +from wtforms.validators import InputRequired + +from ..hostname import normalize_hostname + + +class ModerateDomainForm(FlaskForm): + """Add a source domain to the blocklist or allowlist.""" + + domain = StringField("Source domain", validators=[InputRequired()]) + submit = SubmitField("Add domain") + + def validate_domain(self, field: Field) -> None: + """Require a bare hostname and persist its canonical form.""" + try: + field.data = normalize_hostname(field.data) + except ValueError as exc: + raise ValidationError(str(exc)) from exc diff --git a/webmentions_ssg/forms/received.py b/webmentions_ssg/forms/received.py new file mode 100644 index 0000000..c4a6ed0 --- /dev/null +++ b/webmentions_ssg/forms/received.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Received Webmention filtering form.""" + +from flask import current_app +from flask_wtf import FlaskForm +from wtforms import ( + DateField, + HiddenField, + IntegerField, + SearchField, + SelectField, + ValidationError, +) +from wtforms.validators import AnyOf, NumberRange, Optional + +from .validators import CanonicalDate + +RECEIVED_STATUS_CHOICES = { + "": "All statuses", + "pending": "Pending", + "verifying": "Verifying", + "verified": "Verified", + "declined": "Declined", +} + + +def default_received_page_size() -> int: + """Use 25 rows unless the configured maximum is lower.""" + return min(25, current_app.config["RECEIVED_MAX_PAGE_SIZE"]) + + +class ReceivedWebmentionFilterForm(FlaskForm): + """GET filters for the received Webmention list.""" + + status = SelectField("Status", choices=RECEIVED_STATUS_CHOICES, default="") + q = SearchField("Source or target") + date_from = DateField( + "Updated from (UTC)", validators=[Optional(), CanonicalDate()] + ) + date_to = DateField("Updated to (UTC)", validators=[Optional(), CanonicalDate()]) + sort = HiddenField( + default="received", + validators=[AnyOf(("received", "updated", "status", "source", "target"))], + ) + order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) + per_page = IntegerField( + "Rows per page", + default=default_received_page_size, + validators=[NumberRange(min=1)], + ) + + def validate_per_page(self, field: IntegerField) -> None: + """Enforce the deployment's maximum received-list page size.""" + if ( + field.data is not None + and field.data > current_app.config["RECEIVED_MAX_PAGE_SIZE"] + ): + raise ValidationError("Rows per page exceeds the configured maximum.") + + def validate_date_to(self, field: DateField) -> None: + """Require the end date to be on or after the start date.""" + if ( + self.date_from.data is not None + and field.data is not None + and field.data < self.date_from.data + ): + raise ValidationError("End date must be on or after start date.") diff --git a/webmentions_ssg/forms/user.py b/webmentions_ssg/forms/user.py new file mode 100644 index 0000000..3bb6421 --- /dev/null +++ b/webmentions_ssg/forms/user.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""User authentication forms.""" + +from flask_wtf import FlaskForm +from wtforms import PasswordField, StringField, SubmitField +from wtforms.validators import InputRequired + + +class LoginForm(FlaskForm): + """Form for authenticating a user.""" + + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py new file mode 100644 index 0000000..e65e9cb --- /dev/null +++ b/webmentions_ssg/forms/validators.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Custom Webmention URL and date field validators.""" + +import ipaddress + +import httpx +from flask import current_app +from wtforms import DateField, Field, ValidationError +from wtforms.form import BaseForm +from wtforms.validators import StopValidation + +from ..hostname import normalize_hostname +from ..net import is_public_address +from ..validation import is_control_character + + +class MaxURLBytes: + """Check the UTF-8 byte length of a submitted URL using the current config.""" + + def __init__( + self, config_key: str = "MAX_WEBMENTION_URL_BYTES", message: str | None = None + ) -> None: + self.config_key = config_key + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + if field.data is None: + return + + max_bytes = current_app.config[self.config_key] + + try: + length = len(field.data.encode("utf-8")) + except UnicodeEncodeError as exc: + raise ValidationError(field.gettext("URL must be valid UTF-8.")) from exc + + if length > max_bytes: + raise ValidationError( + self.message or field.gettext(f"URL must not exceed {max_bytes} bytes.") + ) + + +class StrictHTTPURL: + """Require an HTTP(S) URL that httpx can parse without rewriting whitespace. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or is_control_character(character) + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + if parsed.scheme not in ("http", "https"): + raise StopValidation( + field.gettext(f"{field.name} must begin with http or https") + ) + + +class AllowedTargetHostname: + """Validate that a URL uses an allowed hostname. + + :param message: Optional validation error message. + """ + + def __init__(self, message: str | None = None) -> None: + """Initialize the validator. + + :param message: Optional validation error message. + """ + self.message = message + + def __call__(self, form: BaseForm, field: Field) -> None: + """Validate that a URL uses an allowed hostname. + + :param form: Form containing the field. + :param field: Field containing the URL to validate. + :raises ValidationError: If the URL hostname is not configured as allowed. + """ + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError: + hostname = None + + if hostname in current_app.config["ALLOWED_TARGET_HOSTNAMES"]: + return + raise ValidationError(self.message or field.gettext("Invalid input.")) + + +class RejectNonPublicIPLiteral: + """Reject non-public IP literals without performing DNS in the web worker.""" + + def __call__(self, form: BaseForm, field: Field) -> None: + try: + hostname = normalize_hostname(field.data, from_url=True) + except ValueError as exc: + raise ValidationError(field.gettext("Invalid input.")) from exc + + try: + address = ipaddress.ip_address(hostname) + except ValueError: + return # Named hosts are resolved only during the background fetch. + + if not is_public_address(address): + raise ValidationError(field.gettext("URL must use a public address.")) + + +class CanonicalDate: + """Require HTML date inputs to use the canonical YYYY-MM-DD format.""" + + def __call__(self, form: BaseForm, field: DateField) -> None: + if ( + field.data is not None + and field.raw_data + and field.raw_data[0] != field.data.isoformat() + ): + raise ValidationError(field.gettext("Invalid date.")) diff --git a/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py new file mode 100644 index 0000000..e66dc70 --- /dev/null +++ b/webmentions_ssg/migrations/versions/a4f91c2e08d7_rename_source_lists_to_domain_lists.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rename source block and allow lists to domain lists. + +Revision ID: a4f91c2e08d7 +Revises: b8179a2e46c3 +""" + +from alembic import op + +revision = "a4f91c2e08d7" +down_revision = "b8179a2e46c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Preserve existing domain rules when renaming their tables.""" + op.rename_table("blocked_sources", "blocked_domains") + op.rename_table("allowed_sources", "allowed_domains") + + +def downgrade() -> None: + """Restore the former table names without deleting domain rules.""" + op.rename_table("allowed_domains", "allowed_sources") + op.rename_table("blocked_domains", "blocked_sources") diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 8518b15..2917cb8 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,8 +11,8 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.models import source_target_digest as received_url_digest -from webmentions_ssg.models import target_digest as sent_target_digest +from webmentions_ssg.models.utils import source_target_digest as received_url_digest +from webmentions_ssg.models.utils import target_digest as sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py deleted file mode 100644 index 2304788..0000000 --- a/webmentions_ssg/models.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Define database models for users and incoming and outgoing Webmentions.""" - -import json -from datetime import UTC, datetime -from enum import StrEnum -from hashlib import sha256 -from typing import Never -from uuid import UUID, uuid7 - -import sqlalchemy as sa -from flask_login import UserMixin -from sqlalchemy import ( - Boolean, - DateTime, - Enum, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - Uuid, -) -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Mapped, mapped_column, relationship, validates -from werkzeug.security import check_password_hash, generate_password_hash - -from .extensions import Base - - -class ReceivedWebmentionStatus(StrEnum): - """Persisted lifecycle state of an incoming Webmention.""" - - RECEIVED = "received" - VERIFYING = "verifying" - VERIFIED = "verified" - FAILED = "failed" - SOURCE_REMOVED = "source_removed" - - -class SentWebmentionStatus(StrEnum): - """Represent the outcome of a Webmention send attempt.""" - - SENT = "sent" - NO_ENDPOINT = "no_endpoint" - FAILED = "failed" - - -class User(UserMixin, Base): - """Represent an authenticated application user.""" - - __tablename__ = "users" - - id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column(Text(), index=True, unique=True) - password_hash: Mapped[str] = mapped_column(Text(), nullable=False) - - def __repr__(self) -> str: - """Represent the user by username.""" - return f"" - - @property - def password(self) -> Never: - """Prevent access to the user's plain-text password. - - :raises AttributeError: Always, because the password is write-only. - """ - raise AttributeError("Password is write-only") - - @password.setter - def password(self, password: str) -> None: - """Hash and store a new password. - - :param password: Plain-text password to hash. - """ - self.password_hash = generate_password_hash(password) - - def get_id(self) -> str: - """Bind the login session to the current stored password hash. - - Only a one-way fingerprint is placed in Flask's signed cookie, never - the password hash itself. Rehashing the password invalidates all - previously issued session identifiers without a schema change. - - :returns: Session identifier combining the user ID and password-hash - fingerprint. - """ - fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() - return f"{self.id}:{fingerprint}" - - def check_password(self, password: str) -> bool: - """Check a plain-text password against the stored password hash. - - :param password: Plain-text password to check. - :returns: Whether the password matches the stored hash. - """ - return check_password_hash(self.password_hash, password) - - -class BlockedSource(Base): - """A source domain denied before verification or outbound fetching.""" - - __tablename__ = "blocked_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class AllowedSource(Base): - """A source domain accepted when allowlist mode is enabled.""" - - __tablename__ = "allowed_sources" - - id: Mapped[int] = mapped_column(primary_key=True) - domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - -class ReceivedWebmention(Base): - """Represent a Webmention received for verification.""" - - __tablename__ = "received_webmentions" - - uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) - - source: Mapped[str] = mapped_column(Text(), nullable=False) - target: Mapped[str] = mapped_column(Text(), nullable=False) - source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("source", "target") - def _refresh_url_digest(self, key: str, value: str) -> str: - source = value if key == "source" else self.__dict__.get("source") - target = value if key == "target" else self.__dict__.get("target") - if source is not None and target is not None: - self.source_target_digest = source_target_digest(source, target) - return value - - status: Mapped[ReceivedWebmentionStatus] = mapped_column( - Enum( - ReceivedWebmentionStatus, - name="received_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=False, - default=ReceivedWebmentionStatus.RECEIVED, - ) - # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh - # explicit submission resets this counter. - recovery_attempts: Mapped[int] = mapped_column( - Integer, nullable=False, default=0, server_default="0" - ) - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - default=lambda: datetime.now(UTC), - onupdate=lambda: datetime.now(UTC), - nullable=False, - ) - - notification_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - __table_args__ = ( - UniqueConstraint( - "source_target_digest", name="uq_webmention_source_target_digest" - ), - Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), - ) - - def __repr__(self) -> str: - """Represent the received Webmention by source and target.""" - return f" {self.target!r}>" - - @staticmethod - def reopen_verification_values( - *, at: datetime, recovery_attempts: int = 0 - ) -> dict[str, object]: - """Return the columns to reset when verification is explicitly reopened. - - :param at: Timestamp to record for the newly opened verification. - :param recovery_attempts: Number of stale-lease recovery attempts to retain. - :returns: Column values for resetting the verification state. - """ - return { - "status": ReceivedWebmentionStatus.RECEIVED, - "failure_reason": None, - "recovery_attempts": recovery_attempts, - "updated_at": at, - } - - def reopen_verification(self) -> None: - """Reopen an existing row for manual verification. - - Clear the previous outcome and restore the received state with a fresh - timestamp. - """ - for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): - setattr(self, key, value) - - @property - def verified(self) -> bool: - """Return whether the Webmention has been successfully verified.""" - return self.status == ReceivedWebmentionStatus.VERIFIED - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SourceDocument(Base): - """Represent a generated source document containing outgoing Webmentions.""" - - __tablename__ = "source_documents" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) - - content_hash: Mapped[str] = mapped_column(String(32), nullable=False) - revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) - - last_seen_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - revised_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - deleted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - sent_webmentions: Mapped[list[SentWebmention]] = relationship( - back_populates="source", cascade="all, delete-orphan" - ) - - def __repr__(self) -> str: - """Represent the source by its public URL.""" - return f"" - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the source UUID.""" - return uuid7_to_datetime(self.uuid) - - -class SentWebmention(Base): - """Represent an outgoing Webmention associated with a source.""" - - __tablename__ = "sent_webmentions" - - uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), primary_key=True, default=uuid7 - ) - - source_uuid: Mapped[UUID] = mapped_column( - Uuid(as_uuid=True), - ForeignKey("source_documents.uuid", ondelete="CASCADE"), - nullable=False, - index=True, - ) - - target: Mapped[str] = mapped_column(Text(), nullable=False) - target_digest: Mapped[str] = mapped_column(String(64), nullable=False) - - @validates("target") - def _update_target_digest(self, key: str, value: str) -> str: - self.target_digest = target_digest(value) - return value - - linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) - - desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) - processed_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - sent_revision: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - - status: Mapped[SentWebmentionStatus | None] = mapped_column( - Enum( - SentWebmentionStatus, - name="sent_webmention_status", - values_callable=lambda enum_type: [member.value for member in enum_type], - native_enum=False, - ), - nullable=True, - default=None, - ) - - failure_reason: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - - webmention_endpoint: Mapped[str | None] = mapped_column( - Text(), nullable=True, default=None - ) - endpoint_response_status: Mapped[int | None] = mapped_column( - Integer(), nullable=True, default=None - ) - status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) - - # Retry state belongs to the desired revision, not to the target forever. - attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) - next_retry_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - last_attempted_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - last_sent_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, default=None - ) - - source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") - - __table_args__ = ( - UniqueConstraint( - "source_uuid", - "target_digest", - name="uq_sent_webmention_source_uuid_target_digest", - ), - ) - - def __repr__(self) -> str: - """Represent the sent Webmention by its target URL.""" - return f"" - - def reset_for_revision(self, revision: int, linked: bool) -> None: - """Reopen delivery for a revision while preserving send history. - - :param revision: Source revision to make the desired delivery revision. - :param linked: Whether the target still appears in the current source. - """ - self.desired_revision = revision - self.linked = linked - self.attempt_count = 0 - self.next_retry_at = None - - def clear_result_for_resend(self) -> None: - """Retry the current revision, clearing its previous delivery result. - - Reset attempt and response state without changing the desired source revision. - """ - self.reset_for_revision(self.desired_revision, self.linked) - self.processed_revision = None - self.status = None - self.failure_reason = None - self.webmention_endpoint = None - self.endpoint_response_status = None - self.status_url = None - - @hybrid_property - def awaiting_processing(self) -> bool: - """Return whether the Webmention has an unprocessed source revision.""" - return ( - self.processed_revision is None - or self.processed_revision < self.desired_revision - ) - - @awaiting_processing.inplace.expression - @classmethod - def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: - """Express pending-revision status in database queries. - - :returns: SQLAlchemy expression selecting unprocessed source revisions. - """ - return sa.or_( - cls.processed_revision.is_(None), - cls.processed_revision < cls.desired_revision, - ) - - @property - def created_at(self) -> datetime: - """Return the creation time encoded in the Webmention UUID.""" - return uuid7_to_datetime(self.uuid) - - -def uuid7_to_datetime(identifier: UUID) -> datetime: - """Convert a UUID version 7 timestamp to a datetime. - - :param identifier: UUID whose embedded timestamp should be converted. - :returns: Timestamp as a timezone-aware UTC datetime. - """ - return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) - - -def source_target_digest(source: str, target: str) -> str: - """Fingerprint an unambiguous, ordered source/target pair. - - :param source: Webmention source URL. - :param target: Webmention target URL. - :returns: SHA-256 hexadecimal digest of the ordered URL pair. - """ - encoded = json.dumps( - [source, target], ensure_ascii=False, separators=(",", ":") - ).encode("utf-8") - return sha256(encoded).hexdigest() - - -def target_digest(target: str) -> str: - """Fingerprint an outgoing target (unique within each source ID). - - :param target: Outgoing Webmention target URL. - :returns: SHA-256 hexadecimal digest of the target URL. - """ - return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/models/__init__.py b/webmentions_ssg/models/__init__.py new file mode 100644 index 0000000..360fc71 --- /dev/null +++ b/webmentions_ssg/models/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Database models grouped by feature.""" diff --git a/webmentions_ssg/models/moderation.py b/webmentions_ssg/models/moderation.py new file mode 100644 index 0000000..4200c39 --- /dev/null +++ b/webmentions_ssg/models/moderation.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Domain allowlist and blocklist models.""" + +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column + +from ..extensions import Base + + +class BlockedDomain(Base): + """Source domain rejected before submission or verification.""" + + __tablename__ = "blocked_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + +class AllowedDomain(Base): + """A domain accepted when allowlist mode is enabled.""" + + __tablename__ = "allowed_domains" + + id: Mapped[int] = mapped_column(primary_key=True) + domain: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) diff --git a/webmentions_ssg/models/received.py b/webmentions_ssg/models/received.py new file mode 100644 index 0000000..a9047ad --- /dev/null +++ b/webmentions_ssg/models/received.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Incoming Webmention state and persistence.""" + +from datetime import UTC, datetime +from enum import StrEnum +from uuid import UUID + +from sqlalchemy import ( + DateTime, + Enum, + Index, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.orm import Mapped, mapped_column, validates + +from ..extensions import Base +from .utils import source_target_digest, uuid7_to_datetime + + +class ReceivedWebmentionStatus(StrEnum): + """Persisted lifecycle state of an incoming Webmention.""" + + RECEIVED = "received" + VERIFYING = "verifying" + VERIFIED = "verified" + FAILED = "failed" + SOURCE_REMOVED = "source_removed" + + +class ReceivedWebmention(Base): + """Represent a Webmention received for verification.""" + + __tablename__ = "received_webmentions" + + uuid: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) + + source: Mapped[str] = mapped_column(Text(), nullable=False) + target: Mapped[str] = mapped_column(Text(), nullable=False) + source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("source", "target") + def _refresh_url_digest(self, key: str, value: str) -> str: + source = value if key == "source" else self.__dict__.get("source") + target = value if key == "target" else self.__dict__.get("target") + if source is not None and target is not None: + self.source_target_digest = source_target_digest(source, target) + return value + + status: Mapped[ReceivedWebmentionStatus] = mapped_column( + Enum( + ReceivedWebmentionStatus, + name="received_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=False, + default=ReceivedWebmentionStatus.RECEIVED, + ) + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + nullable=False, + ) + + notification_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + __table_args__ = ( + UniqueConstraint( + "source_target_digest", name="uq_webmention_source_target_digest" + ), + Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), + ) + + def __repr__(self) -> str: + """Represent the received Webmention by source and target.""" + return f" {self.target!r}>" + + @staticmethod + def reopen_verification_values( + *, at: datetime, recovery_attempts: int = 0 + ) -> dict[str, object]: + """Return the columns to reset when verification is explicitly reopened. + + :param at: Timestamp to record for the newly opened verification. + :param recovery_attempts: Number of stale-lease recovery attempts to retain. + :returns: Column values for resetting the verification state. + """ + return { + "status": ReceivedWebmentionStatus.RECEIVED, + "failure_reason": None, + "recovery_attempts": recovery_attempts, + "updated_at": at, + } + + def reopen_verification(self) -> None: + """Reopen an existing row for manual verification. + + Clear the previous outcome and restore the received state with a fresh + timestamp. + """ + for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): + setattr(self, key, value) + + @property + def verified(self) -> bool: + """Return whether the Webmention has been successfully verified.""" + return self.status == ReceivedWebmentionStatus.VERIFIED + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/sources.py b/webmentions_ssg/models/sources.py new file mode 100644 index 0000000..f0fd618 --- /dev/null +++ b/webmentions_ssg/models/sources.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source documents and outgoing Webmention state.""" + +from datetime import datetime +from enum import StrEnum +from uuid import UUID, uuid7 + +import sqlalchemy as sa +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates + +from ..extensions import Base +from .utils import target_digest, uuid7_to_datetime + + +class SentWebmentionStatus(StrEnum): + """Represent the outcome of a Webmention send attempt.""" + + SENT = "sent" + NO_ENDPOINT = "no_endpoint" + FAILED = "failed" + + +class SourceDocument(Base): + """Represent a generated source document containing outgoing Webmentions.""" + + __tablename__ = "source_documents" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + content_hash: Mapped[str] = mapped_column(String(32), nullable=False) + revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) + + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + revised_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + sent_webmentions: Mapped[list[SentWebmention]] = relationship( + back_populates="source", cascade="all, delete-orphan" + ) + + def __repr__(self) -> str: + """Represent the source by its public URL.""" + return f"" + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the source UUID.""" + return uuid7_to_datetime(self.uuid) + + +class SentWebmention(Base): + """Represent an outgoing Webmention associated with a source.""" + + __tablename__ = "sent_webmentions" + + uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid7 + ) + + source_uuid: Mapped[UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("source_documents.uuid", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + target: Mapped[str] = mapped_column(Text(), nullable=False) + target_digest: Mapped[str] = mapped_column(String(64), nullable=False) + + @validates("target") + def _update_target_digest(self, key: str, value: str) -> str: + self.target_digest = target_digest(value) + return value + + linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + + desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) + processed_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + sent_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + + status: Mapped[SentWebmentionStatus | None] = mapped_column( + Enum( + SentWebmentionStatus, + name="sent_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), + nullable=True, + default=None, + ) + + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + webmention_endpoint: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + endpoint_response_status: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column(Integer(), nullable=False, default=0) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + last_attempted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") + + __table_args__ = ( + UniqueConstraint( + "source_uuid", + "target_digest", + name="uq_sent_webmention_source_uuid_target_digest", + ), + ) + + def __repr__(self) -> str: + """Represent the sent Webmention by its target URL.""" + return f"" + + def reset_for_revision(self, revision: int, linked: bool) -> None: + """Reopen delivery for a revision while preserving send history. + + :param revision: Source revision to make the desired delivery revision. + :param linked: Whether the target still appears in the current source. + """ + self.desired_revision = revision + self.linked = linked + self.attempt_count = 0 + self.next_retry_at = None + + def clear_result_for_resend(self) -> None: + """Retry the current revision, clearing its previous delivery result. + + Reset attempt and response state without changing the desired source revision. + """ + self.reset_for_revision(self.desired_revision, self.linked) + self.processed_revision = None + self.status = None + self.failure_reason = None + self.webmention_endpoint = None + self.endpoint_response_status = None + self.status_url = None + + @hybrid_property + def awaiting_processing(self) -> bool: + """Return whether the Webmention has an unprocessed source revision.""" + return ( + self.processed_revision is None + or self.processed_revision < self.desired_revision + ) + + @awaiting_processing.inplace.expression + @classmethod + def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: + """Express pending-revision status in database queries. + + :returns: SQLAlchemy expression selecting unprocessed source revisions. + """ + return sa.or_( + cls.processed_revision.is_(None), + cls.processed_revision < cls.desired_revision, + ) + + @property + def created_at(self) -> datetime: + """Return the creation time encoded in the Webmention UUID.""" + return uuid7_to_datetime(self.uuid) diff --git a/webmentions_ssg/models/user.py b/webmentions_ssg/models/user.py new file mode 100644 index 0000000..b09c3d1 --- /dev/null +++ b/webmentions_ssg/models/user.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Application user and session identity model.""" + +from hashlib import sha256 +from typing import Never + +from flask_login import UserMixin +from sqlalchemy import Text +from sqlalchemy.orm import Mapped, mapped_column +from werkzeug.security import check_password_hash, generate_password_hash + +from ..extensions import Base + + +class User(UserMixin, Base): + """Represent an authenticated application user.""" + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(Text(), index=True, unique=True) + password_hash: Mapped[str] = mapped_column(Text(), nullable=False) + + def __repr__(self) -> str: + """Represent the user by username.""" + return f"" + + @property + def password(self) -> Never: + """Prevent access to the user's plain-text password. + + :raises AttributeError: Always, because the password is write-only. + """ + raise AttributeError("Password is write-only") + + @password.setter + def password(self, password: str) -> None: + """Hash and store a new password. + + :param password: Plain-text password to hash. + """ + self.password_hash = generate_password_hash(password) + + def get_id(self) -> str: + """Bind the login session to the current stored password hash. + + Only a one-way fingerprint is placed in Flask's signed cookie, never + the password hash itself. Rehashing the password invalidates all + previously issued session identifiers without a schema change. + + :returns: Session identifier combining the user ID and password-hash + fingerprint. + """ + fingerprint = sha256(self.password_hash.encode("utf-8")).hexdigest() + return f"{self.id}:{fingerprint}" + + def check_password(self, password: str) -> bool: + """Check a plain-text password against the stored password hash. + + :param password: Plain-text password to check. + :returns: Whether the password matches the stored hash. + """ + return check_password_hash(self.password_hash, password) diff --git a/webmentions_ssg/models/utils.py b/webmentions_ssg/models/utils.py new file mode 100644 index 0000000..e961088 --- /dev/null +++ b/webmentions_ssg/models/utils.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""URL fingerprints and UUID timestamp utilities.""" + +import json +from datetime import UTC, datetime +from hashlib import sha256 +from uuid import UUID + + +def uuid7_to_datetime(identifier: UUID) -> datetime: + """Convert a UUID version 7 timestamp to a datetime. + + :param identifier: UUID whose embedded timestamp should be converted. + :returns: Timestamp as a timezone-aware UTC datetime. + """ + return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) + + +def source_target_digest(source: str, target: str) -> str: + """Fingerprint an unambiguous, ordered source/target pair. + + :param source: Webmention source URL. + :param target: Webmention target URL. + :returns: SHA-256 hexadecimal digest of the ordered URL pair. + """ + encoded = json.dumps( + [source, target], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def target_digest(target: str) -> str: + """Fingerprint an outgoing target (unique within each source ID). + + :param target: Outgoing Webmention target URL. + :returns: SHA-256 hexadecimal digest of the target URL. + """ + return sha256(target.encode("utf-8")).hexdigest() diff --git a/webmentions_ssg/moderation_policy.py b/webmentions_ssg/moderation_policy.py new file mode 100644 index 0000000..51e7995 --- /dev/null +++ b/webmentions_ssg/moderation_policy.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Admission policy for incoming Webmention source hostnames.""" + +import ipaddress + +import sqlalchemy as sa +from flask import current_app + +from .extensions import db +from .hostname import normalize_hostname +from .models.moderation import AllowedDomain, BlockedDomain + + +def domain_from_url(source: str) -> str | None: + """Extract and canonicalize the hostname without a network request.""" + try: + return normalize_hostname(source, from_url=True) + except ValueError: + return None + + +def has_matching_domain( + model: type[AllowedDomain | BlockedDomain], domain: str +) -> bool: + """Match the hostname itself and parent domains, never partial labels.""" + candidates = [domain] + # An IP address can only match itself, not a suffix of its textual form. + try: + ipaddress.ip_address(domain) + except ValueError: + parts = domain.split(".") + candidates.extend(".".join(parts[i:]) for i in range(1, len(parts))) + + return ( + db.session.scalar( + sa.select(model.id).where(model.domain.in_(candidates)).limit(1) + ) + is not None + ) + + +def rejection_reason(source: str) -> str | None: + """Return the reason a source cannot be received, or ``None`` if accepted. + + The blocklist always wins. In allowlist mode an empty allowlist rejects + every source. The database, not a stale per-worker cache, is consulted. + """ + if (domain := domain_from_url(source)) is None: + return "Invalid source hostname" + + if has_matching_domain(BlockedDomain, domain): + return "Source domain is blocked" + + if current_app.config["ALLOWLIST_ENABLED"] and not has_matching_domain( + AllowedDomain, domain + ): + return "Source domain is not allowlisted" + + return None diff --git a/webmentions_ssg/source_policy.py b/webmentions_ssg/source_policy.py deleted file mode 100644 index 2c62527..0000000 --- a/webmentions_ssg/source_policy.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Admission policy for incoming Webmention source hostnames.""" - -import ipaddress - -import sqlalchemy as sa -from flask import current_app - -from .extensions import db -from .hostname import normalize_hostname -from .models import AllowedSource, BlockedSource - - -def source_domain_from_url(source: str) -> str | None: - """Extract and canonicalize the source hostname without a network request.""" - try: - return normalize_hostname(source, from_url=True) - except ValueError: - return None - - -def has_matching_domain( - model: type[AllowedSource | BlockedSource], domain: str -) -> bool: - """Match the hostname itself and parent domains, never partial labels.""" - candidates = [domain] - # An IP address can only match itself, not a suffix of its textual form. - try: - ipaddress.ip_address(domain) - except ValueError: - parts = domain.split(".") - candidates.extend(".".join(parts[i:]) for i in range(1, len(parts))) - - return ( - db.session.scalar( - sa.select(model.id).where(model.domain.in_(candidates)).limit(1) - ) - is not None - ) - - -def source_rejection_reason(source: str) -> str | None: - """Return the reason a source cannot be received, or None when accepted. - - The blocklist always wins. In allowlist mode an empty allowlist rejects - every source. The database, not a stale per-worker cache, is consulted. - """ - if (domain := source_domain_from_url(source)) is None: - return "Invalid source hostname" - - if has_matching_domain(BlockedSource, domain): - return "Source domain is blocked" - - if current_app.config["SOURCE_ALLOWLIST_ENABLED"] and not has_matching_domain( - AllowedSource, domain - ): - return "Source domain is not allowlisted" - - return None diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index c503e1f..6a0e6b9 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,7 +11,7 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def email_verified_digest() -> None: diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index b882493..b4b6a83 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 6869477..1f98702 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,7 +14,7 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, SourceDocument +from ..models.sources import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 6b14ace..fc70f27 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -15,7 +15,7 @@ from huey.exceptions import TaskLockedException from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import SentWebmention, SentWebmentionStatus +from ..models.sources import SentWebmention, SentWebmentionStatus from ..net import ( AddressResolutionError, UnsafeAddressError, diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py index d55608c..ca45777 100644 --- a/webmentions_ssg/tasks/verification.py +++ b/webmentions_ssg/tasks/verification.py @@ -15,7 +15,8 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..models.received import ReceivedWebmention, ReceivedWebmentionStatus +from ..moderation_policy import rejection_reason from ..net import ( AddressResolutionError, ResponseTooLargeError, @@ -25,7 +26,6 @@ from ..net import ( media_type_of, read_limited_body, ) -from ..source_policy import source_rejection_reason from ..version import APP_NAME, VERSION from .target_matching import html_mentions_target, text_mentions_target @@ -201,7 +201,7 @@ def determine_verification_outcome( :raises httpx.RequestError: If a transient HTTP request error should be retried. """ # Recheck at worker time: a source may have been blocked since submission. - if reason := source_rejection_reason(claim.source): + if reason := rejection_reason(claim.source): return ReceivedWebmentionStatus.FAILED, reason try: diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index d9c8e87..3395dd5 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -45,8 +45,8 @@ SPDX-License-Identifier: BSD-3-Clause