From a285fdf29e59c3b957247ea9241943c7202e9a28 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 13 Nov 2026 21:20:40 +0100 Subject: refactor(core): consolidate repeated helper logic Centralize SQL row counts, hostname parsing, and content-type normalization so tasks, forms, and scanners share consistent implementations. Simplify periodic task registration and admin lookups with table-driven setup and Flask-SQLAlchemy's built-in 404 helper. --- tests/test_db_helpers.py | 28 ++++++++++++++++++++++++++++ tests/test_forms.py | 12 ++++++++++++ tests/test_net.py | 33 +++++++++++++++++++++++++++++++++ webmentions_ssg/db_helpers.py | 14 ++++++++++++++ webmentions_ssg/forms.py | 14 +++----------- webmentions_ssg/net.py | 16 +++++++++++++++- webmentions_ssg/tasks/consumer.py | 25 ++++++++----------------- webmentions_ssg/tasks/discovery.py | 5 ++--- webmentions_ssg/tasks/receiver.py | 11 +++++------ webmentions_ssg/tasks/retention.py | 29 ++++++++++++----------------- webmentions_ssg/tasks/sender.py | 4 ++-- webmentions_ssg/tasks/source_parser.py | 6 +++--- webmentions_ssg/views/admin.py | 21 +++++---------------- webmentions_ssg/views/endpoint.py | 32 +++++++++++++------------------- 14 files changed, 155 insertions(+), 95 deletions(-) create mode 100644 tests/test_db_helpers.py create mode 100644 webmentions_ssg/db_helpers.py diff --git a/tests/test_db_helpers.py b/tests/test_db_helpers.py new file mode 100644 index 0000000..ac97f2b --- /dev/null +++ b/tests/test_db_helpers.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression tests for shared SQLAlchemy row-count handling.""" + +from sqlalchemy import create_engine, text + +from webmentions_ssg.db_helpers import rows_affected + + +def test_rows_affected_for_update_and_delete() -> None: + engine = create_engine("sqlite://") + try: + with engine.begin() as connection: + connection.execute(text("CREATE TABLE entries (id INTEGER PRIMARY KEY)")) + connection.execute(text("INSERT INTO entries (id) VALUES (1)")) + + result = connection.execute(text("UPDATE entries SET id = 2 WHERE id = 1")) + assert rows_affected(result) == 1 + + result = connection.execute(text("DELETE FROM entries WHERE id = 1")) + assert rows_affected(result) == 0 + + result = connection.execute(text("DELETE FROM entries WHERE id = 2")) + assert rows_affected(result) == 1 + finally: + engine.dispose() diff --git a/tests/test_forms.py b/tests/test_forms.py index a37a6d0..e636eae 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -178,6 +178,18 @@ def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None: assert "URL must use a public address." in form.errors["source"] +def test_endpoint_rejects_private_ip_literal_with_trailing_dot(app: Flask) -> None: + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm( + formdata=MultiDict( + {"source": "http://127.0.0.1./", "target": VALID_TARGET} + ), + meta={"csrf": False}, + ) + assert not form.validate() + assert "URL must use a public address." in form.errors["source"] + + def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: with app.test_request_context("/endpoint", method="POST"): form = EndpointForm( diff --git a/tests/test_net.py b/tests/test_net.py index 8e0744c..42b9261 100644 --- a/tests/test_net.py +++ b/tests/test_net.py @@ -21,6 +21,8 @@ from webmentions_ssg.net import ( PublicNetworkBackend, ResponseTooLargeError, UnsafeAddressError, + content_type_of, + hostname_of, is_http_url, is_public_address, public_addresses, @@ -333,3 +335,34 @@ def test_temporary_http_status(app: Flask, status: int, expected: bool) -> None: from webmentions_ssg.tasks import discovery assert discovery.is_temporary_status(status) is expected + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + pytest.param("https://EXAMPLE.com./path", "example.com", id="trailing-dot"), + pytest.param("http://[2001:db8::1]/", "2001:db8::1", id="ipv6"), + pytest.param("https:///path", None, id="missing-host"), + pytest.param("http://[::1", None, id="malformed-ipv6"), + ], +) +def test_hostname_of(url: str, expected: str | None) -> None: + assert hostname_of(url) == expected + + +@pytest.mark.parametrize( + ("headers", "expected"), + [ + pytest.param({}, "", id="missing"), + pytest.param( + {"Content-Type": " Text/HTML ; charset=UTF-8"}, "text/html", id="params" + ), + pytest.param( + {"content-type": "APPLICATION/XHTML+XML"}, + "application/xhtml+xml", + id="case", + ), + ], +) +def test_content_type_of(headers: dict[str, str], expected: str) -> None: + assert content_type_of(httpx.Headers(headers)) == expected diff --git a/webmentions_ssg/db_helpers.py b/webmentions_ssg/db_helpers.py new file mode 100644 index 0000000..6e1c2d2 --- /dev/null +++ b/webmentions_ssg/db_helpers.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for interpreting SQLAlchemy database results.""" + +from typing import Any, cast + +from sqlalchemy.engine import CursorResult, Result + + +def rows_affected(result: Result[Any]) -> int: + """Return the row count from a SQL UPDATE or DELETE result.""" + return cast(CursorResult[Any], result).rowcount diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py index 005bd20..f5fba51 100644 --- a/webmentions_ssg/forms.py +++ b/webmentions_ssg/forms.py @@ -5,7 +5,6 @@ """Webmention forms and URL validation.""" import ipaddress -from urllib.parse import urlsplit import httpx from flask import current_app @@ -14,7 +13,7 @@ from wtforms import Field, PasswordField, StringField, SubmitField, ValidationEr from wtforms.form import BaseForm from wtforms.validators import InputRequired, StopValidation -from .net import is_public_address +from .net import hostname_of, is_public_address from .validation import is_control_character @@ -96,10 +95,7 @@ class AllowedHostname: :param field: Field containing the URL to validate. :raises ValidationError: If the URL hostname is not configured as allowed. """ - try: - hostname = (urlsplit(field.data).hostname or "").rstrip(".") - except ValueError, TypeError: - hostname = None + hostname = hostname_of(field.data) if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]: return raise ValidationError(self.message or field.gettext("Invalid input.")) @@ -109,11 +105,7 @@ class PublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" def __call__(self, form: BaseForm, field: Field) -> None: - try: - hostname = urlsplit(field.data).hostname - except (ValueError, TypeError) as exc: - raise ValidationError(field.gettext("Invalid input.")) from exc - + hostname = hostname_of(field.data) if hostname is None: raise ValidationError(field.gettext("Invalid input.")) diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py index 7c29edc..02c7f2b 100644 --- a/webmentions_ssg/net.py +++ b/webmentions_ssg/net.py @@ -7,7 +7,7 @@ import ipaddress import socket import time -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from contextlib import ExitStack from http import HTTPStatus from urllib.parse import urlsplit @@ -252,6 +252,20 @@ def public_addresses( return tuple(str(address) for address in addresses) +def hostname_of(url: str) -> str | None: + """Return a normalized URL hostname, or None for malformed URLs.""" + try: + hostname = urlsplit(url).hostname + except (TypeError, ValueError): + return None + return hostname.rstrip(".") if hostname is not None else None + + +def content_type_of(headers: Mapping[str, str]) -> str: + """Return a normalized media type without Content-Type parameters.""" + return headers.get("Content-Type", "").partition(";")[0].strip().lower() + + def is_http_url(url: str) -> bool: """Check whether a URL is an absolute HTTP or HTTPS URL. diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 474e775..362ebe8 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,23 +21,14 @@ def create_consumer() -> Huey: """ app = create_app() - schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) - queue.periodic_task(schedule)(scanner.scan_sources) - - recovery_schedule = crontab( - *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions) - - notification_schedule = crontab( - *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(notification_schedule)(notifications.send_webmention_digest) - - retention_schedule = crontab( - *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(retention_schedule)(retention.purge_old_received_webmentions) + for config_key, task in ( + ("SCANNER_SCHEDULE", scanner.scan_sources), + ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions), + ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest), + ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), + ): + schedule = crontab(*app.config[config_key].split(), strict=True) + queue.periodic_task(schedule)(task) return queue.huey diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py index 107d30a..becac58 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/discovery.py @@ -14,6 +14,7 @@ from flask import current_app from ..net import ( ResponseTooLargeError, + content_type_of, is_http_url, is_temporary_status, read_limited_body, @@ -173,9 +174,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non if not response.is_success: raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) + media_type = content_type_of(response.headers) if media_type not in {"text/html", "application/xhtml+xml"}: return None diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 78089af..1df363f 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -7,12 +7,12 @@ import uuid from datetime import UTC, datetime, timedelta from http import HTTPStatus -from typing import Any, cast import httpx import sqlalchemy as sa from flask import current_app +from ..db_helpers import rows_affected from ..extensions import db, huey from ..models import ReceivedStatus, ReceivedWebmention from ..net import ( @@ -20,6 +20,7 @@ from ..net import ( ResponseTooLargeError, UnsafeAddressError, build_client, + content_type_of, is_temporary_status, read_limited_body, ) @@ -103,9 +104,7 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: """ response, body = fetch_source(source_url) - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) + media_type = content_type_of(response.headers) match media_type: case "text/html" | "application/xhtml+xml": @@ -172,7 +171,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) ) - if cast(sa.CursorResult[Any], claimed).rowcount != 1: + if rows_affected(claimed) != 1: db.session.rollback() return @@ -285,7 +284,7 @@ def recover_stale_webmentions() -> int: ) ) - if cast(sa.CursorResult[Any], changed).rowcount != 1: + if rows_affected(changed) != 1: db.session.rollback() continue diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 754c891..86283b7 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -5,12 +5,11 @@ """Delete old received Webmentions in terminal unsuccessful states.""" from datetime import UTC, datetime, timedelta -from typing import Any, cast import sqlalchemy as sa from flask import current_app -from sqlalchemy.engine import CursorResult +from ..db_helpers import rows_affected from ..extensions import db from ..models import ReceivedStatus, ReceivedWebmention @@ -26,22 +25,18 @@ def purge_old_received_webmentions() -> int: days=current_app.config["RECEIVED_RETENTION_DAYS"] ) - result = cast( - CursorResult[Any], - db.session.execute( - sa.delete(ReceivedWebmention).where( - ReceivedWebmention.status.in_( - (ReceivedStatus.FAILED, ReceivedStatus.DELETED) - ), - ReceivedWebmention.updated_at < cutoff, - ) - ), + result = db.session.execute( + sa.delete(ReceivedWebmention).where( + ReceivedWebmention.status.in_( + (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ), + ReceivedWebmention.updated_at < cutoff, + ) ) db.session.commit() - if result.rowcount: - current_app.logger.info( - "Purged %d old failed/deleted Webmentions", result.rowcount - ) + count = rows_affected(result) + if count: + current_app.logger.info("Purged %d old failed/deleted Webmentions", count) - return result.rowcount + return count diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 4f994d4..92d8dfa 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,13 +6,13 @@ import uuid from datetime import UTC, datetime, timedelta -from typing import Any, cast import httpx import sqlalchemy as sa from flask import current_app from huey.exceptions import TaskLockedException +from ..db_helpers import rows_affected from ..extensions import db, huey from ..models import SentWebmention, SentWebmentionStatus from ..net import ( @@ -159,7 +159,7 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: ) db.session.commit() - if cast(sa.CursorResult[Any], claim).rowcount != 1: + if rows_affected(claim) != 1: return db.session.refresh(webmention) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index 9cbe071..294402c 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -11,14 +11,14 @@ from fnmatch import fnmatchcase from itertools import chain from pathlib import Path from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit +from urllib.parse import quote, urldefrag, urljoin import mf2py import xxhash from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement -from ..net import is_http_url +from ..net import hostname_of, is_http_url REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") @@ -283,7 +283,7 @@ def iter_targets( for value in chain(hrefs, reactions): target = normalize_target(value, base_url=base_url, source_url=source_url) if target is not None: - hostname = (urlsplit(target).hostname or "").rstrip(".") + hostname = hostname_of(target) or "" if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): yield target diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py index 1ca06b5..794bcdf 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/admin.py @@ -54,10 +54,7 @@ def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: if not form.validate_on_submit(): abort(400) - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - abort(404) + webmention = db.get_or_404(ReceivedWebmention, identifier) db.session.delete(webmention) db.session.commit() @@ -84,10 +81,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: if not form.validate_on_submit(): abort(400) - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - abort(404) + webmention = db.get_or_404(ReceivedWebmention, identifier) webmention.reset_for_verification() @@ -145,12 +139,7 @@ def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue: :param source_identifier: Identifier of the source to display. :return: Rendered source details page. """ - source = db.session.scalar( - sa.select(Source).where(Source.uuid == source_identifier) - ) - - if source is None: - abort(404) + source = db.get_or_404(Source, source_identifier) webmentions = db.paginate( sa.select(SentWebmention) @@ -189,9 +178,9 @@ def resend_sent_webmention( if not form.validate_on_submit(): abort(400) - webmention = db.session.get(SentWebmention, webmention_identifier) + webmention = db.get_or_404(SentWebmention, webmention_identifier) - if webmention is None or webmention.source_id != source_identifier: + if webmention.source_id != source_identifier: abort(404) # Requeue the existing desired revision instead of inventing a new source diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index e3f1368..a7badfd 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -6,15 +6,15 @@ import uuid from datetime import UTC, datetime, timedelta -from typing import Any, cast +from typing import cast import sqlalchemy as sa import sqlalchemy.exc as sa_exc -from flask import Response, abort, current_app, render_template, url_for +from flask import Response, current_app, render_template, url_for from flask.typing import ResponseReturnValue -from sqlalchemy.engine import CursorResult from .. import forms +from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter from ..models import ReceivedWebmention, received_url_digest from ..tasks.receiver import verify_webmention @@ -63,20 +63,17 @@ def reopen_if_cooled_down(webmention: ReceivedWebmention) -> bool: seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"] ) - result = cast( - CursorResult[Any], - db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention.uuid, - ReceivedWebmention.updated_at <= cutoff, - ) - .values(ReceivedWebmention.verification_reset_values(at=now)) - .execution_options(synchronize_session=False) - ), + result = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention.uuid, + ReceivedWebmention.updated_at <= cutoff, + ) + .values(ReceivedWebmention.verification_reset_values(at=now)) + .execution_options(synchronize_session=False) ) db.session.commit() - return result.rowcount == 1 + return rows_affected(result) == 1 def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None: @@ -144,10 +141,7 @@ def status(identifier: uuid.UUID) -> ResponseReturnValue: :param identifier: Identifier of the Webmention to display. :return: Rendered Webmention status page. """ - webmention = db.session.get(ReceivedWebmention, identifier) - - if webmention is None: - abort(404) + webmention = db.get_or_404(ReceivedWebmention, identifier) return render_template( "status.html", title="Webmention status", webmention=webmention -- cgit v1.3.1