aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Fink2026-11-13 21:20:40 +0100
committerDennis Fink2026-11-13 21:20:40 +0100
commita285fdf29e59c3b957247ea9241943c7202e9a28 (patch)
tree17ceeb703de23af1e67081d99a8864bbe353fb46
parent78ecd8bc227bdcbbd0e45eea025f7223252ed0da (diff)
downloadwebmentions-ssg-a285fdf29e59c3b957247ea9241943c7202e9a28.tar.gz
webmentions-ssg-a285fdf29e59c3b957247ea9241943c7202e9a28.zip
refactor(core): consolidate repeated helper logic
Centralize SQL row counts, hostname parsing, and content-type normalization so tasks, forms, and scanners share consistent implementations. Simplify periodic task registration and admin lookups with table-driven setup and Flask-SQLAlchemy's built-in 404 helper.
Diffstat (limited to '')
-rw-r--r--tests/test_db_helpers.py28
-rw-r--r--tests/test_forms.py12
-rw-r--r--tests/test_net.py33
-rw-r--r--webmentions_ssg/db_helpers.py14
-rw-r--r--webmentions_ssg/forms.py14
-rw-r--r--webmentions_ssg/net.py16
-rw-r--r--webmentions_ssg/tasks/consumer.py25
-rw-r--r--webmentions_ssg/tasks/discovery.py5
-rw-r--r--webmentions_ssg/tasks/receiver.py11
-rw-r--r--webmentions_ssg/tasks/retention.py29
-rw-r--r--webmentions_ssg/tasks/sender.py4
-rw-r--r--webmentions_ssg/tasks/source_parser.py6
-rw-r--r--webmentions_ssg/views/admin.py21
-rw-r--r--webmentions_ssg/views/endpoint.py32
14 files changed, 155 insertions, 95 deletions
diff --git a/tests/test_db_helpers.py b/tests/test_db_helpers.py
new file mode 100644
index 0000000..ac97f2b
--- /dev/null
+++ b/tests/test_db_helpers.py
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Regression tests for shared SQLAlchemy row-count handling."""
+
+from sqlalchemy import create_engine, text
+
+from webmentions_ssg.db_helpers import rows_affected
+
+
+def test_rows_affected_for_update_and_delete() -> None:
+ engine = create_engine("sqlite://")
+ try:
+ with engine.begin() as connection:
+ connection.execute(text("CREATE TABLE entries (id INTEGER PRIMARY KEY)"))
+ connection.execute(text("INSERT INTO entries (id) VALUES (1)"))
+
+ result = connection.execute(text("UPDATE entries SET id = 2 WHERE id = 1"))
+ assert rows_affected(result) == 1
+
+ result = connection.execute(text("DELETE FROM entries WHERE id = 1"))
+ assert rows_affected(result) == 0
+
+ result = connection.execute(text("DELETE FROM entries WHERE id = 2"))
+ assert rows_affected(result) == 1
+ finally:
+ engine.dispose()
diff --git a/tests/test_forms.py b/tests/test_forms.py
index a37a6d0..e636eae 100644
--- a/tests/test_forms.py
+++ b/tests/test_forms.py
@@ -178,6 +178,18 @@ def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None:
assert "URL must use a public address." in form.errors["source"]
+def test_endpoint_rejects_private_ip_literal_with_trailing_dot(app: Flask) -> None:
+ with app.test_request_context("/endpoint", method="POST"):
+ form = EndpointForm(
+ formdata=MultiDict(
+ {"source": "http://127.0.0.1./", "target": VALID_TARGET}
+ ),
+ meta={"csrf": False},
+ )
+ assert not form.validate()
+ assert "URL must use a public address." in form.errors["source"]
+
+
def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None:
with app.test_request_context("/endpoint", method="POST"):
form = EndpointForm(
diff --git a/tests/test_net.py b/tests/test_net.py
index 8e0744c..42b9261 100644
--- a/tests/test_net.py
+++ b/tests/test_net.py
@@ -21,6 +21,8 @@ from webmentions_ssg.net import (
PublicNetworkBackend,
ResponseTooLargeError,
UnsafeAddressError,
+ content_type_of,
+ hostname_of,
is_http_url,
is_public_address,
public_addresses,
@@ -333,3 +335,34 @@ def test_temporary_http_status(app: Flask, status: int, expected: bool) -> None:
from webmentions_ssg.tasks import discovery
assert discovery.is_temporary_status(status) is expected
+
+
+@pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ pytest.param("https://EXAMPLE.com./path", "example.com", id="trailing-dot"),
+ pytest.param("http://[2001:db8::1]/", "2001:db8::1", id="ipv6"),
+ pytest.param("https:///path", None, id="missing-host"),
+ pytest.param("http://[::1", None, id="malformed-ipv6"),
+ ],
+)
+def test_hostname_of(url: str, expected: str | None) -> None:
+ assert hostname_of(url) == expected
+
+
+@pytest.mark.parametrize(
+ ("headers", "expected"),
+ [
+ pytest.param({}, "", id="missing"),
+ pytest.param(
+ {"Content-Type": " Text/HTML ; charset=UTF-8"}, "text/html", id="params"
+ ),
+ pytest.param(
+ {"content-type": "APPLICATION/XHTML+XML"},
+ "application/xhtml+xml",
+ id="case",
+ ),
+ ],
+)
+def test_content_type_of(headers: dict[str, str], expected: str) -> None:
+ assert content_type_of(httpx.Headers(headers)) == expected
diff --git a/webmentions_ssg/db_helpers.py b/webmentions_ssg/db_helpers.py
new file mode 100644
index 0000000..6e1c2d2
--- /dev/null
+++ b/webmentions_ssg/db_helpers.py
@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Helpers for interpreting SQLAlchemy database results."""
+
+from typing import Any, cast
+
+from sqlalchemy.engine import CursorResult, Result
+
+
+def rows_affected(result: Result[Any]) -> int:
+ """Return the row count from a SQL UPDATE or DELETE result."""
+ return cast(CursorResult[Any], result).rowcount
diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py
index 005bd20..f5fba51 100644
--- a/webmentions_ssg/forms.py
+++ b/webmentions_ssg/forms.py
@@ -5,7 +5,6 @@
"""Webmention forms and URL validation."""
import ipaddress
-from urllib.parse import urlsplit
import httpx
from flask import current_app
@@ -14,7 +13,7 @@ from wtforms import Field, PasswordField, StringField, SubmitField, ValidationEr
from wtforms.form import BaseForm
from wtforms.validators import InputRequired, StopValidation
-from .net import is_public_address
+from .net import hostname_of, is_public_address
from .validation import is_control_character
@@ -96,10 +95,7 @@ class AllowedHostname:
:param field: Field containing the URL to validate.
:raises ValidationError: If the URL hostname is not configured as allowed.
"""
- try:
- hostname = (urlsplit(field.data).hostname or "").rstrip(".")
- except ValueError, TypeError:
- hostname = None
+ hostname = hostname_of(field.data)
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
return
raise ValidationError(self.message or field.gettext("Invalid input."))
@@ -109,11 +105,7 @@ class PublicIPLiteral:
"""Reject non-public IP literals without performing DNS in the web worker."""
def __call__(self, form: BaseForm, field: Field) -> None:
- try:
- hostname = urlsplit(field.data).hostname
- except (ValueError, TypeError) as exc:
- raise ValidationError(field.gettext("Invalid input.")) from exc
-
+ hostname = hostname_of(field.data)
if hostname is None:
raise ValidationError(field.gettext("Invalid input."))
diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py
index 7c29edc..02c7f2b 100644
--- a/webmentions_ssg/net.py
+++ b/webmentions_ssg/net.py
@@ -7,7 +7,7 @@
import ipaddress
import socket
import time
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
from contextlib import ExitStack
from http import HTTPStatus
from urllib.parse import urlsplit
@@ -252,6 +252,20 @@ def public_addresses(
return tuple(str(address) for address in addresses)
+def hostname_of(url: str) -> str | None:
+ """Return a normalized URL hostname, or None for malformed URLs."""
+ try:
+ hostname = urlsplit(url).hostname
+ except (TypeError, ValueError):
+ return None
+ return hostname.rstrip(".") if hostname is not None else None
+
+
+def content_type_of(headers: Mapping[str, str]) -> str:
+ """Return a normalized media type without Content-Type parameters."""
+ return headers.get("Content-Type", "").partition(";")[0].strip().lower()
+
+
def is_http_url(url: str) -> bool:
"""Check whether a URL is an absolute HTTP or HTTPS URL.
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 474e775..362ebe8 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,23 +21,14 @@ def create_consumer() -> Huey:
"""
app = create_app()
- schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
- queue.periodic_task(schedule)(scanner.scan_sources)
-
- recovery_schedule = crontab(
- *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
-
- notification_schedule = crontab(
- *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(notification_schedule)(notifications.send_webmention_digest)
-
- retention_schedule = crontab(
- *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(retention_schedule)(retention.purge_old_received_webmentions)
+ for config_key, task in (
+ ("SCANNER_SCHEDULE", scanner.scan_sources),
+ ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions),
+ ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest),
+ ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions),
+ ):
+ schedule = crontab(*app.config[config_key].split(), strict=True)
+ queue.periodic_task(schedule)(task)
return queue.huey
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index 107d30a..becac58 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -14,6 +14,7 @@ from flask import current_app
from ..net import (
ResponseTooLargeError,
+ content_type_of,
is_http_url,
is_temporary_status,
read_limited_body,
@@ -173,9 +174,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
if not response.is_success:
raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
+ media_type = content_type_of(response.headers)
if media_type not in {"text/html", "application/xhtml+xml"}:
return None
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 78089af..1df363f 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -7,12 +7,12 @@
import uuid
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
-from typing import Any, cast
import httpx
import sqlalchemy as sa
from flask import current_app
+from ..db_helpers import rows_affected
from ..extensions import db, huey
from ..models import ReceivedStatus, ReceivedWebmention
from ..net import (
@@ -20,6 +20,7 @@ from ..net import (
ResponseTooLargeError,
UnsafeAddressError,
build_client,
+ content_type_of,
is_temporary_status,
read_limited_body,
)
@@ -103,9 +104,7 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
response, body = fetch_source(source_url)
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
+ media_type = content_type_of(response.headers)
match media_type:
case "text/html" | "application/xhtml+xml":
@@ -172,7 +171,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
)
- if cast(sa.CursorResult[Any], claimed).rowcount != 1:
+ if rows_affected(claimed) != 1:
db.session.rollback()
return
@@ -285,7 +284,7 @@ def recover_stale_webmentions() -> int:
)
)
- if cast(sa.CursorResult[Any], changed).rowcount != 1:
+ if rows_affected(changed) != 1:
db.session.rollback()
continue
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 754c891..86283b7 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -5,12 +5,11 @@
"""Delete old received Webmentions in terminal unsuccessful states."""
from datetime import UTC, datetime, timedelta
-from typing import Any, cast
import sqlalchemy as sa
from flask import current_app
-from sqlalchemy.engine import CursorResult
+from ..db_helpers import rows_affected
from ..extensions import db
from ..models import ReceivedStatus, ReceivedWebmention
@@ -26,22 +25,18 @@ def purge_old_received_webmentions() -> int:
days=current_app.config["RECEIVED_RETENTION_DAYS"]
)
- result = cast(
- CursorResult[Any],
- db.session.execute(
- sa.delete(ReceivedWebmention).where(
- ReceivedWebmention.status.in_(
- (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
- ),
- ReceivedWebmention.updated_at < cutoff,
- )
- ),
+ result = db.session.execute(
+ sa.delete(ReceivedWebmention).where(
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ ),
+ ReceivedWebmention.updated_at < cutoff,
+ )
)
db.session.commit()
- if result.rowcount:
- current_app.logger.info(
- "Purged %d old failed/deleted Webmentions", result.rowcount
- )
+ count = rows_affected(result)
+ if count:
+ current_app.logger.info("Purged %d old failed/deleted Webmentions", count)
- return result.rowcount
+ return count
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 4f994d4..92d8dfa 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -6,13 +6,13 @@
import uuid
from datetime import UTC, datetime, timedelta
-from typing import Any, cast
import httpx
import sqlalchemy as sa
from flask import current_app
from huey.exceptions import TaskLockedException
+from ..db_helpers import rows_affected
from ..extensions import db, huey
from ..models import SentWebmention, SentWebmentionStatus
from ..net import (
@@ -159,7 +159,7 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
)
db.session.commit()
- if cast(sa.CursorResult[Any], claim).rowcount != 1:
+ if rows_affected(claim) != 1:
return
db.session.refresh(webmention)
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 9cbe071..294402c 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -11,14 +11,14 @@ from fnmatch import fnmatchcase
from itertools import chain
from pathlib import Path
from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
+from urllib.parse import quote, urldefrag, urljoin
import mf2py
import xxhash
from bs4 import BeautifulSoup, Tag
from bs4.element import NavigableString, PageElement
-from ..net import is_http_url
+from ..net import hostname_of, is_http_url
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
@@ -283,7 +283,7 @@ def iter_targets(
for value in chain(hrefs, reactions):
target = normalize_target(value, base_url=base_url, source_url=source_url)
if target is not None:
- hostname = (urlsplit(target).hostname or "").rstrip(".")
+ hostname = hostname_of(target) or ""
if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
yield target
diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py
index 1ca06b5..794bcdf 100644
--- a/webmentions_ssg/views/admin.py
+++ b/webmentions_ssg/views/admin.py
@@ -54,10 +54,7 @@ def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
if not form.validate_on_submit():
abort(400)
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- abort(404)
+ webmention = db.get_or_404(ReceivedWebmention, identifier)
db.session.delete(webmention)
db.session.commit()
@@ -84,10 +81,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
if not form.validate_on_submit():
abort(400)
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- abort(404)
+ webmention = db.get_or_404(ReceivedWebmention, identifier)
webmention.reset_for_verification()
@@ -145,12 +139,7 @@ def sent_source(source_identifier: uuid.UUID) -> ResponseReturnValue:
:param source_identifier: Identifier of the source to display.
:return: Rendered source details page.
"""
- source = db.session.scalar(
- sa.select(Source).where(Source.uuid == source_identifier)
- )
-
- if source is None:
- abort(404)
+ source = db.get_or_404(Source, source_identifier)
webmentions = db.paginate(
sa.select(SentWebmention)
@@ -189,9 +178,9 @@ def resend_sent_webmention(
if not form.validate_on_submit():
abort(400)
- webmention = db.session.get(SentWebmention, webmention_identifier)
+ webmention = db.get_or_404(SentWebmention, webmention_identifier)
- if webmention is None or webmention.source_id != source_identifier:
+ if webmention.source_id != source_identifier:
abort(404)
# Requeue the existing desired revision instead of inventing a new source
diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py
index e3f1368..a7badfd 100644
--- a/webmentions_ssg/views/endpoint.py
+++ b/webmentions_ssg/views/endpoint.py
@@ -6,15 +6,15 @@
import uuid
from datetime import UTC, datetime, timedelta
-from typing import Any, cast
+from typing import cast
import sqlalchemy as sa
import sqlalchemy.exc as sa_exc
-from flask import Response, abort, current_app, render_template, url_for
+from flask import Response, current_app, render_template, url_for
from flask.typing import ResponseReturnValue
-from sqlalchemy.engine import CursorResult
from .. import forms
+from ..db_helpers import rows_affected
from ..extensions import csrf, db, limiter
from ..models import ReceivedWebmention, received_url_digest
from ..tasks.receiver import verify_webmention
@@ -63,20 +63,17 @@ def reopen_if_cooled_down(webmention: ReceivedWebmention) -> bool:
seconds=current_app.config["ENDPOINT_REVERIFY_COOLDOWN_SECONDS"]
)
- result = cast(
- CursorResult[Any],
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention.uuid,
- ReceivedWebmention.updated_at <= cutoff,
- )
- .values(ReceivedWebmention.verification_reset_values(at=now))
- .execution_options(synchronize_session=False)
- ),
+ result = db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention.uuid,
+ ReceivedWebmention.updated_at <= cutoff,
+ )
+ .values(ReceivedWebmention.verification_reset_values(at=now))
+ .execution_options(synchronize_session=False)
)
db.session.commit()
- return result.rowcount == 1
+ return rows_affected(result) == 1
def submit_received(source: str, target: str) -> tuple[uuid.UUID, bool] | None:
@@ -144,10 +141,7 @@ def status(identifier: uuid.UUID) -> ResponseReturnValue:
:param identifier: Identifier of the Webmention to display.
:return: Rendered Webmention status page.
"""
- webmention = db.session.get(ReceivedWebmention, identifier)
-
- if webmention is None:
- abort(404)
+ webmention = db.get_or_404(ReceivedWebmention, identifier)
return render_template(
"status.html", title="Webmention status", webmention=webmention