From d0e245d63e014d268310976fc45429f08d2cbbe9 Mon Sep 17 00:00:00 2001
From: Dennis Fink
Date: Wed, 19 Aug 2026 19:49:55 +0200
Subject: refactor(core): improve typing and test coverage
Restructure database and user CLI commands, make passwords write-only
model properties, and use the UTC datetime constant throughout the
application.
Add PEP 287-style documentation and expand receiver, scanner, sender,
and URL security tests to cover error paths and edge cases.
---
tests/tasks/test_receiver.py | 52 ++++++
tests/tasks/test_scanner.py | 203 ++++++++++++++++++++
tests/tasks/test_sender.py | 377 +++++++++++++++++++++++++++++++++++++-
tests/test_url_security.py | 10 +-
tests/test_views.py | 6 +-
webmentions_ssg/__init__.py | 97 ++++++----
webmentions_ssg/auth.py | 8 +-
webmentions_ssg/models.py | 15 +-
webmentions_ssg/tasks/consumer.py | 11 +-
webmentions_ssg/tasks/receiver.py | 161 ++++++++++------
webmentions_ssg/tasks/scanner.py | 139 ++++++++++++--
webmentions_ssg/tasks/sender.py | 115 ++++++++++--
webmentions_ssg/url_security.py | 15 +-
13 files changed, 1077 insertions(+), 132 deletions(-)
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 811ab63..7e8f531 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -559,3 +559,55 @@ def test_ensure_public_request_rejects_url_without_hostname(
receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL))
is_public_url.assert_called_once_with(SOURCE_URL)
+
+
+@patch("webmentions_ssg.tasks.receiver.BeautifulSoup")
+def test_html_mentions_target_ignores_non_string_attribute(
+ beautiful_soup: Mock, receiver: ModuleType
+) -> None:
+ document = beautiful_soup.return_value
+ document.select_one.return_value = None
+
+ element = Mock()
+ element.get.return_value = ["not", "a", "string"]
+ document.select.return_value = [element]
+
+ assert not receiver.html_mentions_target(b"", SOURCE_URL, TARGET_URL)
+
+
+@patch("webmentions_ssg.tasks.receiver.httpx.Client")
+def test_fetch_source_ignores_invalid_content_length(
+ httpx_client: Mock, app: Flask, receiver: ModuleType
+) -> None:
+ response = httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html", "Content-Length": "invalid"},
+ content=b"
Document
",
+ request=httpx.Request("GET", SOURCE_URL),
+ )
+ set_stream_response(httpx_client, response)
+
+ with app.app_context():
+ fetched_response, body = receiver.fetch_source(SOURCE_URL)
+
+ assert fetched_response is response
+ assert body == b"Document
"
+
+
+@patch("webmentions_ssg.tasks.receiver.fetch_source")
+def test_source_mentions_target_falls_back_for_unknown_encoding(
+ fetch_source: Mock, receiver: ModuleType
+) -> None:
+ body = f"Reply to {TARGET_URL}".encode()
+
+ response = httpx.Response(
+ 200,
+ headers={"Content-Type": "text/plain"},
+ content=body,
+ request=httpx.Request("GET", SOURCE_URL),
+ )
+ response.encoding = "unknown-encoding"
+ fetch_source.return_value = response, body
+
+ assert receiver.source_mentions_target(SOURCE_URL, TARGET_URL)
+ fetch_source.assert_called_once_with(SOURCE_URL)
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 65031b4..89e2995 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -1556,3 +1556,206 @@ def test_scan_source_ignores_reaction_to_matching_hostname(
)
assert scanned.targets == frozenset()
+
+
+def test_canonical_url_rejects_non_http_resolved_url(
+ scanner_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+
+
+
+
+
+ """
+ )
+
+ with pytest.raises(
+ scanner_module.SourceScanError,
+ match="Canonical URL is not a valid HTTP or HTTPS URL",
+ ):
+ scanner_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+
+def test_property_urls_ignores_non_mapping_properties(
+ scanner_module: ModuleType,
+) -> None:
+ entry = {"properties": "invalid"}
+
+ assert tuple(scanner_module.property_urls(entry, "url")) == ()
+
+
+@pytest.mark.parametrize(
+ ("parsed", "message"),
+ [
+ pytest.param(
+ [],
+ "Microformats parser did not return a document object",
+ id="non-object-document",
+ ),
+ pytest.param(
+ {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
+ ),
+ pytest.param(
+ {"items": [{"type": ["h-card"]}]},
+ "Parsed microformats item is not an h-entry",
+ id="wrong-item-type",
+ ),
+ ],
+)
+def test_parse_entry_rejects_invalid_parser_output(
+ scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str
+) -> None:
+ document = parse_html(
+ """
+
+ Hello world
+
+ """
+ )
+
+ element = document.find(class_="h-entry")
+
+ assert element is not None
+
+ monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed)
+
+ with pytest.raises(scanner_module.SourceScanError, match=message):
+ scanner_module.parse_entry(element, SOURCE_URL)
+
+
+@pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(" ", id="empty"),
+ pytest.param("mailto:example@example.com", id="non-http"),
+ ],
+)
+def test_normalize_target_ignores_invalid_target(
+ scanner_module: ModuleType, value: str
+) -> None:
+ assert (
+ scanner_module.normalize_target(
+ value, base_url=SOURCE_URL, source_url=SOURCE_URL
+ )
+ is None
+ )
+
+
+def test_changed_source_url_is_rejected(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ configure_scanner(app, tmp_path)
+
+ write_post(
+ tmp_path,
+ "example",
+ """
+ Hello world.
+ """,
+ )
+
+ run_scan(app, scanner_module)
+
+ changed_url = f"{BASE_URL}changed/"
+
+ write_post(
+ tmp_path,
+ "example",
+ """
+ Hello world.
+ """,
+ canonical=changed_url,
+ entry_url=changed_url,
+ )
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ source = db.session.scalar(sa.select(Source))
+
+ assert source is not None
+ assert source.url == SOURCE_URL
+ assert source.revision == 1
+
+ assert "changed public URL" in caplog.text
+
+
+def test_scan_sources_returns_when_source_directory_is_not_configured(
+ app: Flask, scanner_module: ModuleType
+) -> None:
+ app.config["WEBMENTIONS_SSG_SOURCE_DIRECTORY"] = None
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ assert db.session.scalar(sa.select(Source)) is None
+
+
+def test_scan_sources_rejects_source_directory_that_is_not_directory(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = tmp_path / "source.html"
+ path.write_text("Hello world.
", encoding="utf-8")
+
+ configure_scanner(app, path)
+
+ with pytest.raises(RuntimeError, match="Source directory is not a directory"):
+ run_scan(app, scanner_module)
+
+
+def test_scan_sources_ignores_root_index(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ configure_scanner(app, tmp_path)
+
+ (tmp_path / "index.html").write_text(
+ f"""
+
+
+
+
+
+
+
+
+ Permalink
+
+
+
+
+
+
+ """,
+ encoding="utf-8",
+ )
+
+ write_post(
+ tmp_path,
+ "example",
+ """
+ Hello world.
+ """,
+ )
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+
+ assert len(sources) == 1
+ assert sources[0].path == "example/index.html"
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index 86799f7..1c8ef99 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -1,6 +1,6 @@
import logging
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from types import ModuleType
from unittest.mock import MagicMock, Mock
@@ -10,6 +10,7 @@ from flask import Flask
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
+from webmentions_ssg.url_security import AddressResolutionError
SOURCE_URL = "https://dennisfink.me/blog/example/"
TARGET_URL = "https://example.com/post"
@@ -30,7 +31,7 @@ def create_sent_webmention(
response_status: int | None = None,
status_url: str | None = None,
) -> uuid.UUID:
- now = datetime.now(timezone.utc)
+ now = datetime.now(UTC)
with app.app_context():
source = Source(
@@ -786,3 +787,375 @@ def test_send_webmention_ignores_unknown_identifier(
sender_module.send_webmention.call_local(identifier)
assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text
+
+
+def test_ensure_public_request_accepts_public_url(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ is_public_url = Mock(return_value=True)
+ monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
+
+ sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
+
+ is_public_url.assert_called_once_with(TARGET_URL)
+
+
+def test_ensure_public_request_rejects_non_public_address(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ is_public_url = Mock(return_value=False)
+ monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
+
+ with pytest.raises(
+ sender_module.PermanentSenderError,
+ match="Request resolves to a non-public address",
+ ):
+ sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
+
+ is_public_url.assert_called_once_with(TARGET_URL)
+
+
+def test_ensure_public_request_maps_dns_failure_to_temporary_error(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ is_public_url = Mock(
+ side_effect=AddressResolutionError("Could not resolve hostname")
+ )
+ monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
+
+ with pytest.raises(
+ sender_module.TemporarySenderError,
+ match="Request hostname could not be resolved",
+ ):
+ sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
+
+ is_public_url.assert_called_once_with(TARGET_URL)
+
+
+def test_resolve_endpoint_rejects_non_http_url(sender_module: ModuleType) -> None:
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ with pytest.raises(
+ sender_module.PermanentSenderError, match="Invalid Webmention endpoint"
+ ):
+ sender_module.resolve_endpoint(response, "mailto:example@example.com")
+
+
+def test_endpoint_from_headers_ignores_malformed_link(
+ sender_module: ModuleType,
+) -> None:
+ response = httpx.Response(
+ 200,
+ headers=[("Link", "not-a-link"), ("Link", "; rel=webmention")],
+ request=httpx.Request("GET", TARGET_URL),
+ )
+
+ assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL
+
+
+def test_endpoint_from_html_accepts_string_rel(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ element = Mock()
+ element.get.side_effect = lambda name: {
+ "rel": "webmention alternate",
+ "href": "/webmention",
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [element]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL
+
+
+def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ missing_rel = Mock()
+ missing_rel.get.return_value = None
+
+ invalid_href = Mock()
+ invalid_href.get.side_effect = lambda name: {
+ "rel": ["webmention"],
+ "href": ["invalid"],
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [missing_rel, invalid_href]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") is None
+
+
+def test_read_target_body_rejects_declared_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["WEBMENTIONS_SSG_MAX_TARGET_BYTES"] = 10
+
+ response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_read_target_body_ignores_invalid_content_length(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ response = httpx.Response(
+ 200, headers={"Content-Length": "invalid"}, content=b"Document"
+ )
+
+ with app.app_context():
+ body = sender_module.read_target_body(response)
+
+ assert body == b"Document"
+
+
+def test_read_target_body_rejects_streamed_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["WEBMENTIONS_SSG_MAX_TARGET_BYTES"] = 10
+
+ class BodyStream(httpx.SyncByteStream):
+ def __iter__(self):
+ yield b"x" * 6
+ yield b"x" * 6
+
+ response = httpx.Response(200, stream=BodyStream())
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_discovery_uses_link_header_from_get(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(200, headers={"Link": "; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert [request.method for request in requests] == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ ("status_code", "exception_name"),
+ [
+ pytest.param(503, "TemporarySenderError", id="temporary"),
+ pytest.param(404, "PermanentSenderError", id="permanent"),
+ ],
+)
+def test_discovery_maps_get_status_to_exception(
+ app: Flask, sender_module: ModuleType, status_code: int, exception_name: str
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(status_code)
+
+ exception_type = getattr(sender_module, exception_name)
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"),
+ ):
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+
+def test_discovery_returns_none_for_unsupported_media_type(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200, headers={"Content-Type": "application/json"}, content=b"{}"
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint is None
+
+
+@pytest.mark.parametrize(
+ ("status_code", "location", "expected_status_url"),
+ [
+ pytest.param(
+ 201, "/webmention/status/123", STATUS_URL, id="created-with-location"
+ ),
+ pytest.param(202, None, None, id="accepted"),
+ ],
+)
+def test_post_webmention(
+ sender_module: ModuleType,
+ status_code: int,
+ location: str | None,
+ expected_status_url: str | None,
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ headers = {}
+
+ if location is not None:
+ headers["Location"] = location
+
+ return httpx.Response(status_code, headers=headers)
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ result = sender_module.post_webmention(
+ client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
+ )
+
+ assert result == (status_code, expected_status_url)
+ assert len(requests) == 1
+
+ request = requests[0]
+
+ assert request.method == "POST"
+ assert str(request.url) == ENDPOINT_URL
+
+ form = httpx.QueryParams(request.content.decode())
+
+ assert form["source"] == SOURCE_URL
+ assert form["target"] == TARGET_URL
+
+
+def test_send_webmention_ignores_unsupported_result_for_superseded_revision(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_sent_webmention(app)
+
+ _, _, discover, post = mock_sender_requests(
+ sender_module, monkeypatch, endpoint=None
+ )
+
+ def supersede_revision(*args, **kwargs) -> None:
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+
+ webmention.desired_revision = 2
+ db.session.commit()
+
+ discover.side_effect = supersede_revision
+
+ sender_module.send_webmention.call_local(identifier)
+
+ post.assert_not_called()
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+ assert webmention.desired_revision == 2
+ assert webmention.processed_revision is None
+ assert webmention.sent_revision is None
+ assert webmention.status is None
+ assert webmention.pending
+
+
+def test_send_webmention_ignores_temporary_failure_for_superseded_revision(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_sent_webmention(app)
+
+ _, _, _, post = mock_sender_requests(sender_module, monkeypatch)
+
+ def supersede_revision(*args, **kwargs) -> tuple[int, None]:
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+
+ webmention.desired_revision = 2
+ db.session.commit()
+
+ raise sender_module.TemporarySenderError("Temporary failure")
+
+ post.side_effect = supersede_revision
+
+ sender_module.send_webmention.call_local(identifier)
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+ assert webmention.desired_revision == 2
+ assert webmention.processed_revision is None
+ assert webmention.sent_revision is None
+ assert webmention.status is None
+ assert webmention.pending
+
+
+def test_send_webmention_ignores_permanent_failure_for_superseded_revision(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_sent_webmention(app)
+
+ _, _, _, post = mock_sender_requests(sender_module, monkeypatch)
+
+ def supersede_revision(*args, **kwargs) -> tuple[int, None]:
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+
+ webmention.desired_revision = 2
+ db.session.commit()
+
+ return (400, None)
+
+ post.side_effect = supersede_revision
+
+ sender_module.send_webmention.call_local(identifier)
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+
+ assert webmention is not None
+ assert webmention.desired_revision == 2
+ assert webmention.processed_revision is None
+ assert webmention.sent_revision is None
+ assert webmention.status is None
+ assert webmention.pending
diff --git a/tests/test_url_security.py b/tests/test_url_security.py
index ffa0378..70d5146 100644
--- a/tests/test_url_security.py
+++ b/tests/test_url_security.py
@@ -3,7 +3,11 @@ from unittest.mock import Mock, patch
import dns.resolver
import pytest
-from webmentions_ssg.url_security import AddressResolutionError, is_public_url
+from webmentions_ssg.url_security import (
+ AddressResolutionError,
+ is_http_url,
+ is_public_url,
+)
@patch("webmentions_ssg.url_security.dns.resolver.resolve_name")
@@ -66,3 +70,7 @@ def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> No
is_public_url("/relative/url")
resolve_name.assert_not_called()
+
+
+def test_is_http_url_rejects_malformed_url() -> None:
+ assert not is_http_url("http://[::1")
diff --git a/tests/test_views.py b/tests/test_views.py
index aac281e..67ed190 100644
--- a/tests/test_views.py
+++ b/tests/test_views.py
@@ -1,5 +1,5 @@
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from unittest.mock import Mock, call, patch
import pytest
@@ -19,7 +19,7 @@ from webmentions_ssg.models import (
def log_in(app: Flask, client: FlaskClient) -> None:
with app.app_context():
- user = User(username="admin")
+ user = User(username="admin", password="test")
db.session.add(user)
db.session.commit()
@@ -32,7 +32,7 @@ def log_in(app: Flask, client: FlaskClient) -> None:
def create_sent_source(app: Flask) -> uuid.UUID:
- now = datetime.now(timezone.utc)
+ now = datetime.now(UTC)
with app.app_context():
source = Source(
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index b799678..3928195 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -11,6 +11,7 @@ import click
import coloredlogs
import sqlalchemy as sa
from flask import Flask
+from flask.cli import AppGroup
from flask_bootstrap import Bootstrap5
from flask_migrate import Migrate, upgrade
from flask_sqlalchemy import SQLAlchemy
@@ -35,14 +36,25 @@ DATABASE = SQLAlchemy(model_class=Base)
MIGRATE = Migrate()
HUEY = Huey()
+USERS_CLI = AppGroup("users", help="Handle users")
+DB_CLI = AppGroup("db", help="Handle database")
+
def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
- """Creates and configures the Flask application."""
+ """
+ Create and configure the Flask application.
+
+ The application configuration is selected based on the development
+ environment unless an explicit configuration class is provided. Logging,
+ extensions, authentication, Huey, blueprints, and CLI commands are then
+ initialized.
- development = os.environ.get(
- "WEBMENTIONS_SSG_DEV",
- "",
- ).lower() in {
+ :param config_class: Configuration class to use, or ``None`` to select the
+ development or production configuration automatically.
+ :return: Configured Flask application.
+ """
+
+ development = os.environ.get("WEBMENTIONS_SSG_DEV", "").lower() in {
"1",
"true",
"yes",
@@ -53,8 +65,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
config_class = DevelopmentConfig if development else ProductionConfig
app = Flask(
- __name__,
- instance_path=str(Path("testing").absolute()) if development else None,
+ __name__, instance_path=str(Path("testing").absolute()) if development else None
)
app.config.from_object(config_class)
@@ -84,9 +95,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
werkzeug_logger.addHandler(log_file_handler)
coloredlogs.install(
- level=app.config["LOG_LEVEL"],
- logger=app.logger,
- fmt=app.config["LOG_FORMAT"],
+ level=app.config["LOG_LEVEL"], logger=app.logger, fmt=app.config["LOG_FORMAT"]
)
coloredlogs.install(
level=app.config["LOG_LEVEL"],
@@ -106,44 +115,54 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
@app.context_processor
def inject_version():
- return dict(VERSION=VERSION)
+ """
+ Add the application version to the template context.
+
+ :return: Template context containing the application version.
+ """
+ return {"VERSION": VERSION}
from .views import root_page
app.register_blueprint(root_page)
- @app.cli.command("create-db")
- def create_db() -> None:
- """Create or update the database to the latest migration."""
- upgrade()
+ app.cli.add_command(USERS_CLI)
- @app.cli.command("create-user")
- @click.argument("username")
- @click.password_option(
- confirmation_prompt=True,
- )
- def create_user(
- username: str,
- password: str,
- ) -> None:
- """Create a user."""
+ return app
- from .models import User
- if (
- DATABASE.session.scalar(sa.select(User).where(User.username == username))
- is not None
- ):
- raise click.ClickException(f"User {username!r} already exists.")
+@USERS_CLI.command("create", help="Create or add users")
+@click.option("--username", prompt=True)
+@click.password_option(confirmation_prompt=True)
+def create_user(username: str, password: str) -> None:
+ """
+ Create a new user.
- user = User(
- username=username,
- )
- user.set_password(password)
+ :param username: Username of the user to create.
+ :param password: Plain-text password to hash and store for the user.
+ :raises click.ClickException: If a user with the given username already
+ exists.
+ """
- DATABASE.session.add(user)
- DATABASE.session.commit()
+ from .models import User
- click.echo(f"Created user {username!r}.")
+ if (
+ DATABASE.session.scalar(sa.select(User).where(User.username == username))
+ is not None
+ ):
+ raise click.ClickException(f"User {username!r} already exists.")
- return app
+ user = User(username=username, password=password)
+
+ DATABASE.session.add(user)
+ DATABASE.session.commit()
+
+ click.echo(f"Created user {username!r}.")
+
+
+@DB_CLI.command("create")
+def create_db() -> None:
+ """
+ Upgrade the database to the latest migration revision.
+ """
+ upgrade()
diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py
index f8fd28c..7bd93a3 100644
--- a/webmentions_ssg/auth.py
+++ b/webmentions_ssg/auth.py
@@ -9,5 +9,11 @@ AUTH.login_view = "root.login"
@AUTH.user_loader
-def load_user(id):
+def load_user(id: int) -> User | None:
+ """
+ Load a user from the database by identifier.
+
+ :param id: User identifier stored in the login session.
+ :return: Matching user, or ``None`` if no user exists with the identifier.
+ """
return db.session.get(User, int(id))
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index b6679e8..f89e587 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from enum import StrEnum
from flask_login import UserMixin
@@ -38,7 +38,12 @@ class User(UserMixin, Base):
def __repr__(self) -> str:
return f""
- def set_password(self, password: str) -> None:
+ @property
+ def password(self) -> None:
+ raise AttributeError("Password is write-only")
+
+ @password.setter
+ def password(self, password: str) -> None:
self.password_hash = generate_password_hash(password)
def check_password(self, password: str) -> bool:
@@ -60,8 +65,8 @@ class ReceivedWebmention(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
- default=lambda: datetime.now(timezone.utc),
- onupdate=lambda: datetime.now(timezone.utc),
+ default=lambda: datetime.now(UTC),
+ onupdate=lambda: datetime.now(UTC),
nullable=False,
)
@@ -185,4 +190,4 @@ class SentWebmention(Base):
def uuid7_to_datetime(identifier: uuid.UUID) -> datetime:
- return datetime.fromtimestamp(identifier.time / 1000, tz=timezone.utc)
+ return datetime.fromtimestamp(identifier.time / 1000, tz=UTC)
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index e53fce5..3bfe861 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -4,9 +4,18 @@ from .. import HUEY, create_app
def create_consumer() -> Huey:
+ """
+ Create and configure the Huey consumer.
+
+ The application is initialized, task modules are imported to register their
+ tasks, and the source scanner is registered with the configured periodic
+ schedule.
+
+ :return: Configured Huey instance.
+ """
app = create_app()
- from . import receiver, scanner, sender # noqa: E402, F401
+ from . import receiver, scanner, sender # noqa: F401
schedule = crontab(
*app.config["WEBMENTIONS_SSG_SCANNER_SCHEDULE"].split(), strict=True
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 39def73..4aba3d8 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -47,7 +47,17 @@ HTML_URL_ATTRIBUTES = {
def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """Check valid HTML URL attributes for the exact target URL."""
+ """
+ Check whether an HTML document mentions the target URL.
+
+ URL references are resolved against the document's ``base`` element when
+ present, or against the source URL otherwise.
+
+ :param body: HTML document body.
+ :param source_url: URL from which the document was retrieved.
+ :param target_url: Exact URL to look for.
+ :return: Whether a supported HTML URL attribute references the target URL.
+ """
document = BeautifulSoup(body, "html.parser")
base_url = source_url
@@ -59,12 +69,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
for attribute, selectors in HTML_URL_ATTRIBUTES.items():
selector = ", ".join(
- [
- "{selector}[{attribute}]".format(
- selector=selector_string, attribute=attribute
- )
- for selector_string in selectors
- ]
+ [f"{selector_string}[{attribute}]" for selector_string in selectors]
)
for element in document.select(selector):
@@ -78,12 +83,25 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
def text_mentions_target(body: str, target_url: str) -> bool:
- """Check whether plain text contains the exact target IRI."""
+ """
+ Check whether plain text contains the exact target IRI.
+
+ :param body: Plain-text document body.
+ :param target_url: Exact IRI to look for.
+ :return: Whether the target IRI occurs in the document.
+ """
return any(match.group() == target_url for match in IRI_PATTERN.finditer(body))
def ensure_public_request(request: httpx.Request) -> None:
- """Prevent requests to non-public network addresses."""
+ """
+ Ensure that an HTTP request targets a public network address.
+
+ :param request: HTTP request to validate.
+ :raises VerificationError: If the URL has no hostname or resolves to a
+ non-public address.
+ :raises TemporaryFetchError: If the hostname cannot be resolved.
+ """
try:
if not is_public_url(str(request.url)):
raise VerificationError("Source resolves to a non-public address")
@@ -94,56 +112,83 @@ def ensure_public_request(request: httpx.Request) -> None:
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- """Fetch a source with limits on redirects, time, and response size."""
-
- with httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(
- current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0)
- ),
- follow_redirects=True,
- max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20),
- trust_env=False,
- event_hooks={"request": [ensure_public_request]},
- ) as client:
- with client.stream("GET", source_url) as response:
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
-
- max_source_bytes = current_app.config.get(
- "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000
- )
+ """
+ Fetch a webmention source document.
+
+ The request follows redirects while enforcing the configured timeout,
+ redirect limit, and maximum response size.
+
+ :param source_url: URL of the source document.
+ :return: HTTP response and response body.
+ :raises SourceGoneError: If the source returns HTTP 410.
+ :raises TemporaryFetchError: If the source returns a temporary HTTP error.
+ :raises VerificationError: If the source returns another unsuccessful HTTP
+ status or exceeds the configured maximum response size.
+ """
+
+ with (
+ httpx.Client(
+ headers={
+ "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ },
+ timeout=httpx.Timeout(
+ current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0)
+ ),
+ follow_redirects=True,
+ max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20),
+ trust_env=False,
+ event_hooks={"request": [ensure_public_request]},
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if status in {408, 425, 429} or 500 <= status <= 599:
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
+
+ max_source_bytes = current_app.config.get(
+ "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000
+ )
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
+ if (content_length := response.headers.get("Content-Length")) is not None:
+ try:
+ if int(content_length) > max_source_bytes:
+ raise VerificationError("Source document is too large")
+ except ValueError:
+ pass
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
+ body = bytearray()
+ for chunk in response.iter_bytes(chunk_size=64 * 1024):
+ body.extend(chunk)
- if len(body) > max_source_bytes:
- raise VerificationError("Source document is too large")
+ if len(body) > max_source_bytes:
+ raise VerificationError("Source document is too large")
- return response, bytes(body)
+ return response, bytes(body)
def source_mentions_target(source_url: str, target_url: str) -> bool:
- """Fetch the source and verify it according to its media type."""
+ """
+ Fetch a source and check whether it mentions the target URL.
+
+ HTML and XHTML sources are inspected for supported URL attributes, while
+ plain-text sources are searched for the exact target IRI.
+
+ :param source_url: URL of the source document.
+ :param target_url: Exact target URL to look for.
+ :return: Whether the source mentions the target URL.
+ :raises VerificationError: If the source has an unsupported or missing
+ content type.
+ :raises SourceGoneError: If the source explicitly reports that it is gone.
+ :raises TemporaryFetchError: If fetching the source fails temporarily.
+ """
response, body = fetch_source(source_url)
@@ -170,7 +215,17 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
@huey.task(retries=2, retry_delay=50)
def verify_webmention(webmention_uuid: uuid.UUID) -> None:
- """Verify a ReceivedWebmention and store the result."""
+ """
+ Verify a received webmention and store its verification status.
+
+ Temporary fetch failures are stored before being re-raised so that Huey can
+ retry the task.
+
+ :param webmention_uuid: Identifier of the received webmention to verify.
+ :raises TemporaryFetchError: If fetching the source fails temporarily.
+ :raises httpx.RequestError: If an HTTP request error occurs while fetching
+ the source.
+ """
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index ff75697..fa6a47b 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -36,7 +36,13 @@ class ScannedSource:
def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """Derive the public source URL from its relative filesystem path."""
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
directory = relative_path.parent.as_posix()
return urljoin(base_url, f"{quote(directory, safe='/')}/")
@@ -45,7 +51,18 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
def canonical_url(
document: BeautifulSoup, *, relative_path: Path, base_url: str | None
) -> str | None:
- """Return the canonical URL declared by the document."""
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
href = link.get("href")
@@ -72,7 +89,13 @@ def canonical_url(
def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """Yield URL values from a microformats property."""
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
properties = entry.get("properties")
if not isinstance(properties, dict):
@@ -87,7 +110,14 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """Parse the source h-entry with mf2py."""
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
parsed = mf2py.parse(doc=str(element), url=base_url)
if not isinstance(parsed, dict):
@@ -114,7 +144,18 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
def primary_entry(
document: BeautifulSoup, source_url: str
) -> tuple[Tag, dict[str, Any]]:
- """Return and validate the source h-entry."""
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
entries = document.find_all(class_="h-entry")
if len(entries) != 1:
@@ -134,7 +175,14 @@ def primary_entry(
def content_element(entry: Tag) -> Tag:
- """Return the source h-entry's e-content element."""
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
contents = entry.find_all(class_="e-content")
if len(contents) != 1:
@@ -144,7 +192,18 @@ def content_element(entry: Tag) -> Tag:
def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """Resolve and validate a possible Webmention target URL."""
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
value = value.strip()
if not value:
@@ -169,7 +228,20 @@ def iter_targets(
source_url: str,
ignored_hostnames: tuple[str, ...] = (),
) -> Iterator[str]:
- """Yield outgoing Webmention targets from an h-entry."""
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
hrefs = (
href
for element in content.find_all("a", href=True)
@@ -195,7 +267,18 @@ def scan_source_file(
base_url: str | None,
ignored_hostnames: tuple[str, ...] = (),
) -> ScannedSource:
- """Parse one generated source document."""
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
document = BeautifulSoup(path.read_bytes(), "html5lib")
relative_path = path.relative_to(root)
@@ -232,7 +315,16 @@ def scan_source_file(
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
- """Create a source from a newly discovered document."""
+ """
+ Create a source model from a newly discovered document.
+
+ A sent Webmention is created for each discovered target at the initial source
+ revision.
+
+ :param scanned: Scanned source data.
+ :param scan_time: Time at which the source was discovered.
+ :return: Newly created source model.
+ """
source = Source(
path=scanned.path,
url=scanned.url,
@@ -254,7 +346,19 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
def update_source(
source: Source, scanned: ScannedSource, scan_time: datetime
) -> Source:
- """Update a source from a newly scanned revision."""
+ """
+ Update an existing source from newly scanned data.
+
+ A changed or previously deleted source receives a new revision. Existing sent
+ Webmentions are updated to reflect the current targets, and newly discovered
+ targets are added.
+
+ :param source: Existing source model to update.
+ :param scanned: Newly scanned source data.
+ :param scan_time: Time at which the source was scanned.
+ :return: Updated source model.
+ :raises SourceScanError: If the public URL of the source has changed.
+ """
if source.url != scanned.url:
raise SourceScanError(
f"Source path {source.path!r} changed public URL "
@@ -293,7 +397,15 @@ def update_source(
@huey.lock_task("scan-webmention-sources")
def scan_sources() -> None:
- """Scan generated source documents and queue pending Webmentions."""
+ """
+ Scan generated source documents and queue pending Webmentions.
+
+ New and changed sources are persisted, missing sources are marked as deleted,
+ and Webmentions requiring processing are queued for sending.
+
+ :raises RuntimeError: If the configured source directory or base URL is
+ invalid.
+ """
directory = current_app.config.get("WEBMENTIONS_SSG_SOURCE_DIRECTORY")
if directory is None:
@@ -399,4 +511,7 @@ def scan_sources() -> None:
@huey.task()
def manual_scan_sources() -> None:
+ """
+ Run a source scan as a Huey task.
+ """
return scan_sources()
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index fa9b624..629c538 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -1,6 +1,6 @@
import re
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from urllib.parse import urljoin
import httpx
@@ -31,12 +31,23 @@ class PermanentSenderError(SenderError):
def temporary_http_status(status: int) -> bool:
- """Return whether an HTTP status should be retried."""
+ """
+ Return whether an HTTP status indicates a temporary failure.
+
+ :param status: HTTP response status code.
+ :return: Whether the request should be retried.
+ """
return status in {408, 425, 429} or 500 <= status <= 599
def ensure_public_request(request: httpx.Request) -> None:
- """Prevent requests to non-public network addresses."""
+ """
+ Ensure that an HTTP request targets a public network address.
+
+ :param request: HTTP request to validate.
+ :raises PermanentSenderError: If the request resolves to a non-public address.
+ :raises TemporarySenderError: If the request hostname cannot be resolved.
+ """
try:
if not is_public_url(str(request.url)):
raise PermanentSenderError("Request resolves to a non-public address")
@@ -45,7 +56,15 @@ def ensure_public_request(request: httpx.Request) -> None:
def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """Resolve and validate a discovered Webmention endpoint."""
+ """
+ Resolve and validate a discovered Webmention endpoint.
+
+ :param response: Response from which the endpoint was discovered.
+ :param href: Endpoint reference to resolve.
+ :return: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
+ HTTPS URL.
+ """
endpoint = urljoin(str(response.url), href.strip())
if not is_http_url(endpoint):
@@ -55,7 +74,12 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str:
def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """Parse a Link header value into its target and relations."""
+ """
+ Parse a Link header value into its target and relations.
+
+ :param value: Link header value to parse.
+ :return: Link target and relation names, or ``None`` if the value is invalid.
+ """
value = value.strip()
if not value.startswith("<"):
@@ -80,7 +104,13 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None:
def endpoint_from_headers(response: httpx.Response) -> str | None:
- """Return the first Webmention endpoint advertised by HTTP Link."""
+ """
+ Find a Webmention endpoint in the response Link headers.
+
+ :param response: HTTP response whose headers should be inspected.
+ :return: First advertised Webmention endpoint, or ``None`` if none is found.
+ :raises PermanentSenderError: If a discovered endpoint is invalid.
+ """
for header in response.headers.get_list("Link"):
for value in LINK_SPLIT.split(header):
if (link := parse_link_value(value)) is None:
@@ -95,7 +125,16 @@ def endpoint_from_headers(response: httpx.Response) -> str | None:
def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """Return the first HTML Webmention endpoint in document order."""
+ """
+ Find a Webmention endpoint in an HTML document.
+
+ ``link`` and ``a`` elements are inspected in document order.
+
+ :param response: HTTP response from which the document was retrieved.
+ :param body: HTML document body.
+ :return: First advertised Webmention endpoint, or ``None`` if none is found.
+ :raises PermanentSenderError: If a discovered endpoint is invalid.
+ """
document = BeautifulSoup(body, "html5lib")
for element in document.find_all(["link", "a"], href=True):
@@ -124,7 +163,14 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
def read_target_body(response: httpx.Response) -> bytes:
- """Read a target document up to the configured size limit."""
+ """
+ Read a target document up to the configured size limit.
+
+ :param response: Streaming HTTP response to read.
+ :return: Response body.
+ :raises PermanentSenderError: If the document exceeds the configured maximum
+ size.
+ """
max_bytes = current_app.config.get("WEBMENTIONS_SSG_MAX_TARGET_BYTES", 1_000_000)
if (content_length := response.headers.get("Content-Length")) is not None:
@@ -146,7 +192,20 @@ def read_target_body(response: httpx.Response) -> bytes:
def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """Discover the Webmention endpoint advertised by a target."""
+ """
+ Discover the Webmention endpoint advertised by a target URL.
+
+ Endpoint discovery first checks the response to a HEAD request and then falls
+ back to a GET request, inspecting both HTTP Link headers and supported HTML
+ documents.
+
+ :param client: HTTP client to use for discovery requests.
+ :param target: Target URL whose Webmention endpoint should be discovered.
+ :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :raises TemporarySenderError: If the target returns a temporary HTTP failure.
+ :raises PermanentSenderError: If the target returns a permanent HTTP failure
+ or advertises an invalid endpoint.
+ """
head_response = client.head(target, headers=DISCOVERY_HEADERS)
if endpoint := endpoint_from_headers(head_response):
@@ -175,7 +234,15 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
def post_webmention(
client: httpx.Client, *, endpoint: str, source: str, target: str
) -> tuple[int, str | None]:
- """POST a Webmention and return its status code and status URL."""
+ """
+ Send a Webmention to a discovered endpoint.
+
+ :param client: HTTP client to use for the request.
+ :param endpoint: Webmention endpoint URL.
+ :param source: Source URL of the Webmention.
+ :param target: Target URL of the Webmention.
+ :return: HTTP response status code and optional status URL.
+ """
with client.stream(
"POST", endpoint, data={"source": source, "target": target}
) as response:
@@ -190,7 +257,16 @@ def post_webmention(
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
- """Return whether an attempt still represents the desired revision."""
+ """
+ Check whether a send attempt still represents the desired revision.
+
+ The Webmention is refreshed from the database before comparing its desired
+ and processed revisions.
+
+ :param webmention: Sent Webmention being processed.
+ :param revision: Source revision represented by the current attempt.
+ :return: Whether the attempt is still current and requires processing.
+ """
db.session.refresh(webmention)
return webmention.desired_revision == revision and (
@@ -201,7 +277,18 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@huey.task(retries=2, retry_delay=50)
def send_webmention(webmention_uuid: uuid.UUID) -> None:
- """Discover a receiver endpoint and send one Webmention."""
+ """
+ Discover a receiver endpoint and send a Webmention.
+
+ The result of the attempt is stored on the corresponding sent Webmention.
+ Temporary failures are re-raised so Huey can retry them, while permanent
+ failures mark the current revision as processed.
+
+ :param webmention_uuid: Identifier of the sent Webmention to process.
+ :raises TemporarySenderError: If endpoint discovery or delivery fails for a
+ potentially temporary reason.
+ :raises httpx.RequestError: If an HTTP request fails.
+ """
webmention = db.session.get(SentWebmention, webmention_uuid)
if webmention is None:
@@ -220,7 +307,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
source = webmention.source.url
target = webmention.target
- webmention.last_attempted_at = datetime.now(timezone.utc)
+ webmention.last_attempted_at = datetime.now(UTC)
webmention.endpoint = None
webmention.response_status = None
webmention.status_url = None
@@ -310,6 +397,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
webmention.response_status = response_status
webmention.status_url = status_url
- webmention.last_sent_at = datetime.now(timezone.utc)
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
index 51c037a..9cd261e 100644
--- a/webmentions_ssg/url_security.py
+++ b/webmentions_ssg/url_security.py
@@ -10,6 +10,14 @@ class AddressResolutionError(Exception):
def is_public_url(url: str) -> bool:
+ """
+ Check whether a URL resolves exclusively to public IP addresses.
+
+ :param url: URL whose hostname should be resolved.
+ :return: Whether all resolved addresses are globally routable.
+ :raises ValueError: If the URL does not contain a hostname.
+ :raises AddressResolutionError: If the hostname cannot be resolved.
+ """
hostname = urlsplit(url).hostname
if hostname is None:
@@ -30,7 +38,12 @@ def is_public_url(url: str) -> bool:
def is_http_url(url: str) -> bool:
- """Return whether a URL is an absolute HTTP or HTTPS URL."""
+ """
+ Check whether a URL is an absolute HTTP or HTTPS URL.
+
+ :param url: URL to validate.
+ :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ """
try:
parsed = urlsplit(url)
except ValueError:
--
cgit v1.3.1