aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
authorDennis Fink2026-08-19 19:49:55 +0200
committerDennis Fink2026-08-19 19:49:55 +0200
commitd0e245d63e014d268310976fc45429f08d2cbbe9 (patch)
treeef2e96570723190baf02f05c2c4644a19ff1af78 /webmentions_ssg
parentd4a03623544aac0ce911d8471ff4b033bc3255e1 (diff)
downloadwebmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.tar.gz
webmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.zip
refactor(core): improve typing and test coverage
Restructure database and user CLI commands, make passwords write-only model properties, and use the UTC datetime constant throughout the application. Add PEP 287-style documentation and expand receiver, scanner, sender, and URL security tests to cover error paths and edge cases.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/__init__.py97
-rw-r--r--webmentions_ssg/auth.py8
-rw-r--r--webmentions_ssg/models.py15
-rw-r--r--webmentions_ssg/tasks/consumer.py11
-rw-r--r--webmentions_ssg/tasks/receiver.py155
-rw-r--r--webmentions_ssg/tasks/scanner.py139
-rw-r--r--webmentions_ssg/tasks/sender.py115
-rw-r--r--webmentions_ssg/url_security.py15
8 files changed, 432 insertions, 123 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index b799678..3928195 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -11,6 +11,7 @@ import click
import coloredlogs
import sqlalchemy as sa
from flask import Flask
+from flask.cli import AppGroup
from flask_bootstrap import Bootstrap5
from flask_migrate import Migrate, upgrade
from flask_sqlalchemy import SQLAlchemy
@@ -35,14 +36,25 @@ DATABASE = SQLAlchemy(model_class=Base)
MIGRATE = Migrate()
HUEY = Huey()
+USERS_CLI = AppGroup("users", help="Handle users")
+DB_CLI = AppGroup("db", help="Handle database")
+
def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
- """Creates and configures the Flask application."""
+ """
+ Create and configure the Flask application.
+
+ The application configuration is selected based on the development
+ environment unless an explicit configuration class is provided. Logging,
+ extensions, authentication, Huey, blueprints, and CLI commands are then
+ initialized.
- development = os.environ.get(
- "WEBMENTIONS_SSG_DEV",
- "",
- ).lower() in {
+ :param config_class: Configuration class to use, or ``None`` to select the
+ development or production configuration automatically.
+ :return: Configured Flask application.
+ """
+
+ development = os.environ.get("WEBMENTIONS_SSG_DEV", "").lower() in {
"1",
"true",
"yes",
@@ -53,8 +65,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
config_class = DevelopmentConfig if development else ProductionConfig
app = Flask(
- __name__,
- instance_path=str(Path("testing").absolute()) if development else None,
+ __name__, instance_path=str(Path("testing").absolute()) if development else None
)
app.config.from_object(config_class)
@@ -84,9 +95,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
werkzeug_logger.addHandler(log_file_handler)
coloredlogs.install(
- level=app.config["LOG_LEVEL"],
- logger=app.logger,
- fmt=app.config["LOG_FORMAT"],
+ level=app.config["LOG_LEVEL"], logger=app.logger, fmt=app.config["LOG_FORMAT"]
)
coloredlogs.install(
level=app.config["LOG_LEVEL"],
@@ -106,44 +115,54 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
@app.context_processor
def inject_version():
- return dict(VERSION=VERSION)
+ """
+ Add the application version to the template context.
+
+ :return: Template context containing the application version.
+ """
+ return {"VERSION": VERSION}
from .views import root_page
app.register_blueprint(root_page)
- @app.cli.command("create-db")
- def create_db() -> None:
- """Create or update the database to the latest migration."""
- upgrade()
+ app.cli.add_command(USERS_CLI)
- @app.cli.command("create-user")
- @click.argument("username")
- @click.password_option(
- confirmation_prompt=True,
- )
- def create_user(
- username: str,
- password: str,
- ) -> None:
- """Create a user."""
+ return app
- from .models import User
- if (
- DATABASE.session.scalar(sa.select(User).where(User.username == username))
- is not None
- ):
- raise click.ClickException(f"User {username!r} already exists.")
+@USERS_CLI.command("create", help="Create or add users")
+@click.option("--username", prompt=True)
+@click.password_option(confirmation_prompt=True)
+def create_user(username: str, password: str) -> None:
+ """
+ Create a new user.
- user = User(
- username=username,
- )
- user.set_password(password)
+ :param username: Username of the user to create.
+ :param password: Plain-text password to hash and store for the user.
+ :raises click.ClickException: If a user with the given username already
+ exists.
+ """
- DATABASE.session.add(user)
- DATABASE.session.commit()
+ from .models import User
- click.echo(f"Created user {username!r}.")
+ if (
+ DATABASE.session.scalar(sa.select(User).where(User.username == username))
+ is not None
+ ):
+ raise click.ClickException(f"User {username!r} already exists.")
- return app
+ user = User(username=username, password=password)
+
+ DATABASE.session.add(user)
+ DATABASE.session.commit()
+
+ click.echo(f"Created user {username!r}.")
+
+
+@DB_CLI.command("create")
+def create_db() -> None:
+ """
+ Upgrade the database to the latest migration revision.
+ """
+ upgrade()
diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py
index f8fd28c..7bd93a3 100644
--- a/webmentions_ssg/auth.py
+++ b/webmentions_ssg/auth.py
@@ -9,5 +9,11 @@ AUTH.login_view = "root.login"
@AUTH.user_loader
-def load_user(id):
+def load_user(id: int) -> User | None:
+ """
+ Load a user from the database by identifier.
+
+ :param id: User identifier stored in the login session.
+ :return: Matching user, or ``None`` if no user exists with the identifier.
+ """
return db.session.get(User, int(id))
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index b6679e8..f89e587 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from enum import StrEnum
from flask_login import UserMixin
@@ -38,7 +38,12 @@ class User(UserMixin, Base):
def __repr__(self) -> str:
return f"<User {self.username}>"
- def set_password(self, password: str) -> None:
+ @property
+ def password(self) -> None:
+ raise AttributeError("Password is write-only")
+
+ @password.setter
+ def password(self, password: str) -> None:
self.password_hash = generate_password_hash(password)
def check_password(self, password: str) -> bool:
@@ -60,8 +65,8 @@ class ReceivedWebmention(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
- default=lambda: datetime.now(timezone.utc),
- onupdate=lambda: datetime.now(timezone.utc),
+ default=lambda: datetime.now(UTC),
+ onupdate=lambda: datetime.now(UTC),
nullable=False,
)
@@ -185,4 +190,4 @@ class SentWebmention(Base):
def uuid7_to_datetime(identifier: uuid.UUID) -> datetime:
- return datetime.fromtimestamp(identifier.time / 1000, tz=timezone.utc)
+ return datetime.fromtimestamp(identifier.time / 1000, tz=UTC)
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index e53fce5..3bfe861 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -4,9 +4,18 @@ from .. import HUEY, create_app
def create_consumer() -> Huey:
+ """
+ Create and configure the Huey consumer.
+
+ The application is initialized, task modules are imported to register their
+ tasks, and the source scanner is registered with the configured periodic
+ schedule.
+
+ :return: Configured Huey instance.
+ """
app = create_app()
- from . import receiver, scanner, sender # noqa: E402, F401
+ from . import receiver, scanner, sender # noqa: F401
schedule = crontab(
*app.config["WEBMENTIONS_SSG_SCANNER_SCHEDULE"].split(), strict=True
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 39def73..4aba3d8 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -47,7 +47,17 @@ HTML_URL_ATTRIBUTES = {
def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """Check valid HTML URL attributes for the exact target URL."""
+ """
+ Check whether an HTML document mentions the target URL.
+
+ URL references are resolved against the document's ``base`` element when
+ present, or against the source URL otherwise.
+
+ :param body: HTML document body.
+ :param source_url: URL from which the document was retrieved.
+ :param target_url: Exact URL to look for.
+ :return: Whether a supported HTML URL attribute references the target URL.
+ """
document = BeautifulSoup(body, "html.parser")
base_url = source_url
@@ -59,12 +69,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
for attribute, selectors in HTML_URL_ATTRIBUTES.items():
selector = ", ".join(
- [
- "{selector}[{attribute}]".format(
- selector=selector_string, attribute=attribute
- )
- for selector_string in selectors
- ]
+ [f"{selector_string}[{attribute}]" for selector_string in selectors]
)
for element in document.select(selector):
@@ -78,12 +83,25 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
def text_mentions_target(body: str, target_url: str) -> bool:
- """Check whether plain text contains the exact target IRI."""
+ """
+ Check whether plain text contains the exact target IRI.
+
+ :param body: Plain-text document body.
+ :param target_url: Exact IRI to look for.
+ :return: Whether the target IRI occurs in the document.
+ """
return any(match.group() == target_url for match in IRI_PATTERN.finditer(body))
def ensure_public_request(request: httpx.Request) -> None:
- """Prevent requests to non-public network addresses."""
+ """
+ Ensure that an HTTP request targets a public network address.
+
+ :param request: HTTP request to validate.
+ :raises VerificationError: If the URL has no hostname or resolves to a
+ non-public address.
+ :raises TemporaryFetchError: If the hostname cannot be resolved.
+ """
try:
if not is_public_url(str(request.url)):
raise VerificationError("Source resolves to a non-public address")
@@ -94,56 +112,83 @@ def ensure_public_request(request: httpx.Request) -> None:
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- """Fetch a source with limits on redirects, time, and response size."""
+ """
+ Fetch a webmention source document.
- with httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(
- current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0)
- ),
- follow_redirects=True,
- max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20),
- trust_env=False,
- event_hooks={"request": [ensure_public_request]},
- ) as client:
- with client.stream("GET", source_url) as response:
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
+ The request follows redirects while enforcing the configured timeout,
+ redirect limit, and maximum response size.
- max_source_bytes = current_app.config.get(
- "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000
- )
+ :param source_url: URL of the source document.
+ :return: HTTP response and response body.
+ :raises SourceGoneError: If the source returns HTTP 410.
+ :raises TemporaryFetchError: If the source returns a temporary HTTP error.
+ :raises VerificationError: If the source returns another unsuccessful HTTP
+ status or exceeds the configured maximum response size.
+ """
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
+ with (
+ httpx.Client(
+ headers={
+ "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ },
+ timeout=httpx.Timeout(
+ current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0)
+ ),
+ follow_redirects=True,
+ max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20),
+ trust_env=False,
+ event_hooks={"request": [ensure_public_request]},
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if status in {408, 425, 429} or 500 <= status <= 599:
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
+ max_source_bytes = current_app.config.get(
+ "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000
+ )
- if len(body) > max_source_bytes:
+ if (content_length := response.headers.get("Content-Length")) is not None:
+ try:
+ if int(content_length) > max_source_bytes:
raise VerificationError("Source document is too large")
+ except ValueError:
+ pass
+
+ body = bytearray()
+ for chunk in response.iter_bytes(chunk_size=64 * 1024):
+ body.extend(chunk)
- return response, bytes(body)
+ if len(body) > max_source_bytes:
+ raise VerificationError("Source document is too large")
+
+ return response, bytes(body)
def source_mentions_target(source_url: str, target_url: str) -> bool:
- """Fetch the source and verify it according to its media type."""
+ """
+ Fetch a source and check whether it mentions the target URL.
+
+ HTML and XHTML sources are inspected for supported URL attributes, while
+ plain-text sources are searched for the exact target IRI.
+
+ :param source_url: URL of the source document.
+ :param target_url: Exact target URL to look for.
+ :return: Whether the source mentions the target URL.
+ :raises VerificationError: If the source has an unsupported or missing
+ content type.
+ :raises SourceGoneError: If the source explicitly reports that it is gone.
+ :raises TemporaryFetchError: If fetching the source fails temporarily.
+ """
response, body = fetch_source(source_url)
@@ -170,7 +215,17 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
@huey.task(retries=2, retry_delay=50)
def verify_webmention(webmention_uuid: uuid.UUID) -> None:
- """Verify a ReceivedWebmention and store the result."""
+ """
+ Verify a received webmention and store its verification status.
+
+ Temporary fetch failures are stored before being re-raised so that Huey can
+ retry the task.
+
+ :param webmention_uuid: Identifier of the received webmention to verify.
+ :raises TemporaryFetchError: If fetching the source fails temporarily.
+ :raises httpx.RequestError: If an HTTP request error occurs while fetching
+ the source.
+ """
webmention = db.session.get(ReceivedWebmention, webmention_uuid)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index ff75697..fa6a47b 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -36,7 +36,13 @@ class ScannedSource:
def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """Derive the public source URL from its relative filesystem path."""
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
directory = relative_path.parent.as_posix()
return urljoin(base_url, f"{quote(directory, safe='/')}/")
@@ -45,7 +51,18 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
def canonical_url(
document: BeautifulSoup, *, relative_path: Path, base_url: str | None
) -> str | None:
- """Return the canonical URL declared by the document."""
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
href = link.get("href")
@@ -72,7 +89,13 @@ def canonical_url(
def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """Yield URL values from a microformats property."""
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
properties = entry.get("properties")
if not isinstance(properties, dict):
@@ -87,7 +110,14 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """Parse the source h-entry with mf2py."""
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
parsed = mf2py.parse(doc=str(element), url=base_url)
if not isinstance(parsed, dict):
@@ -114,7 +144,18 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
def primary_entry(
document: BeautifulSoup, source_url: str
) -> tuple[Tag, dict[str, Any]]:
- """Return and validate the source h-entry."""
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
entries = document.find_all(class_="h-entry")
if len(entries) != 1:
@@ -134,7 +175,14 @@ def primary_entry(
def content_element(entry: Tag) -> Tag:
- """Return the source h-entry's e-content element."""
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
contents = entry.find_all(class_="e-content")
if len(contents) != 1:
@@ -144,7 +192,18 @@ def content_element(entry: Tag) -> Tag:
def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """Resolve and validate a possible Webmention target URL."""
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
value = value.strip()
if not value:
@@ -169,7 +228,20 @@ def iter_targets(
source_url: str,
ignored_hostnames: tuple[str, ...] = (),
) -> Iterator[str]:
- """Yield outgoing Webmention targets from an h-entry."""
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
hrefs = (
href
for element in content.find_all("a", href=True)
@@ -195,7 +267,18 @@ def scan_source_file(
base_url: str | None,
ignored_hostnames: tuple[str, ...] = (),
) -> ScannedSource:
- """Parse one generated source document."""
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
document = BeautifulSoup(path.read_bytes(), "html5lib")
relative_path = path.relative_to(root)
@@ -232,7 +315,16 @@ def scan_source_file(
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
- """Create a source from a newly discovered document."""
+ """
+ Create a source model from a newly discovered document.
+
+ A sent Webmention is created for each discovered target at the initial source
+ revision.
+
+ :param scanned: Scanned source data.
+ :param scan_time: Time at which the source was discovered.
+ :return: Newly created source model.
+ """
source = Source(
path=scanned.path,
url=scanned.url,
@@ -254,7 +346,19 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
def update_source(
source: Source, scanned: ScannedSource, scan_time: datetime
) -> Source:
- """Update a source from a newly scanned revision."""
+ """
+ Update an existing source from newly scanned data.
+
+ A changed or previously deleted source receives a new revision. Existing sent
+ Webmentions are updated to reflect the current targets, and newly discovered
+ targets are added.
+
+ :param source: Existing source model to update.
+ :param scanned: Newly scanned source data.
+ :param scan_time: Time at which the source was scanned.
+ :return: Updated source model.
+ :raises SourceScanError: If the public URL of the source has changed.
+ """
if source.url != scanned.url:
raise SourceScanError(
f"Source path {source.path!r} changed public URL "
@@ -293,7 +397,15 @@ def update_source(
@huey.lock_task("scan-webmention-sources")
def scan_sources() -> None:
- """Scan generated source documents and queue pending Webmentions."""
+ """
+ Scan generated source documents and queue pending Webmentions.
+
+ New and changed sources are persisted, missing sources are marked as deleted,
+ and Webmentions requiring processing are queued for sending.
+
+ :raises RuntimeError: If the configured source directory or base URL is
+ invalid.
+ """
directory = current_app.config.get("WEBMENTIONS_SSG_SOURCE_DIRECTORY")
if directory is None:
@@ -399,4 +511,7 @@ def scan_sources() -> None:
@huey.task()
def manual_scan_sources() -> None:
+ """
+ Run a source scan as a Huey task.
+ """
return scan_sources()
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index fa9b624..629c538 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -1,6 +1,6 @@
import re
import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from urllib.parse import urljoin
import httpx
@@ -31,12 +31,23 @@ class PermanentSenderError(SenderError):
def temporary_http_status(status: int) -> bool:
- """Return whether an HTTP status should be retried."""
+ """
+ Return whether an HTTP status indicates a temporary failure.
+
+ :param status: HTTP response status code.
+ :return: Whether the request should be retried.
+ """
return status in {408, 425, 429} or 500 <= status <= 599
def ensure_public_request(request: httpx.Request) -> None:
- """Prevent requests to non-public network addresses."""
+ """
+ Ensure that an HTTP request targets a public network address.
+
+ :param request: HTTP request to validate.
+ :raises PermanentSenderError: If the request resolves to a non-public address.
+ :raises TemporarySenderError: If the request hostname cannot be resolved.
+ """
try:
if not is_public_url(str(request.url)):
raise PermanentSenderError("Request resolves to a non-public address")
@@ -45,7 +56,15 @@ def ensure_public_request(request: httpx.Request) -> None:
def resolve_endpoint(response: httpx.Response, href: str) -> str:
- """Resolve and validate a discovered Webmention endpoint."""
+ """
+ Resolve and validate a discovered Webmention endpoint.
+
+ :param response: Response from which the endpoint was discovered.
+ :param href: Endpoint reference to resolve.
+ :return: Absolute Webmention endpoint URL.
+ :raises PermanentSenderError: If the resolved endpoint is not an HTTP or
+ HTTPS URL.
+ """
endpoint = urljoin(str(response.url), href.strip())
if not is_http_url(endpoint):
@@ -55,7 +74,12 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str:
def parse_link_value(value: str) -> tuple[str, set[str]] | None:
- """Parse a Link header value into its target and relations."""
+ """
+ Parse a Link header value into its target and relations.
+
+ :param value: Link header value to parse.
+ :return: Link target and relation names, or ``None`` if the value is invalid.
+ """
value = value.strip()
if not value.startswith("<"):
@@ -80,7 +104,13 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None:
def endpoint_from_headers(response: httpx.Response) -> str | None:
- """Return the first Webmention endpoint advertised by HTTP Link."""
+ """
+ Find a Webmention endpoint in the response Link headers.
+
+ :param response: HTTP response whose headers should be inspected.
+ :return: First advertised Webmention endpoint, or ``None`` if none is found.
+ :raises PermanentSenderError: If a discovered endpoint is invalid.
+ """
for header in response.headers.get_list("Link"):
for value in LINK_SPLIT.split(header):
if (link := parse_link_value(value)) is None:
@@ -95,7 +125,16 @@ def endpoint_from_headers(response: httpx.Response) -> str | None:
def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
- """Return the first HTML Webmention endpoint in document order."""
+ """
+ Find a Webmention endpoint in an HTML document.
+
+ ``link`` and ``a`` elements are inspected in document order.
+
+ :param response: HTTP response from which the document was retrieved.
+ :param body: HTML document body.
+ :return: First advertised Webmention endpoint, or ``None`` if none is found.
+ :raises PermanentSenderError: If a discovered endpoint is invalid.
+ """
document = BeautifulSoup(body, "html5lib")
for element in document.find_all(["link", "a"], href=True):
@@ -124,7 +163,14 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None:
def read_target_body(response: httpx.Response) -> bytes:
- """Read a target document up to the configured size limit."""
+ """
+ Read a target document up to the configured size limit.
+
+ :param response: Streaming HTTP response to read.
+ :return: Response body.
+ :raises PermanentSenderError: If the document exceeds the configured maximum
+ size.
+ """
max_bytes = current_app.config.get("WEBMENTIONS_SSG_MAX_TARGET_BYTES", 1_000_000)
if (content_length := response.headers.get("Content-Length")) is not None:
@@ -146,7 +192,20 @@ def read_target_body(response: httpx.Response) -> bytes:
def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None:
- """Discover the Webmention endpoint advertised by a target."""
+ """
+ Discover the Webmention endpoint advertised by a target URL.
+
+ Endpoint discovery first checks the response to a HEAD request and then falls
+ back to a GET request, inspecting both HTTP Link headers and supported HTML
+ documents.
+
+ :param client: HTTP client to use for discovery requests.
+ :param target: Target URL whose Webmention endpoint should be discovered.
+ :return: Discovered Webmention endpoint, or ``None`` if none is advertised.
+ :raises TemporarySenderError: If the target returns a temporary HTTP failure.
+ :raises PermanentSenderError: If the target returns a permanent HTTP failure
+ or advertises an invalid endpoint.
+ """
head_response = client.head(target, headers=DISCOVERY_HEADERS)
if endpoint := endpoint_from_headers(head_response):
@@ -175,7 +234,15 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
def post_webmention(
client: httpx.Client, *, endpoint: str, source: str, target: str
) -> tuple[int, str | None]:
- """POST a Webmention and return its status code and status URL."""
+ """
+ Send a Webmention to a discovered endpoint.
+
+ :param client: HTTP client to use for the request.
+ :param endpoint: Webmention endpoint URL.
+ :param source: Source URL of the Webmention.
+ :param target: Target URL of the Webmention.
+ :return: HTTP response status code and optional status URL.
+ """
with client.stream(
"POST", endpoint, data={"source": source, "target": target}
) as response:
@@ -190,7 +257,16 @@ def post_webmention(
def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
- """Return whether an attempt still represents the desired revision."""
+ """
+ Check whether a send attempt still represents the desired revision.
+
+ The Webmention is refreshed from the database before comparing its desired
+ and processed revisions.
+
+ :param webmention: Sent Webmention being processed.
+ :param revision: Source revision represented by the current attempt.
+ :return: Whether the attempt is still current and requires processing.
+ """
db.session.refresh(webmention)
return webmention.desired_revision == revision and (
@@ -201,7 +277,18 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
@huey.task(retries=2, retry_delay=50)
def send_webmention(webmention_uuid: uuid.UUID) -> None:
- """Discover a receiver endpoint and send one Webmention."""
+ """
+ Discover a receiver endpoint and send a Webmention.
+
+ The result of the attempt is stored on the corresponding sent Webmention.
+ Temporary failures are re-raised so Huey can retry them, while permanent
+ failures mark the current revision as processed.
+
+ :param webmention_uuid: Identifier of the sent Webmention to process.
+ :raises TemporarySenderError: If endpoint discovery or delivery fails for a
+ potentially temporary reason.
+ :raises httpx.RequestError: If an HTTP request fails.
+ """
webmention = db.session.get(SentWebmention, webmention_uuid)
if webmention is None:
@@ -220,7 +307,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
source = webmention.source.url
target = webmention.target
- webmention.last_attempted_at = datetime.now(timezone.utc)
+ webmention.last_attempted_at = datetime.now(UTC)
webmention.endpoint = None
webmention.response_status = None
webmention.status_url = None
@@ -310,6 +397,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
webmention.response_status = response_status
webmention.status_url = status_url
- webmention.last_sent_at = datetime.now(timezone.utc)
+ webmention.last_sent_at = datetime.now(UTC)
db.session.commit()
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
index 51c037a..9cd261e 100644
--- a/webmentions_ssg/url_security.py
+++ b/webmentions_ssg/url_security.py
@@ -10,6 +10,14 @@ class AddressResolutionError(Exception):
def is_public_url(url: str) -> bool:
+ """
+ Check whether a URL resolves exclusively to public IP addresses.
+
+ :param url: URL whose hostname should be resolved.
+ :return: Whether all resolved addresses are globally routable.
+ :raises ValueError: If the URL does not contain a hostname.
+ :raises AddressResolutionError: If the hostname cannot be resolved.
+ """
hostname = urlsplit(url).hostname
if hostname is None:
@@ -30,7 +38,12 @@ def is_public_url(url: str) -> bool:
def is_http_url(url: str) -> bool:
- """Return whether a URL is an absolute HTTP or HTTPS URL."""
+ """
+ Check whether a URL is an absolute HTTP or HTTPS URL.
+
+ :param url: URL to validate.
+ :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ """
try:
parsed = urlsplit(url)
except ValueError: