diff options
| author | Dennis Fink | 2026-08-19 19:49:55 +0200 |
|---|---|---|
| committer | Dennis Fink | 2026-08-19 19:49:55 +0200 |
| commit | d0e245d63e014d268310976fc45429f08d2cbbe9 (patch) | |
| tree | ef2e96570723190baf02f05c2c4644a19ff1af78 | |
| parent | d4a03623544aac0ce911d8471ff4b033bc3255e1 (diff) | |
| download | webmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.tar.gz webmentions-ssg-d0e245d63e014d268310976fc45429f08d2cbbe9.zip | |
refactor(core): improve typing and test coverage
Restructure database and user CLI commands, make passwords write-only
model properties, and use the UTC datetime constant throughout the
application.
Add PEP 287-style documentation and expand receiver, scanner, sender,
and URL security tests to cover error paths and edge cases.
| -rw-r--r-- | tests/tasks/test_receiver.py | 52 | ||||
| -rw-r--r-- | tests/tasks/test_scanner.py | 203 | ||||
| -rw-r--r-- | tests/tasks/test_sender.py | 377 | ||||
| -rw-r--r-- | tests/test_url_security.py | 10 | ||||
| -rw-r--r-- | tests/test_views.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/__init__.py | 97 | ||||
| -rw-r--r-- | webmentions_ssg/auth.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 15 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 155 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 139 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 115 | ||||
| -rw-r--r-- | webmentions_ssg/url_security.py | 15 |
13 files changed, 1074 insertions, 129 deletions
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 811ab63..7e8f531 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -559,3 +559,55 @@ def test_ensure_public_request_rejects_url_without_hostname( receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) is_public_url.assert_called_once_with(SOURCE_URL) + + +@patch("webmentions_ssg.tasks.receiver.BeautifulSoup") +def test_html_mentions_target_ignores_non_string_attribute( + beautiful_soup: Mock, receiver: ModuleType +) -> None: + document = beautiful_soup.return_value + document.select_one.return_value = None + + element = Mock() + element.get.return_value = ["not", "a", "string"] + document.select.return_value = [element] + + assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL) + + +@patch("webmentions_ssg.tasks.receiver.httpx.Client") +def test_fetch_source_ignores_invalid_content_length( + httpx_client: Mock, app: Flask, receiver: ModuleType +) -> None: + response = httpx.Response( + 200, + headers={"Content-Type": "text/html", "Content-Length": "invalid"}, + content=b"<p>Document</p>", + request=httpx.Request("GET", SOURCE_URL), + ) + set_stream_response(httpx_client, response) + + with app.app_context(): + fetched_response, body = receiver.fetch_source(SOURCE_URL) + + assert fetched_response is response + assert body == b"<p>Document</p>" + + +@patch("webmentions_ssg.tasks.receiver.fetch_source") +def test_source_mentions_target_falls_back_for_unknown_encoding( + fetch_source: Mock, receiver: ModuleType +) -> None: + body = f"Reply to {TARGET_URL}".encode() + + response = httpx.Response( + 200, + headers={"Content-Type": "text/plain"}, + content=body, + request=httpx.Request("GET", SOURCE_URL), + ) + response.encoding = "unknown-encoding" + fetch_source.return_value = response, body + + assert receiver.source_mentions_target(SOURCE_URL, TARGET_URL) + fetch_source.assert_called_once_with(SOURCE_URL) diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 65031b4..89e2995 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -1556,3 +1556,206 @@ def test_scan_source_ignores_reaction_to_matching_hostname( ) assert scanned.targets == frozenset() + + +def test_canonical_url_rejects_non_http_resolved_url( + scanner_module: ModuleType, +) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="mailto:example@example.com" + > + </head> + </html> + """ + ) + + with pytest.raises( + scanner_module.SourceScanError, + match="Canonical URL is not a valid HTTP or HTTPS URL", + ): + scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_property_urls_ignores_non_mapping_properties( + scanner_module: ModuleType, +) -> None: + entry = {"properties": "invalid"} + + assert tuple(scanner_module.property_urls(entry, "url")) == () + + +@pytest.mark.parametrize( + ("parsed", "message"), + [ + pytest.param( + [], + "Microformats parser did not return a document object", + id="non-object-document", + ), + pytest.param( + {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" + ), + pytest.param( + {"items": [{"type": ["h-card"]}]}, + "Parsed microformats item is not an h-entry", + id="wrong-item-type", + ), + ], +) +def test_parse_entry_rejects_invalid_parser_output( + scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed) + + with pytest.raises(scanner_module.SourceScanError, match=message): + scanner_module.parse_entry(element, SOURCE_URL) + + +@pytest.mark.parametrize( + "value", + [ + pytest.param(" ", id="empty"), + pytest.param("mailto:example@example.com", id="non-http"), + ], +) +def test_normalize_target_ignores_invalid_target( + scanner_module: ModuleType, value: str +) -> None: + assert ( + scanner_module.normalize_target( + value, base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) + + +def test_changed_source_url_is_rejected( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + configure_scanner(app, tmp_path) + + write_post( + tmp_path, + "example", + """ + <p>Hello world.</p> + """, + ) + + run_scan(app, scanner_module) + + changed_url = f"{BASE_URL}changed/" + + write_post( + tmp_path, + "example", + """ + <p>Hello world.</p> + """, + canonical=changed_url, + entry_url=changed_url, + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + + assert source is not None + assert source.url == SOURCE_URL + assert source.revision == 1 + + assert "changed public URL" in caplog.text + + +def test_scan_sources_returns_when_source_directory_is_not_configured( + app: Flask, scanner_module: ModuleType +) -> None: + app.config["WEBMENTIONS_SSG_SOURCE_DIRECTORY"] = None + + run_scan(app, scanner_module) + + with app.app_context(): + assert db.session.scalar(sa.select(Source)) is None + + +def test_scan_sources_rejects_source_directory_that_is_not_directory( + app: Flask, scanner_module: ModuleType, tmp_path: Path +) -> None: + path = tmp_path / "source.html" + path.write_text("<p>Hello world.</p>", encoding="utf-8") + + configure_scanner(app, path) + + with pytest.raises(RuntimeError, match="Source directory is not a directory"): + run_scan(app, scanner_module) + + +def test_scan_sources_ignores_root_index( + app: Flask, scanner_module: ModuleType, tmp_path: Path +) -> None: + configure_scanner(app, tmp_path) + + (tmp_path / "index.html").write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{BASE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{BASE_URL}"> + Permalink + </a> + + <div class="e-content"> + <p>Blog index.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + write_post( + tmp_path, + "example", + """ + <p>Hello world.</p> + """, + ) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + + assert len(sources) == 1 + assert sources[0].path == "example/index.html" diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 86799f7..1c8ef99 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from types import ModuleType from unittest.mock import MagicMock, Mock @@ -10,6 +10,7 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source +from webmentions_ssg.url_security import AddressResolutionError SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" @@ -30,7 +31,7 @@ def create_sent_webmention( response_status: int | None = None, status_url: str | None = None, ) -> uuid.UUID: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) with app.app_context(): source = Source( @@ -786,3 +787,375 @@ def test_send_webmention_ignores_unknown_identifier( sender_module.send_webmention.call_local(identifier) assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text + + +def test_ensure_public_request_accepts_public_url( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_public_url = Mock(return_value=True) + monkeypatch.setattr(sender_module, "is_public_url", is_public_url) + + sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) + + is_public_url.assert_called_once_with(TARGET_URL) + + +def test_ensure_public_request_rejects_non_public_address( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_public_url = Mock(return_value=False) + monkeypatch.setattr(sender_module, "is_public_url", is_public_url) + + with pytest.raises( + sender_module.PermanentSenderError, + match="Request resolves to a non-public address", + ): + sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) + + is_public_url.assert_called_once_with(TARGET_URL) + + +def test_ensure_public_request_maps_dns_failure_to_temporary_error( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_public_url = Mock( + side_effect=AddressResolutionError("Could not resolve hostname") + ) + monkeypatch.setattr(sender_module, "is_public_url", is_public_url) + + with pytest.raises( + sender_module.TemporarySenderError, + match="Request hostname could not be resolved", + ): + sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) + + is_public_url.assert_called_once_with(TARGET_URL) + + +def test_resolve_endpoint_rejects_non_http_url(sender_module: ModuleType) -> None: + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + with pytest.raises( + sender_module.PermanentSenderError, match="Invalid Webmention endpoint" + ): + sender_module.resolve_endpoint(response, "mailto:example@example.com") + + +def test_endpoint_from_headers_ignores_malformed_link( + sender_module: ModuleType, +) -> None: + response = httpx.Response( + 200, + headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")], + request=httpx.Request("GET", TARGET_URL), + ) + + assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL + + +def test_endpoint_from_html_accepts_string_rel( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + element = Mock() + element.get.side_effect = lambda name: { + "rel": "webmention alternate", + "href": "/webmention", + }[name] + + document = Mock() + document.find_all.return_value = [element] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL + + +def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + missing_rel = Mock() + missing_rel.get.return_value = None + + invalid_href = Mock() + invalid_href.get.side_effect = lambda name: { + "rel": ["webmention"], + "href": ["invalid"], + }[name] + + document = Mock() + document.find_all.return_value = [missing_rel, invalid_href] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") is None + + +def test_read_target_body_rejects_declared_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["WEBMENTIONS_SSG_MAX_TARGET_BYTES"] = 10 + + response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_read_target_body_ignores_invalid_content_length( + app: Flask, sender_module: ModuleType +) -> None: + response = httpx.Response( + 200, headers={"Content-Length": "invalid"}, content=b"Document" + ) + + with app.app_context(): + body = sender_module.read_target_body(response) + + assert body == b"Document" + + +def test_read_target_body_rejects_streamed_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["WEBMENTIONS_SSG_MAX_TARGET_BYTES"] = 10 + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + response = httpx.Response(200, stream=BodyStream()) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_discovery_uses_link_header_from_get( + app: Flask, sender_module: ModuleType +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + pytest.param(503, "TemporarySenderError", id="temporary"), + pytest.param(404, "PermanentSenderError", id="permanent"), + ], +) +def test_discovery_maps_get_status_to_exception( + app: Flask, sender_module: ModuleType, status_code: int, exception_name: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(status_code) + + exception_type = getattr(sender_module, exception_name) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + +def test_discovery_returns_none_for_unsupported_media_type( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, headers={"Content-Type": "application/json"}, content=b"{}" + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +@pytest.mark.parametrize( + ("status_code", "location", "expected_status_url"), + [ + pytest.param( + 201, "/webmention/status/123", STATUS_URL, id="created-with-location" + ), + pytest.param(202, None, None, id="accepted"), + ], +) +def test_post_webmention( + sender_module: ModuleType, + status_code: int, + location: str | None, + expected_status_url: str | None, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + headers = {} + + if location is not None: + headers["Location"] = location + + return httpx.Response(status_code, headers=headers) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (status_code, expected_status_url) + assert len(requests) == 1 + + request = requests[0] + + assert request.method == "POST" + assert str(request.url) == ENDPOINT_URL + + form = httpx.QueryParams(request.content.decode()) + + assert form["source"] == SOURCE_URL + assert form["target"] == TARGET_URL + + +def test_send_webmention_ignores_unsupported_result_for_superseded_revision( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention(app) + + _, _, discover, post = mock_sender_requests( + sender_module, monkeypatch, endpoint=None + ) + + def supersede_revision(*args, **kwargs) -> None: + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + + webmention.desired_revision = 2 + db.session.commit() + + discover.side_effect = supersede_revision + + sender_module.send_webmention.call_local(identifier) + + post.assert_not_called() + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.processed_revision is None + assert webmention.sent_revision is None + assert webmention.status is None + assert webmention.pending + + +def test_send_webmention_ignores_temporary_failure_for_superseded_revision( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention(app) + + _, _, _, post = mock_sender_requests(sender_module, monkeypatch) + + def supersede_revision(*args, **kwargs) -> tuple[int, None]: + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + + webmention.desired_revision = 2 + db.session.commit() + + raise sender_module.TemporarySenderError("Temporary failure") + + post.side_effect = supersede_revision + + sender_module.send_webmention.call_local(identifier) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.processed_revision is None + assert webmention.sent_revision is None + assert webmention.status is None + assert webmention.pending + + +def test_send_webmention_ignores_permanent_failure_for_superseded_revision( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention(app) + + _, _, _, post = mock_sender_requests(sender_module, monkeypatch) + + def supersede_revision(*args, **kwargs) -> tuple[int, None]: + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + + webmention.desired_revision = 2 + db.session.commit() + + return (400, None) + + post.side_effect = supersede_revision + + sender_module.send_webmention.call_local(identifier) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.processed_revision is None + assert webmention.sent_revision is None + assert webmention.status is None + assert webmention.pending diff --git a/tests/test_url_security.py b/tests/test_url_security.py index ffa0378..70d5146 100644 --- a/tests/test_url_security.py +++ b/tests/test_url_security.py @@ -3,7 +3,11 @@ from unittest.mock import Mock, patch import dns.resolver import pytest -from webmentions_ssg.url_security import AddressResolutionError, is_public_url +from webmentions_ssg.url_security import ( + AddressResolutionError, + is_http_url, + is_public_url, +) @patch("webmentions_ssg.url_security.dns.resolver.resolve_name") @@ -66,3 +70,7 @@ def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> No is_public_url("/relative/url") resolve_name.assert_not_called() + + +def test_is_http_url_rejects_malformed_url() -> None: + assert not is_http_url("http://[::1") diff --git a/tests/test_views.py b/tests/test_views.py index aac281e..67ed190 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import Mock, call, patch import pytest @@ -19,7 +19,7 @@ from webmentions_ssg.models import ( def log_in(app: Flask, client: FlaskClient) -> None: with app.app_context(): - user = User(username="admin") + user = User(username="admin", password="test") db.session.add(user) db.session.commit() @@ -32,7 +32,7 @@ def log_in(app: Flask, client: FlaskClient) -> None: def create_sent_source(app: Flask) -> uuid.UUID: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) with app.app_context(): source = Source( diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index b799678..3928195 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -11,6 +11,7 @@ import click import coloredlogs import sqlalchemy as sa from flask import Flask +from flask.cli import AppGroup from flask_bootstrap import Bootstrap5 from flask_migrate import Migrate, upgrade from flask_sqlalchemy import SQLAlchemy @@ -35,14 +36,25 @@ DATABASE = SQLAlchemy(model_class=Base) MIGRATE = Migrate() HUEY = Huey() +USERS_CLI = AppGroup("users", help="Handle users") +DB_CLI = AppGroup("db", help="Handle database") + def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: - """Creates and configures the Flask application.""" + """ + Create and configure the Flask application. + + The application configuration is selected based on the development + environment unless an explicit configuration class is provided. Logging, + extensions, authentication, Huey, blueprints, and CLI commands are then + initialized. - development = os.environ.get( - "WEBMENTIONS_SSG_DEV", - "", - ).lower() in { + :param config_class: Configuration class to use, or ``None`` to select the + development or production configuration automatically. + :return: Configured Flask application. + """ + + development = os.environ.get("WEBMENTIONS_SSG_DEV", "").lower() in { "1", "true", "yes", @@ -53,8 +65,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: config_class = DevelopmentConfig if development else ProductionConfig app = Flask( - __name__, - instance_path=str(Path("testing").absolute()) if development else None, + __name__, instance_path=str(Path("testing").absolute()) if development else None ) app.config.from_object(config_class) @@ -84,9 +95,7 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: werkzeug_logger.addHandler(log_file_handler) coloredlogs.install( - level=app.config["LOG_LEVEL"], - logger=app.logger, - fmt=app.config["LOG_FORMAT"], + level=app.config["LOG_LEVEL"], logger=app.logger, fmt=app.config["LOG_FORMAT"] ) coloredlogs.install( level=app.config["LOG_LEVEL"], @@ -106,44 +115,54 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: @app.context_processor def inject_version(): - return dict(VERSION=VERSION) + """ + Add the application version to the template context. + + :return: Template context containing the application version. + """ + return {"VERSION": VERSION} from .views import root_page app.register_blueprint(root_page) - @app.cli.command("create-db") - def create_db() -> None: - """Create or update the database to the latest migration.""" - upgrade() + app.cli.add_command(USERS_CLI) - @app.cli.command("create-user") - @click.argument("username") - @click.password_option( - confirmation_prompt=True, - ) - def create_user( - username: str, - password: str, - ) -> None: - """Create a user.""" + return app - from .models import User - if ( - DATABASE.session.scalar(sa.select(User).where(User.username == username)) - is not None - ): - raise click.ClickException(f"User {username!r} already exists.") +@USERS_CLI.command("create", help="Create or add users") +@click.option("--username", prompt=True) +@click.password_option(confirmation_prompt=True) +def create_user(username: str, password: str) -> None: + """ + Create a new user. - user = User( - username=username, - ) - user.set_password(password) + :param username: Username of the user to create. + :param password: Plain-text password to hash and store for the user. + :raises click.ClickException: If a user with the given username already + exists. + """ - DATABASE.session.add(user) - DATABASE.session.commit() + from .models import User - click.echo(f"Created user {username!r}.") + if ( + DATABASE.session.scalar(sa.select(User).where(User.username == username)) + is not None + ): + raise click.ClickException(f"User {username!r} already exists.") - return app + user = User(username=username, password=password) + + DATABASE.session.add(user) + DATABASE.session.commit() + + click.echo(f"Created user {username!r}.") + + +@DB_CLI.command("create") +def create_db() -> None: + """ + Upgrade the database to the latest migration revision. + """ + upgrade() diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py index f8fd28c..7bd93a3 100644 --- a/webmentions_ssg/auth.py +++ b/webmentions_ssg/auth.py @@ -9,5 +9,11 @@ AUTH.login_view = "root.login" @AUTH.user_loader -def load_user(id): +def load_user(id: int) -> User | None: + """ + Load a user from the database by identifier. + + :param id: User identifier stored in the login session. + :return: Matching user, or ``None`` if no user exists with the identifier. + """ return db.session.get(User, int(id)) diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index b6679e8..f89e587 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -1,7 +1,7 @@ from __future__ import annotations import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from enum import StrEnum from flask_login import UserMixin @@ -38,7 +38,12 @@ class User(UserMixin, Base): def __repr__(self) -> str: return f"<User {self.username}>" - def set_password(self, password: str) -> None: + @property + def password(self) -> None: + raise AttributeError("Password is write-only") + + @password.setter + def password(self, password: str) -> None: self.password_hash = generate_password_hash(password) def check_password(self, password: str) -> bool: @@ -60,8 +65,8 @@ class ReceivedWebmention(Base): updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - onupdate=lambda: datetime.now(timezone.utc), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), nullable=False, ) @@ -185,4 +190,4 @@ class SentWebmention(Base): def uuid7_to_datetime(identifier: uuid.UUID) -> datetime: - return datetime.fromtimestamp(identifier.time / 1000, tz=timezone.utc) + return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index e53fce5..3bfe861 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -4,9 +4,18 @@ from .. import HUEY, create_app def create_consumer() -> Huey: + """ + Create and configure the Huey consumer. + + The application is initialized, task modules are imported to register their + tasks, and the source scanner is registered with the configured periodic + schedule. + + :return: Configured Huey instance. + """ app = create_app() - from . import receiver, scanner, sender # noqa: E402, F401 + from . import receiver, scanner, sender # noqa: F401 schedule = crontab( *app.config["WEBMENTIONS_SSG_SCANNER_SCHEDULE"].split(), strict=True diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 39def73..4aba3d8 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -47,7 +47,17 @@ HTML_URL_ATTRIBUTES = { def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """Check valid HTML URL attributes for the exact target URL.""" + """ + Check whether an HTML document mentions the target URL. + + URL references are resolved against the document's ``base`` element when + present, or against the source URL otherwise. + + :param body: HTML document body. + :param source_url: URL from which the document was retrieved. + :param target_url: Exact URL to look for. + :return: Whether a supported HTML URL attribute references the target URL. + """ document = BeautifulSoup(body, "html.parser") base_url = source_url @@ -59,12 +69,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: for attribute, selectors in HTML_URL_ATTRIBUTES.items(): selector = ", ".join( - [ - "{selector}[{attribute}]".format( - selector=selector_string, attribute=attribute - ) - for selector_string in selectors - ] + [f"{selector_string}[{attribute}]" for selector_string in selectors] ) for element in document.select(selector): @@ -78,12 +83,25 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: def text_mentions_target(body: str, target_url: str) -> bool: - """Check whether plain text contains the exact target IRI.""" + """ + Check whether plain text contains the exact target IRI. + + :param body: Plain-text document body. + :param target_url: Exact IRI to look for. + :return: Whether the target IRI occurs in the document. + """ return any(match.group() == target_url for match in IRI_PATTERN.finditer(body)) def ensure_public_request(request: httpx.Request) -> None: - """Prevent requests to non-public network addresses.""" + """ + Ensure that an HTTP request targets a public network address. + + :param request: HTTP request to validate. + :raises VerificationError: If the URL has no hostname or resolves to a + non-public address. + :raises TemporaryFetchError: If the hostname cannot be resolved. + """ try: if not is_public_url(str(request.url)): raise VerificationError("Source resolves to a non-public address") @@ -94,56 +112,83 @@ def ensure_public_request(request: httpx.Request) -> None: def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - """Fetch a source with limits on redirects, time, and response size.""" + """ + Fetch a webmention source document. - with httpx.Client( - headers={ - "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", - }, - timeout=httpx.Timeout( - current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) - ), - follow_redirects=True, - max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), - trust_env=False, - event_hooks={"request": [ensure_public_request]}, - ) as client: - with client.stream("GET", source_url) as response: - match response.status_code: - case 200: - pass - case 410: - raise SourceGoneError("Source returned HTTP 410") - case status: - if status in {408, 425, 429} or 500 <= status <= 599: - raise TemporaryFetchError(f"Source returned HTTP {status}") - else: - raise VerificationError(f"Source returned HTTP {status}") + The request follows redirects while enforcing the configured timeout, + redirect limit, and maximum response size. - max_source_bytes = current_app.config.get( - "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000 - ) + :param source_url: URL of the source document. + :return: HTTP response and response body. + :raises SourceGoneError: If the source returns HTTP 410. + :raises TemporaryFetchError: If the source returns a temporary HTTP error. + :raises VerificationError: If the source returns another unsuccessful HTTP + status or exceeds the configured maximum response size. + """ - if (content_length := response.headers.get("Content-Length")) is not None: - try: - if int(content_length) > max_source_bytes: - raise VerificationError("Source document is too large") - except ValueError: - pass + with ( + httpx.Client( + headers={ + "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", + }, + timeout=httpx.Timeout( + current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) + ), + follow_redirects=True, + max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), + trust_env=False, + event_hooks={"request": [ensure_public_request]}, + ) as client, + client.stream("GET", source_url) as response, + ): + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if status in {408, 425, 429} or 500 <= status <= 599: + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") - body = bytearray() - for chunk in response.iter_bytes(chunk_size=64 * 1024): - body.extend(chunk) + max_source_bytes = current_app.config.get( + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000 + ) - if len(body) > max_source_bytes: + if (content_length := response.headers.get("Content-Length")) is not None: + try: + if int(content_length) > max_source_bytes: raise VerificationError("Source document is too large") + except ValueError: + pass + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) - return response, bytes(body) + if len(body) > max_source_bytes: + raise VerificationError("Source document is too large") + + return response, bytes(body) def source_mentions_target(source_url: str, target_url: str) -> bool: - """Fetch the source and verify it according to its media type.""" + """ + Fetch a source and check whether it mentions the target URL. + + HTML and XHTML sources are inspected for supported URL attributes, while + plain-text sources are searched for the exact target IRI. + + :param source_url: URL of the source document. + :param target_url: Exact target URL to look for. + :return: Whether the source mentions the target URL. + :raises VerificationError: If the source has an unsupported or missing + content type. + :raises SourceGoneError: If the source explicitly reports that it is gone. + :raises TemporaryFetchError: If fetching the source fails temporarily. + """ response, body = fetch_source(source_url) @@ -170,7 +215,17 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: - """Verify a ReceivedWebmention and store the result.""" + """ + Verify a received webmention and store its verification status. + + Temporary fetch failures are stored before being re-raised so that Huey can + retry the task. + + :param webmention_uuid: Identifier of the received webmention to verify. + :raises TemporaryFetchError: If fetching the source fails temporarily. + :raises httpx.RequestError: If an HTTP request error occurs while fetching + the source. + """ webmention = db.session.get(ReceivedWebmention, webmention_uuid) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index ff75697..fa6a47b 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -36,7 +36,13 @@ class ScannedSource: def source_url_for_path(relative_path: Path, base_url: str) -> str: - """Derive the public source URL from its relative filesystem path.""" + """ + Derive the public source URL from a relative filesystem path. + + :param relative_path: Path of the source document relative to the source root. + :param base_url: Base URL under which source documents are published. + :return: Public URL corresponding to the source document. + """ directory = relative_path.parent.as_posix() return urljoin(base_url, f"{quote(directory, safe='/')}/") @@ -45,7 +51,18 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: def canonical_url( document: BeautifulSoup, *, relative_path: Path, base_url: str | None ) -> str | None: - """Return the canonical URL declared by the document.""" + """ + Return the canonical URL declared by a source document. + + Relative canonical URLs are resolved against the configured source base URL. + + :param document: Parsed HTML source document. + :param relative_path: Path of the source document relative to the source root. + :param base_url: Configured source base URL, if available. + :return: Canonical URL, or ``None`` if none is declared. + :raises SourceScanError: If the canonical URL is empty, cannot be resolved, + or does not resolve to an HTTP or HTTPS URL. + """ if (link := document.select_one('link[rel~="canonical"][href]')) is not None: href = link.get("href") @@ -72,7 +89,13 @@ def canonical_url( def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """Yield URL values from a microformats property.""" + """ + Yield URL values from a microformats property. + + :param entry: Parsed microformats entry. + :param property_name: Name of the property to inspect. + :return: Iterator over string values of the property. + """ properties = entry.get("properties") if not isinstance(properties, dict): @@ -87,7 +110,14 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """Parse the source h-entry with mf2py.""" + """ + Parse and validate an h-entry with mf2py. + + :param element: HTML element containing the h-entry. + :param base_url: Base URL used when parsing the microformats data. + :return: Parsed h-entry. + :raises SourceScanError: If parsing does not produce exactly one valid h-entry. + """ parsed = mf2py.parse(doc=str(element), url=base_url) if not isinstance(parsed, dict): @@ -114,7 +144,18 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: def primary_entry( document: BeautifulSoup, source_url: str ) -> tuple[Tag, dict[str, Any]]: - """Return and validate the source h-entry.""" + """ + Return and validate the primary h-entry of a source document. + + The document must contain exactly one h-entry whose ``u-url`` matches the + source URL. + + :param document: Parsed HTML source document. + :param source_url: Public URL of the source document. + :return: h-entry element and its parsed microformats representation. + :raises SourceScanError: If the document does not contain exactly one h-entry + or its ``u-url`` does not match the source URL. + """ entries = document.find_all(class_="h-entry") if len(entries) != 1: @@ -134,7 +175,14 @@ def primary_entry( def content_element(entry: Tag) -> Tag: - """Return the source h-entry's e-content element.""" + """ + Return the e-content element of an h-entry. + + :param entry: HTML element containing the h-entry. + :return: The h-entry's e-content element. + :raises SourceScanError: If the h-entry does not contain exactly one + e-content element. + """ contents = entry.find_all(class_="e-content") if len(contents) != 1: @@ -144,7 +192,18 @@ def content_element(entry: Tag) -> Tag: def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """Resolve and validate a possible Webmention target URL.""" + """ + Resolve and validate a possible Webmention target URL. + + Empty values, non-HTTP URLs, and URLs referring to the source itself are + discarded. + + :param value: URL reference to normalize. + :param base_url: Base URL against which relative references are resolved. + :param source_url: URL of the source document. + :return: Normalized target URL, or ``None`` if the value is not a valid + Webmention target. + """ value = value.strip() if not value: @@ -169,7 +228,20 @@ def iter_targets( source_url: str, ignored_hostnames: tuple[str, ...] = (), ) -> Iterator[str]: - """Yield outgoing Webmention targets from an h-entry.""" + """ + Yield outgoing Webmention targets from an h-entry. + + Targets are collected from links in the entry content and supported + microformats reaction properties. Invalid, self-referencing, and ignored + targets are excluded. + + :param content: e-content element of the h-entry. + :param mf2_entry: Parsed microformats representation of the h-entry. + :param base_url: Base URL against which relative targets are resolved. + :param source_url: URL of the source document. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Iterator over outgoing Webmention target URLs. + """ hrefs = ( href for element in content.find_all("a", href=True) @@ -195,7 +267,18 @@ def scan_source_file( base_url: str | None, ignored_hostnames: tuple[str, ...] = (), ) -> ScannedSource: - """Parse one generated source document.""" + """ + Scan one generated source document for outgoing Webmentions. + + :param path: Path of the generated HTML document. + :param root: Root directory containing generated source documents. + :param base_url: Configured source base URL, if available. + :param ignored_hostnames: Hostname patterns whose targets should be ignored. + :return: Scanned source metadata and discovered targets. + :raises SourceScanError: If the document cannot be safely interpreted as a + Webmention source. + :raises OSError: If the source document cannot be read. + """ document = BeautifulSoup(path.read_bytes(), "html5lib") relative_path = path.relative_to(root) @@ -232,7 +315,16 @@ def scan_source_file( def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: - """Create a source from a newly discovered document.""" + """ + Create a source model from a newly discovered document. + + A sent Webmention is created for each discovered target at the initial source + revision. + + :param scanned: Scanned source data. + :param scan_time: Time at which the source was discovered. + :return: Newly created source model. + """ source = Source( path=scanned.path, url=scanned.url, @@ -254,7 +346,19 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: def update_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: - """Update a source from a newly scanned revision.""" + """ + Update an existing source from newly scanned data. + + A changed or previously deleted source receives a new revision. Existing sent + Webmentions are updated to reflect the current targets, and newly discovered + targets are added. + + :param source: Existing source model to update. + :param scanned: Newly scanned source data. + :param scan_time: Time at which the source was scanned. + :return: Updated source model. + :raises SourceScanError: If the public URL of the source has changed. + """ if source.url != scanned.url: raise SourceScanError( f"Source path {source.path!r} changed public URL " @@ -293,7 +397,15 @@ def update_source( @huey.lock_task("scan-webmention-sources") def scan_sources() -> None: - """Scan generated source documents and queue pending Webmentions.""" + """ + Scan generated source documents and queue pending Webmentions. + + New and changed sources are persisted, missing sources are marked as deleted, + and Webmentions requiring processing are queued for sending. + + :raises RuntimeError: If the configured source directory or base URL is + invalid. + """ directory = current_app.config.get("WEBMENTIONS_SSG_SOURCE_DIRECTORY") if directory is None: @@ -399,4 +511,7 @@ def scan_sources() -> None: @huey.task() def manual_scan_sources() -> None: + """ + Run a source scan as a Huey task. + """ return scan_sources() diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index fa9b624..629c538 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -1,6 +1,6 @@ import re import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from urllib.parse import urljoin import httpx @@ -31,12 +31,23 @@ class PermanentSenderError(SenderError): def temporary_http_status(status: int) -> bool: - """Return whether an HTTP status should be retried.""" + """ + Return whether an HTTP status indicates a temporary failure. + + :param status: HTTP response status code. + :return: Whether the request should be retried. + """ return status in {408, 425, 429} or 500 <= status <= 599 def ensure_public_request(request: httpx.Request) -> None: - """Prevent requests to non-public network addresses.""" + """ + Ensure that an HTTP request targets a public network address. + + :param request: HTTP request to validate. + :raises PermanentSenderError: If the request resolves to a non-public address. + :raises TemporarySenderError: If the request hostname cannot be resolved. + """ try: if not is_public_url(str(request.url)): raise PermanentSenderError("Request resolves to a non-public address") @@ -45,7 +56,15 @@ def ensure_public_request(request: httpx.Request) -> None: def resolve_endpoint(response: httpx.Response, href: str) -> str: - """Resolve and validate a discovered Webmention endpoint.""" + """ + Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :return: Absolute Webmention endpoint URL. + :raises PermanentSenderError: If the resolved endpoint is not an HTTP or + HTTPS URL. + """ endpoint = urljoin(str(response.url), href.strip()) if not is_http_url(endpoint): @@ -55,7 +74,12 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str: def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """Parse a Link header value into its target and relations.""" + """ + Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :return: Link target and relation names, or ``None`` if the value is invalid. + """ value = value.strip() if not value.startswith("<"): @@ -80,7 +104,13 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None: def endpoint_from_headers(response: httpx.Response) -> str | None: - """Return the first Webmention endpoint advertised by HTTP Link.""" + """ + Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :return: First advertised Webmention endpoint, or ``None`` if none is found. + :raises PermanentSenderError: If a discovered endpoint is invalid. + """ for header in response.headers.get_list("Link"): for value in LINK_SPLIT.split(header): if (link := parse_link_value(value)) is None: @@ -95,7 +125,16 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """Return the first HTML Webmention endpoint in document order.""" + """ + Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :return: First advertised Webmention endpoint, or ``None`` if none is found. + :raises PermanentSenderError: If a discovered endpoint is invalid. + """ document = BeautifulSoup(body, "html5lib") for element in document.find_all(["link", "a"], href=True): @@ -124,7 +163,14 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: def read_target_body(response: httpx.Response) -> bytes: - """Read a target document up to the configured size limit.""" + """ + Read a target document up to the configured size limit. + + :param response: Streaming HTTP response to read. + :return: Response body. + :raises PermanentSenderError: If the document exceeds the configured maximum + size. + """ max_bytes = current_app.config.get("WEBMENTIONS_SSG_MAX_TARGET_BYTES", 1_000_000) if (content_length := response.headers.get("Content-Length")) is not None: @@ -146,7 +192,20 @@ def read_target_body(response: httpx.Response) -> bytes: def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """Discover the Webmention endpoint advertised by a target.""" + """ + Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :return: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises TemporarySenderError: If the target returns a temporary HTTP failure. + :raises PermanentSenderError: If the target returns a permanent HTTP failure + or advertises an invalid endpoint. + """ head_response = client.head(target, headers=DISCOVERY_HEADERS) if endpoint := endpoint_from_headers(head_response): @@ -175,7 +234,15 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non def post_webmention( client: httpx.Client, *, endpoint: str, source: str, target: str ) -> tuple[int, str | None]: - """POST a Webmention and return its status code and status URL.""" + """ + Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :return: HTTP response status code and optional status URL. + """ with client.stream( "POST", endpoint, data={"source": source, "target": target} ) as response: @@ -190,7 +257,16 @@ def post_webmention( def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: - """Return whether an attempt still represents the desired revision.""" + """ + Check whether a send attempt still represents the desired revision. + + The Webmention is refreshed from the database before comparing its desired + and processed revisions. + + :param webmention: Sent Webmention being processed. + :param revision: Source revision represented by the current attempt. + :return: Whether the attempt is still current and requires processing. + """ db.session.refresh(webmention) return webmention.desired_revision == revision and ( @@ -201,7 +277,18 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: @huey.task(retries=2, retry_delay=50) def send_webmention(webmention_uuid: uuid.UUID) -> None: - """Discover a receiver endpoint and send one Webmention.""" + """ + Discover a receiver endpoint and send a Webmention. + + The result of the attempt is stored on the corresponding sent Webmention. + Temporary failures are re-raised so Huey can retry them, while permanent + failures mark the current revision as processed. + + :param webmention_uuid: Identifier of the sent Webmention to process. + :raises TemporarySenderError: If endpoint discovery or delivery fails for a + potentially temporary reason. + :raises httpx.RequestError: If an HTTP request fails. + """ webmention = db.session.get(SentWebmention, webmention_uuid) if webmention is None: @@ -220,7 +307,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: source = webmention.source.url target = webmention.target - webmention.last_attempted_at = datetime.now(timezone.utc) + webmention.last_attempted_at = datetime.now(UTC) webmention.endpoint = None webmention.response_status = None webmention.status_url = None @@ -310,6 +397,6 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.response_status = response_status webmention.status_url = status_url - webmention.last_sent_at = datetime.now(timezone.utc) + webmention.last_sent_at = datetime.now(UTC) db.session.commit() diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py index 51c037a..9cd261e 100644 --- a/webmentions_ssg/url_security.py +++ b/webmentions_ssg/url_security.py @@ -10,6 +10,14 @@ class AddressResolutionError(Exception): def is_public_url(url: str) -> bool: + """ + Check whether a URL resolves exclusively to public IP addresses. + + :param url: URL whose hostname should be resolved. + :return: Whether all resolved addresses are globally routable. + :raises ValueError: If the URL does not contain a hostname. + :raises AddressResolutionError: If the hostname cannot be resolved. + """ hostname = urlsplit(url).hostname if hostname is None: @@ -30,7 +38,12 @@ def is_public_url(url: str) -> bool: def is_http_url(url: str) -> bool: - """Return whether a URL is an absolute HTTP or HTTPS URL.""" + """ + Check whether a URL is an absolute HTTP or HTTPS URL. + + :param url: URL to validate. + :return: Whether the URL uses HTTP or HTTPS and contains a hostname. + """ try: parsed = urlsplit(url) except ValueError: |
