From 9b00526ab79257b65cedd71595848f09ecf84279 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Sun, 15 Nov 2026 21:18:19 +0100 Subject: refactor(core)!: clarify Webmention domain terminology Rename ambiguous models, statuses, task modules, fields, and helpers so names reflect source documents, verification, endpoint discovery, and delivery state. Migrate persisted status values and database identifiers without losing existing Webmention data. BREAKING CHANGE: persisted status values and database schema names have changed, and several public Python names have been renamed. --- tests/tasks/test_endpoint_discovery.py | 977 +++++++++++++++++++++++++++++++++ 1 file changed, 977 insertions(+) create mode 100644 tests/tasks/test_endpoint_discovery.py (limited to 'tests/tasks/test_endpoint_discovery.py') diff --git a/tests/tasks/test_endpoint_discovery.py b/tests/tasks/test_endpoint_discovery.py new file mode 100644 index 0000000..98c167b --- /dev/null +++ b/tests/tasks/test_endpoint_discovery.py @@ -0,0 +1,977 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from http import HTTPStatus +from types import ModuleType +from unittest.mock import Mock + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg.tasks import endpoint_discovery + +SOURCE_URL = "https://dennisfink.me/blog/example/" +TARGET_URL = "https://example.com/post" +ENDPOINT_URL = "https://example.com/webmention" +STATUS_URL = "https://example.com/webmention/status/123" + + +@pytest.fixture +def sender_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import endpoint_discovery + + return endpoint_discovery + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ( + '; rel="webmention"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + "; rel=webmention", + ("https://example.com/webmention", {"webmention"}), + ), + ( + '; rel="webmention alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + '; REL="WebMention Alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + ( + '; type="text/html"; ' + 'rel="webmention"; title="Endpoint"' + ), + ("https://example.com/webmention", {"webmention"}), + ), + ( + '; type="text/css"', + ("https://example.com/stylesheet", set()), + ), + ), +) +def test_parse_link_header_value( + sender_module: ModuleType, value: str, expected: tuple[str, set[str]] +) -> None: + assert sender_module.parse_link_header_value(value) == expected + + +@pytest.mark.parametrize( + "value", ("", "https://example.com/webmention", " None: + assert sender_module.parse_link_header_value(value) is None + + +@pytest.mark.parametrize( + ("target", "headers", "html", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/1", + (("Link", "; rel=webmention"),), + "", + "https://example.com/test/1/webmention?head=true", + id="1-http-link-unquoted-rel-relative-url", + ), + pytest.param( + "https://example.com/test/2", + ( + ( + "Link", + ( + ";" + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/2/webmention?head=true", + id="2-http-link-unquoted-rel-absolute-url", + ), + pytest.param( + "https://example.com/test/3", + (), + """ + + """, + "https://example.com/test/3/webmention", + id="3-html-link-relative-url", + ), + pytest.param( + "https://example.com/test/4", + (), + """ + + """, + "https://example.com/test/4/webmention", + id="4-html-link-absolute-url", + ), + pytest.param( + "https://example.com/test/5", + (), + """ + + Endpoint + + """, + "https://example.com/test/5/webmention", + id="5-html-a-relative-url", + ), + pytest.param( + "https://example.com/test/6", + (), + """ + + Endpoint + + """, + "https://example.com/test/6/webmention", + id="6-html-a-absolute-url", + ), + pytest.param( + "https://example.com/test/7", + ( + ( + "LinK", + ( + "; " + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/7/webmention?head=true", + id="7-http-link-strange-casing", + ), + pytest.param( + "https://example.com/test/8", + ( + ( + "Link", + ( + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/8/webmention?head=true", + id="8-http-link-quoted-rel", + ), + pytest.param( + "https://example.com/test/9", + (), + """ + + """, + "https://example.com/test/9/webmention", + id="9-multiple-html-rel-values", + ), + pytest.param( + "https://example.com/test/10", + ( + ( + "Link", + ( + "; " + 'rel="webmention somethingelse"' + ), + ), + ), + "", + "https://example.com/test/10/webmention?head=true", + id="10-multiple-link-header-rel-values", + ), + pytest.param( + "https://example.com/test/11", + (("Link", '; rel="webmention"'),), + """ + + + + Wrong endpoint + + """, + "https://example.com/test/11/webmention", + id="11-http-link-precedence", + ), + pytest.param( + "https://example.com/test/12", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/12/webmention", + id="12-exact-rel-match", + ), + pytest.param( + "https://example.com/test/13", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/13/webmention", + id="13-endpoint-inside-html-comment", + ), + pytest.param( + "https://example.com/test/14", + (), + """ + + <a + rel="webmention" + href="/test/14/webmention/error" + > + False endpoint + </a> + + + + Correct endpoint + + """, + "https://example.com/test/14/webmention", + id="14-endpoint-in-escaped-html", + ), + pytest.param( + "https://example.com/test/15", + (), + """ + + """, + "https://example.com/test/15", + id="15-empty-href", + ), + pytest.param( + "https://example.com/test/16", + (), + """ + + First endpoint + + + + """, + "https://example.com/test/16/webmention", + id="16-a-before-link", + ), + pytest.param( + "https://example.com/test/17", + (), + """ + + + + Second endpoint + + """, + "https://example.com/test/17/webmention", + id="17-link-before-a", + ), + pytest.param( + "https://example.com/test/18", + ( + ("Link", '; rel="other"'), + ( + "Link", + ( + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/18/webmention?head=true", + id="18-multiple-http-link-headers", + ), + pytest.param( + "https://example.com/test/19", + ( + ( + "Link", + ( + "; " + 'rel="other", ' + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/19/webmention?head=true", + id="19-single-header-multiple-values", + ), + pytest.param( + "https://example.com/test/20", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/20/webmention", + id="20-link-without-href", + ), + pytest.param( + "https://example.com/test/21", + (), + """ + + """, + "https://example.com/test/21/webmention?query=yes", + id="21-endpoint-query-string", + ), + pytest.param( + "https://example.com/test/22", + (), + """ + + """, + "https://example.com/test/22/webmention", + id="22-endpoint-relative-to-path", + ), + ), +) +def test_discovery( + app: Flask, + sender_module: ModuleType, + target: str, + headers: tuple[tuple[str, str], ...], + html: str, + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK, headers=headers) + + return httpx.Response( + HTTPStatus.OK, headers={"Content-Type": "text/html"}, text=html + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +@pytest.mark.parametrize( + ("target", "responses", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/23/page", + { + "/test/23/page": (HTTPStatus.FOUND, {"Location": "page/redirect-key"}), + "/test/23/page/redirect-key": ( + HTTPStatus.OK, + {"Link": ("; rel=webmention")}, + ), + }, + ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), + id="relative-endpoint-after-redirect", + ), + ), +) +def test_discovery_redirect( + app: Flask, + sender_module: ModuleType, + target: str, + responses: dict[str, tuple[int, dict[str, str]]], + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + try: + status, headers = responses[request.url.path] + except KeyError: + raise AssertionError(f"Unexpected request: {request.url}") from None + + return httpx.Response(status, headers=headers) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + return httpx.Response( + HTTPStatus.OK, headers={"Link": "; rel=webmention"} + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD"] + + +def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK) + + return httpx.Response( + HTTPStatus.OK, + headers={"Content-Type": "text/html"}, + text=""" + + """, + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "failure", + [ + pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), + pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), + ], +) +def test_discovery_falls_back_to_get_after_head_request_error( + app: Flask, sender_module: ModuleType, failure: httpx.RequestError +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + + if request.method == "HEAD": + raise failure + + return httpx.Response( + HTTPStatus.OK, headers={"Link": "; rel=webmention"} + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["HEAD", "GET"] + + +def test_discovery_does_not_suppress_unexpected_head_error( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def request_hook(request: httpx.Request) -> None: + if request.method == "HEAD": + raise sender_module.RetryableDeliveryError("HEAD DNS lookup failed") + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + return httpx.Response( + HTTPStatus.OK, headers={"Link": "; rel=webmention"} + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), + event_hooks={"request": [request_hook]}, + ) as client, + pytest.raises( + sender_module.RetryableDeliveryError, match="HEAD DNS lookup failed" + ), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == [] + + +def test_discovery_get_error_after_head_error_is_not_suppressed( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + raise httpx.ConnectError(f"{request.method} failed") + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(httpx.ConnectError, match="GET failed"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "invalid_href", + [ + pytest.param("javascript:alert(1)", id="unsupported-scheme"), + pytest.param("http://[invalid", id="malformed-url"), + ], +) +def test_discovery_skips_invalid_html_candidate( + app: Flask, sender_module: ModuleType, invalid_href: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK) + + return httpx.Response( + HTTPStatus.OK, + headers={"Content-Type": "text/html"}, + text=( + f'' + 'Valid endpoint' + ), + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_skips_invalid_link_header_candidate( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + HTTPStatus.OK, + headers=[ + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ], + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_returns_none_when_all_html_candidates_are_invalid( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response( + HTTPStatus.OK, headers={"Link": "; rel=webmention"} + ) + + return httpx.Response( + HTTPStatus.OK, + headers={"Content-Type": "text/html"}, + text='', + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +def test_resolve_endpoint_url_rejects_invalid_url( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_http_url = Mock(return_value=False) + monkeypatch.setattr(endpoint_discovery, "is_http_url", is_http_url) + response = httpx.Response(HTTPStatus.OK, request=httpx.Request("GET", TARGET_URL)) + with pytest.raises( + sender_module.PermanentDeliveryError, match="Invalid Webmention endpoint" + ): + sender_module.resolve_endpoint_url(response, "/webmention") + is_http_url.assert_called_once_with(ENDPOINT_URL) + + +def test_find_endpoint_in_headers_ignores_malformed_link( + sender_module: ModuleType, +) -> None: + response = httpx.Response( + HTTPStatus.OK, + headers=[("Link", "not-a-link"), ("Link", "; rel=webmention")], + request=httpx.Request("GET", TARGET_URL), + ) + assert sender_module.find_endpoint_in_headers(response) == ENDPOINT_URL + + +def test_find_endpoint_in_html_accepts_string_rel(sender_module: ModuleType) -> None: + response = httpx.Response(HTTPStatus.OK, request=httpx.Request("GET", TARGET_URL)) + body = b'' + + assert sender_module.find_endpoint_in_html(response, body) == ENDPOINT_URL + + +def test_find_endpoint_in_html_accepts_uppercase_rel(sender_module: ModuleType) -> None: + response = httpx.Response(HTTPStatus.OK, request=httpx.Request("GET", TARGET_URL)) + body = b'' + + assert sender_module.find_endpoint_in_html(response, body) == ENDPOINT_URL + + +def test_find_endpoint_in_html_ignores_elements_without_matching_rel_or_href( + sender_module: ModuleType, +) -> None: + response = httpx.Response(HTTPStatus.OK, request=httpx.Request("GET", TARGET_URL)) + body = b'No endpoint' + + assert sender_module.find_endpoint_in_html(response, body) is None + + +def test_read_target_body_rejects_declared_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_DISCOVERY_TARGET_BYTES"] = 10 + + response = httpx.Response( + HTTPStatus.OK, headers={"Content-Length": "11"}, content=b"x" * 11 + ) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentDeliveryError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_read_target_body_does_not_mask_unexpected_value_error( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + sender_module, + "read_limited_body", + Mock(side_effect=ValueError("unrelated processing error")), + ) + + with ( + app.app_context(), + pytest.raises(ValueError, match="unrelated processing error"), + ): + sender_module.read_target_body(httpx.Response(HTTPStatus.OK)) + + +def test_read_target_body_ignores_invalid_content_length( + app: Flask, sender_module: ModuleType +) -> None: + response = httpx.Response( + HTTPStatus.OK, headers={"Content-Length": "invalid"}, content=b"Document" + ) + + with app.app_context(): + body = sender_module.read_target_body(response) + + assert body == b"Document" + + +def test_read_target_body_rejects_streamed_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_DISCOVERY_TARGET_BYTES"] = 10 + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + response = httpx.Response(HTTPStatus.OK, stream=BodyStream()) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentDeliveryError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_discovery_uses_link_header_from_get( + app: Flask, sender_module: ModuleType +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK) + + return httpx.Response( + HTTPStatus.OK, headers={"Link": "; rel=webmention"} + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + pytest.param( + HTTPStatus.SERVICE_UNAVAILABLE, "RetryableDeliveryError", id="temporary" + ), + pytest.param(HTTPStatus.NOT_FOUND, "PermanentDeliveryError", id="permanent"), + ], +) +def test_discovery_maps_get_status_to_exception( + app: Flask, sender_module: ModuleType, status_code: int, exception_name: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK) + + return httpx.Response(status_code) + + exception_type = getattr(sender_module, exception_name) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + +def test_discovery_returns_none_for_unsupported_media_type( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(HTTPStatus.OK) + + return httpx.Response( + HTTPStatus.OK, headers={"Content-Type": "application/json"}, content=b"{}" + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +@pytest.mark.parametrize( + ("status_code", "location", "expected_status_url"), + [ + pytest.param( + HTTPStatus.CREATED, + "/webmention/status/123", + STATUS_URL, + id="created-with-location", + ), + pytest.param(HTTPStatus.ACCEPTED, None, None, id="accepted"), + ], +) +def test_post_webmention( + sender_module: ModuleType, + status_code: int, + location: str | None, + expected_status_url: str | None, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + headers = {} + + if location is not None: + headers["Location"] = location + + return httpx.Response(status_code, headers=headers) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (status_code, expected_status_url) + assert len(requests) == 1 + + request = requests[0] + + assert request.method == "POST" + assert str(request.url) == ENDPOINT_URL + + form = httpx.QueryParams(request.content.decode()) + + assert form["source"] == SOURCE_URL + assert form["target"] == TARGET_URL + + +@pytest.mark.parametrize( + ("code", "location", "expected"), + [ + ( + HTTPStatus.CREATED, + "/webmention/status/123", + "https://example.com/webmention/status/123", + ), + ( + HTTPStatus.CREATED, + "https://status.example.org/123", + "https://status.example.org/123", + ), + (HTTPStatus.CREATED, "status/123", "https://example.com/status/123"), + (HTTPStatus.CREATED, "javascript:alert(1)", None), + (HTTPStatus.CREATED, "data:text/html,evil", None), + (HTTPStatus.CREATED, "ftp://example.com/status", None), + (HTTPStatus.CREATED, "http://[invalid", None), + (HTTPStatus.ACCEPTED, "https://example.com/status", None), + ], +) +def test_post_webmention_validates_location( + sender_module: ModuleType, code: int, location: str, expected: str | None +) -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(code, headers={"Location": location}) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (code, expected) -- cgit v1.3.1