diff options
| author | Dennis Fink | 2026-11-12 19:57:08 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-12 19:57:08 +0100 |
| commit | 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch) | |
| tree | f7ed41a374a8a4136792039a262dc62aa8cf8c59 /tests/tasks/test_discovery.py | |
| parent | 7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff) | |
| download | webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip | |
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and
application extension setup so each module owns a focused concern.
Reorganize the test suite around the new boundaries while preserving
existing Webmention behavior and coverage.
Diffstat (limited to '')
| -rw-r--r-- | tests/tasks/test_discovery.py | 951 |
1 files changed, 951 insertions, 0 deletions
diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py new file mode 100644 index 0000000..2fc30b1 --- /dev/null +++ b/tests/tasks/test_discovery.py @@ -0,0 +1,951 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType +from unittest.mock import Mock + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg.tasks import discovery + +SOURCE_URL = "https://dennisfink.me/blog/example/" +TARGET_URL = "https://example.com/post" +ENDPOINT_URL = "https://example.com/webmention" +STATUS_URL = "https://example.com/webmention/status/123" + + +@pytest.fixture +def sender_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import discovery + + return discovery + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ( + '<https://example.com/webmention>; rel="webmention"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + "<https://example.com/webmention>; rel=webmention", + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/webmention>; rel="webmention alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + '<https://example.com/webmention>; REL="WebMention Alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + ( + '<https://example.com/webmention>; type="text/html"; ' + 'rel="webmention"; title="Endpoint"' + ), + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/stylesheet>; type="text/css"', + ("https://example.com/stylesheet", set()), + ), + ), +) +def test_parse_link_value( + sender_module: ModuleType, value: str, expected: tuple[str, set[str]] +) -> None: + assert sender_module.parse_link_value(value) == expected + + +@pytest.mark.parametrize( + "value", ("", "https://example.com/webmention", "<https://example.com/webmention") +) +def test_parse_link_value_rejects_malformed_value( + sender_module: ModuleType, value: str +) -> None: + assert sender_module.parse_link_value(value) is None + + +@pytest.mark.parametrize( + ("target", "headers", "html", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/1", + (("Link", "</test/1/webmention?head=true>; rel=webmention"),), + "", + "https://example.com/test/1/webmention?head=true", + id="1-http-link-unquoted-rel-relative-url", + ), + pytest.param( + "https://example.com/test/2", + ( + ( + "Link", + ( + "<https://example.com/test/2/webmention?head=true>;" + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/2/webmention?head=true", + id="2-http-link-unquoted-rel-absolute-url", + ), + pytest.param( + "https://example.com/test/3", + (), + """ + <link + rel="webmention" + href="/test/3/webmention" + > + """, + "https://example.com/test/3/webmention", + id="3-html-link-relative-url", + ), + pytest.param( + "https://example.com/test/4", + (), + """ + <link + rel="webmention" + href="https://example.com/test/4/webmention" + > + """, + "https://example.com/test/4/webmention", + id="4-html-link-absolute-url", + ), + pytest.param( + "https://example.com/test/5", + (), + """ + <a + rel="webmention" + href="/test/5/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/5/webmention", + id="5-html-a-relative-url", + ), + pytest.param( + "https://example.com/test/6", + (), + """ + <a + rel="webmention" + href="https://example.com/test/6/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/6/webmention", + id="6-html-a-absolute-url", + ), + pytest.param( + "https://example.com/test/7", + ( + ( + "LinK", + ( + "<https://example.com/test/7/webmention?head=true>; " + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/7/webmention?head=true", + id="7-http-link-strange-casing", + ), + pytest.param( + "https://example.com/test/8", + ( + ( + "Link", + ( + "<https://example.com/test/8/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/8/webmention?head=true", + id="8-http-link-quoted-rel", + ), + pytest.param( + "https://example.com/test/9", + (), + """ + <link + rel="webmention somethingelse" + href="https://example.com/test/9/webmention" + > + """, + "https://example.com/test/9/webmention", + id="9-multiple-html-rel-values", + ), + pytest.param( + "https://example.com/test/10", + ( + ( + "Link", + ( + "<https://example.com/test/10/webmention?head=true>; " + 'rel="webmention somethingelse"' + ), + ), + ), + "", + "https://example.com/test/10/webmention?head=true", + id="10-multiple-link-header-rel-values", + ), + pytest.param( + "https://example.com/test/11", + (("Link", '</test/11/webmention>; rel="webmention"'),), + """ + <link + rel="webmention" + href="/test/11/webmention/error" + > + + <a + rel="webmention" + href="/test/11/webmention/error" + > + Wrong endpoint + </a> + """, + "https://example.com/test/11/webmention", + id="11-http-link-precedence", + ), + pytest.param( + "https://example.com/test/12", + (), + """ + <link + rel="not-webmention" + href="/test/12/webmention/error" + > + + <a + rel="webmention" + href="/test/12/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/12/webmention", + id="12-exact-rel-match", + ), + pytest.param( + "https://example.com/test/13", + (), + """ + <!-- + <a + rel="webmention" + href="/test/13/webmention/error" + > + False endpoint + </a> + --> + + <a + rel="webmention" + href="/test/13/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/13/webmention", + id="13-endpoint-inside-html-comment", + ), + pytest.param( + "https://example.com/test/14", + (), + """ + <code> + <a + rel="webmention" + href="/test/14/webmention/error" + > + False endpoint + </a> + </code> + + <a + rel="webmention" + href="/test/14/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/14/webmention", + id="14-endpoint-in-escaped-html", + ), + pytest.param( + "https://example.com/test/15", + (), + """ + <link + rel="webmention" + href="" + > + """, + "https://example.com/test/15", + id="15-empty-href", + ), + pytest.param( + "https://example.com/test/16", + (), + """ + <a + rel="webmention" + href="/test/16/webmention" + > + First endpoint + </a> + + <link + rel="webmention" + href="/test/16/webmention/error" + > + """, + "https://example.com/test/16/webmention", + id="16-a-before-link", + ), + pytest.param( + "https://example.com/test/17", + (), + """ + <link + rel="webmention" + href="/test/17/webmention" + > + + <a + rel="webmention" + href="/test/17/webmention/error" + > + Second endpoint + </a> + """, + "https://example.com/test/17/webmention", + id="17-link-before-a", + ), + pytest.param( + "https://example.com/test/18", + ( + ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'), + ( + "Link", + ( + "<https://example.com/test/18/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/18/webmention?head=true", + id="18-multiple-http-link-headers", + ), + pytest.param( + "https://example.com/test/19", + ( + ( + "Link", + ( + "<https://example.com/test/19/webmention/error>; " + 'rel="other", ' + "<https://example.com/test/19/webmention?head=true>; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/19/webmention?head=true", + id="19-single-header-multiple-values", + ), + pytest.param( + "https://example.com/test/20", + (), + """ + <link rel="webmention"> + + <a + rel="webmention" + href="/test/20/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/20/webmention", + id="20-link-without-href", + ), + pytest.param( + "https://example.com/test/21", + (), + """ + <link + rel="webmention" + href="/test/21/webmention?query=yes" + > + """, + "https://example.com/test/21/webmention?query=yes", + id="21-endpoint-query-string", + ), + pytest.param( + "https://example.com/test/22", + (), + """ + <link + rel="webmention" + href="22/webmention" + > + """, + "https://example.com/test/22/webmention", + id="22-endpoint-relative-to-path", + ), + ), +) +def test_discovery( + app: Flask, + sender_module: ModuleType, + target: str, + headers: tuple[tuple[str, str], ...], + html: str, + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers=headers) + + return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +@pytest.mark.parametrize( + ("target", "responses", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/23/page", + { + "/test/23/page": (302, {"Location": "page/redirect-key"}), + "/test/23/page/redirect-key": ( + 200, + {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")}, + ), + }, + ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), + id="relative-endpoint-after-redirect", + ), + ), +) +def test_discovery_redirect( + app: Flask, + sender_module: ModuleType, + target: str, + responses: dict[str, tuple[int, dict[str, str]]], + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + try: + status, headers = responses[request.url.path] + except KeyError: + raise AssertionError(f"Unexpected request: {request.url}") from None + + return httpx.Response(status, headers=headers) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD"] + + +def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=""" + <link + rel="webmention" + href="/webmention" + > + """, + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "failure", + [ + pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), + pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), + ], +) +def test_discovery_falls_back_to_get_after_head_request_error( + app: Flask, sender_module: ModuleType, failure: httpx.RequestError +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + + if request.method == "HEAD": + raise failure + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["HEAD", "GET"] + + +def test_discovery_falls_back_to_get_after_head_dns_failure( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def request_hook(request: httpx.Request) -> None: + if request.method == "HEAD": + raise sender_module.TemporarySenderError("HEAD DNS lookup failed") + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), + event_hooks={"request": [request_hook]}, + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["GET"] + + +def test_discovery_get_error_after_head_error_is_not_suppressed( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + raise httpx.ConnectError(f"{request.method} failed") + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(httpx.ConnectError, match="GET failed"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "invalid_href", + [ + pytest.param("javascript:alert(1)", id="unsupported-scheme"), + pytest.param("http://[invalid", id="malformed-url"), + ], +) +def test_discovery_skips_invalid_html_candidate( + app: Flask, sender_module: ModuleType, invalid_href: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=( + f'<link rel="webmention" href="{invalid_href}">' + '<a rel="webmention" href="/webmention">Valid endpoint</a>' + ), + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_skips_invalid_link_header_candidate( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers=[ + ("Link", "<mailto:invalid@example.com>; rel=webmention"), + ("Link", "<http://[invalid>; rel=webmention"), + ("Link", "</webmention>; rel=webmention"), + ], + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_returns_none_when_all_html_candidates_are_invalid( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"}) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text='<link rel="webmention" href="javascript:void(0)">', + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +def test_resolve_endpoint_rejects_invalid_url( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_http_url = Mock(return_value=False) + monkeypatch.setattr(discovery, "is_http_url", is_http_url) + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + with pytest.raises( + sender_module.PermanentSenderError, match="Invalid Webmention endpoint" + ): + sender_module.resolve_endpoint(response, "/webmention") + is_http_url.assert_called_once_with(ENDPOINT_URL) + + +def test_endpoint_from_headers_ignores_malformed_link( + sender_module: ModuleType, +) -> None: + response = httpx.Response( + 200, + headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")], + request=httpx.Request("GET", TARGET_URL), + ) + assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL + + +def test_endpoint_from_html_accepts_string_rel( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + element = Mock() + element.get.side_effect = lambda name: { + "rel": "webmention alternate", + "href": "/webmention", + }[name] + + document = Mock() + document.find_all.return_value = [element] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL + + +def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + missing_rel = Mock() + missing_rel.get.return_value = None + + invalid_href = Mock() + invalid_href.get.side_effect = lambda name: { + "rel": ["webmention"], + "href": ["invalid"], + }[name] + + document = Mock() + document.find_all.return_value = [missing_rel, invalid_href] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") is None + + +def test_read_target_body_rejects_declared_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_read_target_body_ignores_invalid_content_length( + app: Flask, sender_module: ModuleType +) -> None: + response = httpx.Response( + 200, headers={"Content-Length": "invalid"}, content=b"Document" + ) + + with app.app_context(): + body = sender_module.read_target_body(response) + + assert body == b"Document" + + +def test_read_target_body_rejects_streamed_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + response = httpx.Response(200, stream=BodyStream()) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_discovery_uses_link_header_from_get( + app: Flask, sender_module: ModuleType +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + pytest.param(503, "TemporarySenderError", id="temporary"), + pytest.param(404, "PermanentSenderError", id="permanent"), + ], +) +def test_discovery_maps_get_status_to_exception( + app: Flask, sender_module: ModuleType, status_code: int, exception_name: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(status_code) + + exception_type = getattr(sender_module, exception_name) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + +def test_discovery_returns_none_for_unsupported_media_type( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, headers={"Content-Type": "application/json"}, content=b"{}" + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +@pytest.mark.parametrize( + ("status_code", "location", "expected_status_url"), + [ + pytest.param( + 201, "/webmention/status/123", STATUS_URL, id="created-with-location" + ), + pytest.param(202, None, None, id="accepted"), + ], +) +def test_post_webmention( + sender_module: ModuleType, + status_code: int, + location: str | None, + expected_status_url: str | None, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + headers = {} + + if location is not None: + headers["Location"] = location + + return httpx.Response(status_code, headers=headers) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (status_code, expected_status_url) + assert len(requests) == 1 + + request = requests[0] + + assert request.method == "POST" + assert str(request.url) == ENDPOINT_URL + + form = httpx.QueryParams(request.content.decode()) + + assert form["source"] == SOURCE_URL + assert form["target"] == TARGET_URL + + +@pytest.mark.parametrize( + ("code", "location", "expected"), + [ + (201, "/webmention/status/123", "https://example.com/webmention/status/123"), + (201, "https://status.example.org/123", "https://status.example.org/123"), + (201, "status/123", "https://example.com/status/123"), + (201, "javascript:alert(1)", None), + (201, "data:text/html,evil", None), + (201, "ftp://example.com/status", None), + (201, "http://[invalid", None), + (202, "https://example.com/status", None), + ], +) +def test_post_webmention_validates_location( + sender_module: ModuleType, code: int, location: str, expected: str | None +) -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(code, headers={"Location": location}) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (code, expected) |
