From a2992533873cb53acb052cdcd4288430b24af587 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 19:47:19 +0100 Subject: fix(sender): make endpoint discovery more resilient Fall back to GET when HEAD fails so targets that reject or mishandle HEAD can still advertise a Webmention endpoint. Ignore malformed or unsupported endpoint candidates and continue searching for the first valid HTTP endpoint. --- tests/tasks/test_sender.py | 151 ++++++++++++++++++++++++++++++++++++++++ webmentions_ssg/tasks/sender.py | 39 +++++++---- 2 files changed, 177 insertions(+), 13 deletions(-) diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index d43f36f..0db0d8c 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -661,6 +661,157 @@ def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> N assert [request.method for request in requests] == ["HEAD", "GET"] +@pytest.mark.parametrize( + "failure", + [ + pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), + pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), + ], +) +def test_discovery_falls_back_to_get_after_head_request_error( + app: Flask, sender_module: ModuleType, failure: httpx.RequestError +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + + if request.method == "HEAD": + raise failure + + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["HEAD", "GET"] + + +def test_discovery_falls_back_to_get_after_head_dns_failure( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def request_hook(request: httpx.Request) -> None: + if request.method == "HEAD": + raise sender_module.TemporarySenderError("HEAD DNS lookup failed") + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), + event_hooks={"request": [request_hook]}, + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["GET"] + + +def test_discovery_get_error_after_head_error_is_not_suppressed( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + raise httpx.ConnectError(f"{request.method} failed") + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(httpx.ConnectError, match="GET failed"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "invalid_href", + [ + pytest.param("javascript:alert(1)", id="unsupported-scheme"), + pytest.param("http://[invalid", id="malformed-url"), + ], +) +def test_discovery_skips_invalid_html_candidate( + app: Flask, sender_module: ModuleType, invalid_href: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=( + f'' + 'Valid endpoint' + ), + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_skips_invalid_link_header_candidate( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers=[ + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ], + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_returns_none_when_all_html_candidates_are_invalid( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text='', + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + def test_send_webmention_persists_success( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index a4a8367..725b5c8 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -73,7 +73,10 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str: :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL. """ - endpoint = urljoin(str(response.url), href.strip()) + try: + endpoint = urljoin(str(response.url), href.strip()) + except ValueError as exc: + raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc if not is_http_url(endpoint): raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") @@ -116,8 +119,8 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: Find a Webmention endpoint in the response Link headers. :param response: HTTP response whose headers should be inspected. - :return: First advertised Webmention endpoint, or ``None`` if none is found. - :raises PermanentSenderError: If a discovered endpoint is invalid. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. """ for header in response.headers.get_list("Link"): for value in LINK_SPLIT.split(header): @@ -127,7 +130,10 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: href, relations = link if "webmention" in relations: - return resolve_endpoint(response, href) + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue return None @@ -140,8 +146,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: :param response: HTTP response from which the document was retrieved. :param body: HTML document body. - :return: First advertised Webmention endpoint, or ``None`` if none is found. - :raises PermanentSenderError: If a discovered endpoint is invalid. + :return: First valid advertised Webmention endpoint, or ``None`` if none + is found. Invalid candidates are ignored. """ document = BeautifulSoup(body, "html5lib") @@ -165,7 +171,10 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: if not isinstance(href, str): continue - return resolve_endpoint(response, href) + try: + return resolve_endpoint(response, href) + except PermanentSenderError: + continue return None @@ -211,13 +220,17 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non :param target: Target URL whose Webmention endpoint should be discovered. :return: Discovered Webmention endpoint, or ``None`` if none is advertised. :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If the target returns a permanent HTTP failure - or advertises an invalid endpoint. + :raises PermanentSenderError: If GET returns a permanent HTTP failure. """ - head_response = client.head(target, headers=DISCOVERY_HEADERS) - - if endpoint := endpoint_from_headers(head_response): - return endpoint + try: + head_response = client.head(target, headers=DISCOVERY_HEADERS) + except httpx.RequestError, TemporarySenderError: + # HEAD is only an optimization. Servers and intermediaries may reject + # or drop it even when fetching the document with GET would succeed. + pass + else: + if endpoint := endpoint_from_headers(head_response): + return endpoint with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: if endpoint := endpoint_from_headers(response): -- cgit v1.3.1