From 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Thu, 12 Nov 2026 19:57:08 +0100 Subject: refactor(core): split modules by responsibility Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage. --- tests/tasks/scanner_helpers.py | 72 +++ tests/tasks/test_discovery.py | 951 ++++++++++++++++++++++++++++++++ tests/tasks/test_json_serializer.py | 2 +- tests/tasks/test_mentions.py | 176 ++++++ tests/tasks/test_receiver.py | 160 +----- tests/tasks/test_scanner.py | 1035 +---------------------------------- tests/tasks/test_sender.py | 946 -------------------------------- tests/tasks/test_source_parser.py | 1008 ++++++++++++++++++++++++++++++++++ 8 files changed, 2217 insertions(+), 2133 deletions(-) create mode 100644 tests/tasks/scanner_helpers.py create mode 100644 tests/tasks/test_discovery.py create mode 100644 tests/tasks/test_mentions.py create mode 100644 tests/tasks/test_source_parser.py (limited to 'tests/tasks') diff --git a/tests/tasks/scanner_helpers.py b/tests/tasks/scanner_helpers.py new file mode 100644 index 0000000..4100028 --- /dev/null +++ b/tests/tasks/scanner_helpers.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path + +from bs4 import BeautifulSoup +from flask import Flask + +BASE_URL = "https://dennisfink.me/blog/" +SOURCE_URL = f"{BASE_URL}example/" + + +def parse_html(html: str) -> BeautifulSoup: + return BeautifulSoup(html, "html5lib") + + +def write_post( + root: Path, + slug: str, + content: str, + *, + canonical: str | None = None, + entry_url: str | None = None, +) -> Path: + directory = root / slug + directory.mkdir(parents=True, exist_ok=True) + + source_url = f"{BASE_URL}{slug}/" + + if canonical is None: + canonical = source_url + + if entry_url is None: + entry_url = source_url + + path = directory / "index.html" + path.write_text( + f""" + + + + + + +
+ + Permalink + + +
+ {content} +
+
+ + + """, + encoding="utf-8", + ) + + return path + + +def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None: + app.config["SOURCE_DIRECTORY"] = str(root) + app.config["SOURCE_BASE_URL"] = base_url diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py new file mode 100644 index 0000000..2fc30b1 --- /dev/null +++ b/tests/tasks/test_discovery.py @@ -0,0 +1,951 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType +from unittest.mock import Mock + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg.tasks import discovery + +SOURCE_URL = "https://dennisfink.me/blog/example/" +TARGET_URL = "https://example.com/post" +ENDPOINT_URL = "https://example.com/webmention" +STATUS_URL = "https://example.com/webmention/status/123" + + +@pytest.fixture +def sender_module(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import discovery + + return discovery + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ( + '; rel="webmention"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + "; rel=webmention", + ("https://example.com/webmention", {"webmention"}), + ), + ( + '; rel="webmention alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + '; REL="WebMention Alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + ( + '; type="text/html"; ' + 'rel="webmention"; title="Endpoint"' + ), + ("https://example.com/webmention", {"webmention"}), + ), + ( + '; type="text/css"', + ("https://example.com/stylesheet", set()), + ), + ), +) +def test_parse_link_value( + sender_module: ModuleType, value: str, expected: tuple[str, set[str]] +) -> None: + assert sender_module.parse_link_value(value) == expected + + +@pytest.mark.parametrize( + "value", ("", "https://example.com/webmention", " None: + assert sender_module.parse_link_value(value) is None + + +@pytest.mark.parametrize( + ("target", "headers", "html", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/1", + (("Link", "; rel=webmention"),), + "", + "https://example.com/test/1/webmention?head=true", + id="1-http-link-unquoted-rel-relative-url", + ), + pytest.param( + "https://example.com/test/2", + ( + ( + "Link", + ( + ";" + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/2/webmention?head=true", + id="2-http-link-unquoted-rel-absolute-url", + ), + pytest.param( + "https://example.com/test/3", + (), + """ + + """, + "https://example.com/test/3/webmention", + id="3-html-link-relative-url", + ), + pytest.param( + "https://example.com/test/4", + (), + """ + + """, + "https://example.com/test/4/webmention", + id="4-html-link-absolute-url", + ), + pytest.param( + "https://example.com/test/5", + (), + """ + + Endpoint + + """, + "https://example.com/test/5/webmention", + id="5-html-a-relative-url", + ), + pytest.param( + "https://example.com/test/6", + (), + """ + + Endpoint + + """, + "https://example.com/test/6/webmention", + id="6-html-a-absolute-url", + ), + pytest.param( + "https://example.com/test/7", + ( + ( + "LinK", + ( + "; " + "rel=webmention" + ), + ), + ), + "", + "https://example.com/test/7/webmention?head=true", + id="7-http-link-strange-casing", + ), + pytest.param( + "https://example.com/test/8", + ( + ( + "Link", + ( + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/8/webmention?head=true", + id="8-http-link-quoted-rel", + ), + pytest.param( + "https://example.com/test/9", + (), + """ + + """, + "https://example.com/test/9/webmention", + id="9-multiple-html-rel-values", + ), + pytest.param( + "https://example.com/test/10", + ( + ( + "Link", + ( + "; " + 'rel="webmention somethingelse"' + ), + ), + ), + "", + "https://example.com/test/10/webmention?head=true", + id="10-multiple-link-header-rel-values", + ), + pytest.param( + "https://example.com/test/11", + (("Link", '; rel="webmention"'),), + """ + + + + Wrong endpoint + + """, + "https://example.com/test/11/webmention", + id="11-http-link-precedence", + ), + pytest.param( + "https://example.com/test/12", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/12/webmention", + id="12-exact-rel-match", + ), + pytest.param( + "https://example.com/test/13", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/13/webmention", + id="13-endpoint-inside-html-comment", + ), + pytest.param( + "https://example.com/test/14", + (), + """ + + <a + rel="webmention" + href="/test/14/webmention/error" + > + False endpoint + </a> + + + + Correct endpoint + + """, + "https://example.com/test/14/webmention", + id="14-endpoint-in-escaped-html", + ), + pytest.param( + "https://example.com/test/15", + (), + """ + + """, + "https://example.com/test/15", + id="15-empty-href", + ), + pytest.param( + "https://example.com/test/16", + (), + """ + + First endpoint + + + + """, + "https://example.com/test/16/webmention", + id="16-a-before-link", + ), + pytest.param( + "https://example.com/test/17", + (), + """ + + + + Second endpoint + + """, + "https://example.com/test/17/webmention", + id="17-link-before-a", + ), + pytest.param( + "https://example.com/test/18", + ( + ("Link", '; rel="other"'), + ( + "Link", + ( + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/18/webmention?head=true", + id="18-multiple-http-link-headers", + ), + pytest.param( + "https://example.com/test/19", + ( + ( + "Link", + ( + "; " + 'rel="other", ' + "; " + 'rel="webmention"' + ), + ), + ), + "", + "https://example.com/test/19/webmention?head=true", + id="19-single-header-multiple-values", + ), + pytest.param( + "https://example.com/test/20", + (), + """ + + + + Correct endpoint + + """, + "https://example.com/test/20/webmention", + id="20-link-without-href", + ), + pytest.param( + "https://example.com/test/21", + (), + """ + + """, + "https://example.com/test/21/webmention?query=yes", + id="21-endpoint-query-string", + ), + pytest.param( + "https://example.com/test/22", + (), + """ + + """, + "https://example.com/test/22/webmention", + id="22-endpoint-relative-to-path", + ), + ), +) +def test_discovery( + app: Flask, + sender_module: ModuleType, + target: str, + headers: tuple[tuple[str, str], ...], + html: str, + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers=headers) + + return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +@pytest.mark.parametrize( + ("target", "responses", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/23/page", + { + "/test/23/page": (302, {"Location": "page/redirect-key"}), + "/test/23/page/redirect-key": ( + 200, + {"Link": ("; rel=webmention")}, + ), + }, + ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), + id="relative-endpoint-after-redirect", + ), + ), +) +def test_discovery_redirect( + app: Flask, + sender_module: ModuleType, + target: str, + responses: dict[str, tuple[int, dict[str, str]]], + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + try: + status, headers = responses[request.url.path] + except KeyError: + raise AssertionError(f"Unexpected request: {request.url}") from None + + return httpx.Response(status, headers=headers) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD"] + + +def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=""" + + """, + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "failure", + [ + pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), + pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), + ], +) +def test_discovery_falls_back_to_get_after_head_request_error( + app: Flask, sender_module: ModuleType, failure: httpx.RequestError +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + + if request.method == "HEAD": + raise failure + + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["HEAD", "GET"] + + +def test_discovery_falls_back_to_get_after_head_dns_failure( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def request_hook(request: httpx.Request) -> None: + if request.method == "HEAD": + raise sender_module.TemporarySenderError("HEAD DNS lookup failed") + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), + event_hooks={"request": [request_hook]}, + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert methods == ["GET"] + + +def test_discovery_get_error_after_head_error_is_not_suppressed( + app: Flask, sender_module: ModuleType +) -> None: + methods: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + methods.append(request.method) + raise httpx.ConnectError(f"{request.method} failed") + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(httpx.ConnectError, match="GET failed"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert methods == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + "invalid_href", + [ + pytest.param("javascript:alert(1)", id="unsupported-scheme"), + pytest.param("http://[invalid", id="malformed-url"), + ], +) +def test_discovery_skips_invalid_html_candidate( + app: Flask, sender_module: ModuleType, invalid_href: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=( + f'' + 'Valid endpoint' + ), + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_skips_invalid_link_header_candidate( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers=[ + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ("Link", "; rel=webmention"), + ], + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + + +def test_discovery_returns_none_when_all_html_candidates_are_invalid( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text='', + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +def test_resolve_endpoint_rejects_invalid_url( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + is_http_url = Mock(return_value=False) + monkeypatch.setattr(discovery, "is_http_url", is_http_url) + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + with pytest.raises( + sender_module.PermanentSenderError, match="Invalid Webmention endpoint" + ): + sender_module.resolve_endpoint(response, "/webmention") + is_http_url.assert_called_once_with(ENDPOINT_URL) + + +def test_endpoint_from_headers_ignores_malformed_link( + sender_module: ModuleType, +) -> None: + response = httpx.Response( + 200, + headers=[("Link", "not-a-link"), ("Link", "; rel=webmention")], + request=httpx.Request("GET", TARGET_URL), + ) + assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL + + +def test_endpoint_from_html_accepts_string_rel( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + element = Mock() + element.get.side_effect = lambda name: { + "rel": "webmention alternate", + "href": "/webmention", + }[name] + + document = Mock() + document.find_all.return_value = [element] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL + + +def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( + sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + missing_rel = Mock() + missing_rel.get.return_value = None + + invalid_href = Mock() + invalid_href.get.side_effect = lambda name: { + "rel": ["webmention"], + "href": ["invalid"], + }[name] + + document = Mock() + document.find_all.return_value = [missing_rel, invalid_href] + + beautiful_soup = Mock(return_value=document) + monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup) + + response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) + + assert sender_module.endpoint_from_html(response, b"") is None + + +def test_read_target_body_rejects_declared_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_read_target_body_ignores_invalid_content_length( + app: Flask, sender_module: ModuleType +) -> None: + response = httpx.Response( + 200, headers={"Content-Length": "invalid"}, content=b"Document" + ) + + with app.app_context(): + body = sender_module.read_target_body(response) + + assert body == b"Document" + + +def test_read_target_body_rejects_streamed_oversized_body( + app: Flask, sender_module: ModuleType +) -> None: + app.config["MAX_TARGET_BYTES"] = 10 + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + response = httpx.Response(200, stream=BodyStream()) + + with ( + app.app_context(), + pytest.raises( + sender_module.PermanentSenderError, match="Target document is too large" + ), + ): + sender_module.read_target_body(response) + + +def test_discovery_uses_link_header_from_get( + app: Flask, sender_module: ModuleType +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(200, headers={"Link": "; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint == ENDPOINT_URL + assert [request.method for request in requests] == ["HEAD", "GET"] + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + pytest.param(503, "TemporarySenderError", id="temporary"), + pytest.param(404, "PermanentSenderError", id="permanent"), + ], +) +def test_discovery_maps_get_status_to_exception( + app: Flask, sender_module: ModuleType, status_code: int, exception_name: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response(status_code) + + exception_type = getattr(sender_module, exception_name) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), + ): + sender_module.discover_webmention_endpoint(client, TARGET_URL) + + +def test_discovery_returns_none_for_unsupported_media_type( + app: Flask, sender_module: ModuleType +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, headers={"Content-Type": "application/json"}, content=b"{}" + ) + + with ( + app.app_context(), + httpx.Client(transport=httpx.MockTransport(handler)) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) + + assert endpoint is None + + +@pytest.mark.parametrize( + ("status_code", "location", "expected_status_url"), + [ + pytest.param( + 201, "/webmention/status/123", STATUS_URL, id="created-with-location" + ), + pytest.param(202, None, None, id="accepted"), + ], +) +def test_post_webmention( + sender_module: ModuleType, + status_code: int, + location: str | None, + expected_status_url: str | None, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + headers = {} + + if location is not None: + headers["Location"] = location + + return httpx.Response(status_code, headers=headers) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (status_code, expected_status_url) + assert len(requests) == 1 + + request = requests[0] + + assert request.method == "POST" + assert str(request.url) == ENDPOINT_URL + + form = httpx.QueryParams(request.content.decode()) + + assert form["source"] == SOURCE_URL + assert form["target"] == TARGET_URL + + +@pytest.mark.parametrize( + ("code", "location", "expected"), + [ + (201, "/webmention/status/123", "https://example.com/webmention/status/123"), + (201, "https://status.example.org/123", "https://status.example.org/123"), + (201, "status/123", "https://example.com/status/123"), + (201, "javascript:alert(1)", None), + (201, "data:text/html,evil", None), + (201, "ftp://example.com/status", None), + (201, "http://[invalid", None), + (202, "https://example.com/status", None), + ], +) +def test_post_webmention_validates_location( + sender_module: ModuleType, code: int, location: str, expected: str | None +) -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(code, headers={"Location": location}) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + result = sender_module.post_webmention( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + assert result == (code, expected) diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py index 108e8a6..c733dce 100644 --- a/tests/tasks/test_json_serializer.py +++ b/tests/tasks/test_json_serializer.py @@ -10,7 +10,7 @@ from datetime import UTC, datetime, timedelta import pytest from flask import Flask -from huey.utils import Error, SKIPPED +from huey.utils import SKIPPED, Error from webmentions_ssg.tasks.extension import Huey from webmentions_ssg.tasks.json_serializer import JSONSerializer diff --git a/tests/tasks/test_mentions.py b/tests/tasks/test_mentions.py new file mode 100644 index 0000000..a5da56c --- /dev/null +++ b/tests/tasks/test_mentions.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from types import ModuleType +from unittest.mock import Mock, patch + +import pytest +from flask import Flask + +SOURCE_URL = "https://source.example/article" +TARGET_URL = "https://dennisfink.me/blog/example/" + + +@pytest.fixture +def receiver(app: Flask) -> ModuleType: + _ = app + from webmentions_ssg.tasks import mentions + + return mentions + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + pytest.param(f'Reply', True, id="a-href"), + pytest.param(f'Target', True, id="area-href"), + pytest.param( + f'', True, id="link-href" + ), + pytest.param(f'', True, id="img-src"), + pytest.param(f'', True, id="audio-src"), + pytest.param(f'', True, id="video-src"), + pytest.param( + f'', True, id="audio-source-src" + ), + pytest.param( + f'', True, id="video-source-src" + ), + pytest.param(f'', True, id="iframe-src"), + pytest.param(f'', True, id="embed-src"), + pytest.param(f'', True, id="script-src"), + pytest.param( + f'', True, id="track-src" + ), + pytest.param( + f'', + True, + id="image-input-src", + ), + pytest.param( + f'', + True, + id="image-input-case-insensitive", + ), + pytest.param( + f'
Quotation
', + True, + id="blockquote-cite", + ), + pytest.param(f'Quotation', True, id="q-cite"), + pytest.param(f'Addition', True, id="ins-cite"), + pytest.param(f'Removal', True, id="del-cite"), + pytest.param( + 'Reply', + True, + id="base-url", + ), + pytest.param(f'', False, id="img-cite-invalid"), + pytest.param( + f'
Quote
', + False, + id="blockquote-src-invalid", + ), + pytest.param(f'Reply', False, id="a-src-invalid"), + pytest.param(f'
', False, id="div-href-invalid"), + pytest.param(f'', False, id="link-src-invalid"), + pytest.param( + f'', + False, + id="text-input-src-invalid", + ), + pytest.param( + f'', + False, + id="picture-source-src-invalid", + ), + pytest.param(f'', False, id="base-is-not-mention"), + pytest.param( + f'Different page', False, id="longer-url" + ), + pytest.param( + f'', + False, + id="invalid-cite-does-not-override-valid-src", + ), + pytest.param(f"

{TARGET_URL}

", False, id="text-content"), + pytest.param( + 'Other site', False, id="missing-target" + ), + ], +) +def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None: + assert ( + receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected + ) + + +def test_html_mentions_relative_target(receiver: ModuleType) -> None: + assert receiver.html_mentions_target( + b'Reply', + "https://source.example/posts/article/", + "https://source.example/posts/target/", + ) + + +@pytest.mark.parametrize( + ("body", "target_url", "expected"), + [ + (TARGET_URL, TARGET_URL, True), + (f"This post replies to {TARGET_URL}", TARGET_URL, True), + ( + f"https://example.com/first {TARGET_URL} https://example.com/last", + TARGET_URL, + True, + ), + pytest.param( + f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma" + ), + pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"), + pytest.param( + f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation" + ), + pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"), + pytest.param( + "https://example.com/wiki/Foo_(bar)", + "https://example.com/wiki/Foo_(bar)", + True, + id="balanced-parentheses-in-url", + ), + pytest.param( + "https://example.com/wiki/Foo_(bar)", + "https://example.com/wiki/Foo_(bar", + False, + id="balanced-parentheses-not-trimmed", + ), + (f"{TARGET_URL}more", TARGET_URL, False), + pytest.param( + f"{TARGET_URL},another-page", + TARGET_URL, + False, + id="comma-within-longer-url", + ), + ("https://dennisfink.me/blog/other/", TARGET_URL, False), + ("/blog/example/", TARGET_URL, False), + ("There are no links here.", TARGET_URL, False), + ], +) +def test_text_mentions_target( + receiver: ModuleType, body: str, target_url: str, expected: bool +) -> None: + assert receiver.text_mentions_target(body, target_url) is expected + + +@patch("webmentions_ssg.tasks.mentions.BeautifulSoup") +def test_html_mentions_target_ignores_non_string_attribute( + beautiful_soup: Mock, receiver: ModuleType +) -> None: + document = beautiful_soup.return_value + document.select_one.return_value = None + + element = Mock() + element.get.return_value = ["not", "a", "string"] + document.select.return_value = [element] + + assert not receiver.html_mentions_target(b"", SOURCE_URL, TARGET_URL) diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 44d1cc2..5441eb7 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -15,7 +15,7 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import ReceivedWebmention -from webmentions_ssg.url_security import AddressResolutionError +from webmentions_ssg.net import AddressResolutionError SOURCE_URL = "https://source.example/article" TARGET_URL = "https://dennisfink.me/blog/example/" @@ -68,148 +68,6 @@ def set_stream_response(httpx_client: Mock, response: httpx.Response) -> Mock: return client -@pytest.mark.parametrize( - ("body", "expected"), - [ - pytest.param(f'Reply', True, id="a-href"), - pytest.param(f'Target', True, id="area-href"), - pytest.param( - f'', True, id="link-href" - ), - pytest.param(f'', True, id="img-src"), - pytest.param(f'', True, id="audio-src"), - pytest.param(f'', True, id="video-src"), - pytest.param( - f'', True, id="audio-source-src" - ), - pytest.param( - f'', True, id="video-source-src" - ), - pytest.param(f'', True, id="iframe-src"), - pytest.param(f'', True, id="embed-src"), - pytest.param(f'', True, id="script-src"), - pytest.param( - f'', True, id="track-src" - ), - pytest.param( - f'', - True, - id="image-input-src", - ), - pytest.param( - f'', - True, - id="image-input-case-insensitive", - ), - pytest.param( - f'
Quotation
', - True, - id="blockquote-cite", - ), - pytest.param(f'Quotation', True, id="q-cite"), - pytest.param(f'Addition', True, id="ins-cite"), - pytest.param(f'Removal', True, id="del-cite"), - pytest.param( - 'Reply', - True, - id="base-url", - ), - pytest.param(f'', False, id="img-cite-invalid"), - pytest.param( - f'
Quote
', - False, - id="blockquote-src-invalid", - ), - pytest.param(f'Reply', False, id="a-src-invalid"), - pytest.param(f'
', False, id="div-href-invalid"), - pytest.param(f'', False, id="link-src-invalid"), - pytest.param( - f'', - False, - id="text-input-src-invalid", - ), - pytest.param( - f'', - False, - id="picture-source-src-invalid", - ), - pytest.param(f'', False, id="base-is-not-mention"), - pytest.param( - f'Different page', False, id="longer-url" - ), - pytest.param( - f'', - False, - id="invalid-cite-does-not-override-valid-src", - ), - pytest.param(f"

{TARGET_URL}

", False, id="text-content"), - pytest.param( - 'Other site', False, id="missing-target" - ), - ], -) -def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None: - assert ( - receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected - ) - - -def test_html_mentions_relative_target(receiver: ModuleType) -> None: - assert receiver.html_mentions_target( - b'Reply', - "https://source.example/posts/article/", - "https://source.example/posts/target/", - ) - - -@pytest.mark.parametrize( - ("body", "target_url", "expected"), - [ - (TARGET_URL, TARGET_URL, True), - (f"This post replies to {TARGET_URL}", TARGET_URL, True), - ( - f"https://example.com/first {TARGET_URL} https://example.com/last", - TARGET_URL, - True, - ), - pytest.param( - f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma" - ), - pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"), - pytest.param( - f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation" - ), - pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"), - pytest.param( - "https://example.com/wiki/Foo_(bar)", - "https://example.com/wiki/Foo_(bar)", - True, - id="balanced-parentheses-in-url", - ), - pytest.param( - "https://example.com/wiki/Foo_(bar)", - "https://example.com/wiki/Foo_(bar", - False, - id="balanced-parentheses-not-trimmed", - ), - (f"{TARGET_URL}more", TARGET_URL, False), - pytest.param( - f"{TARGET_URL},another-page", - TARGET_URL, - False, - id="comma-within-longer-url", - ), - ("https://dennisfink.me/blog/other/", TARGET_URL, False), - ("/blog/example/", TARGET_URL, False), - ("There are no links here.", TARGET_URL, False), - ], -) -def test_text_mentions_target( - receiver: ModuleType, body: str, target_url: str, expected: bool -) -> None: - assert receiver.text_mentions_target(body, target_url) is expected - - @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_returns_response_and_body( httpx_client: Mock, app: Flask, receiver: ModuleType @@ -786,7 +644,7 @@ def test_verify_webmention_ignores_unknown_identifier( def test_fetch_source_rejects_unsafe_destination( httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: - from webmentions_ssg.url_security import UnsafeAddressError + from webmentions_ssg.net import UnsafeAddressError client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = UnsafeAddressError("Local address") @@ -810,20 +668,6 @@ def test_fetch_source_treats_dns_failure_as_temporary( receiver.fetch_source(SOURCE_URL) -@patch("webmentions_ssg.tasks.receiver.BeautifulSoup") -def test_html_mentions_target_ignores_non_string_attribute( - beautiful_soup: Mock, receiver: ModuleType -) -> None: - document = beautiful_soup.return_value - document.select_one.return_value = None - - element = Mock() - element.get.return_value = ["not", "a", "string"] - document.select.return_value = [element] - - assert not receiver.html_mentions_target(b"", SOURCE_URL, TARGET_URL) - - @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_ignores_invalid_content_length( httpx_client: Mock, app: Flask, receiver: ModuleType diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 26e81c4..7eb970c 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -18,9 +18,9 @@ from pytest import MonkeyPatch from webmentions_ssg import DATABASE as db from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source +from webmentions_ssg.tasks import source_parser -BASE_URL = "https://dennisfink.me/blog/" -SOURCE_URL = f"{BASE_URL}example/" +from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post @pytest.fixture @@ -29,6 +29,7 @@ def scanner_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: from webmentions_ssg.tasks import scanner monkeypatch.setattr(scanner, "is_http_url", Mock(return_value=True)) + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) return scanner @@ -80,764 +81,6 @@ def test_manual_scan_propagates_unrelated_errors( scanner_module.manual_scan_sources.call_local() -def parse_html(html: str) -> BeautifulSoup: - return BeautifulSoup(html, "html5lib") - - -def write_post( - root: Path, - slug: str, - content: str, - *, - canonical: str | None = None, - entry_url: str | None = None, -) -> Path: - directory = root / slug - directory.mkdir(parents=True, exist_ok=True) - - source_url = f"{BASE_URL}{slug}/" - - if canonical is None: - canonical = source_url - - if entry_url is None: - entry_url = source_url - - path = directory / "index.html" - path.write_text( - f""" - - - - - - - - - - """, - encoding="utf-8", - ) - - return path - - -def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None: - app.config["SOURCE_DIRECTORY"] = str(root) - app.config["SOURCE_BASE_URL"] = base_url - - -def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - - """ - ) - - element = document.find(class_="h-entry") - - assert element is not None - - entry = scanner_module.parse_entry(element, SOURCE_URL) - - assert "h-entry" in entry["type"] - assert entry["properties"]["url"] == [SOURCE_URL] - - -def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-

Hello world

-
- """ - ) - - element = document.find("article") - - assert element is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one parsed h-entry" - ): - scanner_module.parse_entry(element, SOURCE_URL) - - -def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - - """ - ) - - element, entry = scanner_module.primary_entry(document, SOURCE_URL) - - assert element["id"] == "source" - assert entry["properties"]["name"] == ["Example post"] - assert entry["properties"]["url"] == [SOURCE_URL] - - -def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-

No microformats here.

-
- """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - - - - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" - ): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-
- Hello world -
-
- """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="does not match"): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None: - document = parse_html( - """ - - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="does not match"): - scanner_module.primary_entry(document, SOURCE_URL) - - -def test_content_element_returns_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-
- Hello world -
-
- """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - content = scanner_module.content_element(entry) - - assert content["id"] == "content" - - -def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-

Hello world

-
- """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one e-content, found 0" - ): - scanner_module.content_element(entry) - - -def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None: - document = parse_html( - """ -
-
- First -
- -
- Second -
-
- """ - ) - - entry = document.find(class_="h-entry") - - assert entry is not None - - with pytest.raises( - scanner_module.SourceScanError, match="Expected exactly one e-content, found 2" - ): - scanner_module.content_element(entry) - - -def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None: - document = parse_html( - f""" - - - - - - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - assert result == SOURCE_URL - - -def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None: - document = parse_html( - """ - - - Example - - - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - assert result is None - - -def test_canonical_url_resolves_relative_url_with_base_url( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.side_effect = [False, True] - - document = parse_html( - """ - - - - - - """ - ) - - result = scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - assert result == ("https://dennisfink.me/blog/example/canonical/") - - -def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None: - scanner_module.is_http_url.return_value = False - - document = parse_html( - """ - - - - - - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="relative canonical"): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=None - ) - - -def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None: - document = parse_html( - """ - - - - - - """ - ) - - with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - -def test_scan_source_uses_canonical_without_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - - Target - - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.path == "example/index.html" - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - assert len(scanned.content_hash) == 32 - - -def test_scan_source_ignores_markup_noise_in_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - "

Hello world!

" - 'Target', - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - write_post( - tmp_path, - "example", - '

Hello world!

' - ' Target ', - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == second.targets - assert first.content_hash == second.content_hash - - -def test_scan_source_uses_stable_time_value_inside_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - '', - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.content_hash == second.content_hash - - -def test_scan_source_hash_changes_when_image_changes( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", 'A photo') - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.content_hash != second.content_hash - - -def test_scan_source_hash_includes_reaction_targets_outside_content( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", "

Same content.

") - html = path.read_text(encoding="utf-8") - path.write_text( - html.replace( - '
', - 'Like' - '
', - ), - encoding="utf-8", - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace( - "https://example.com/first", "https://example.com/second" - ), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == frozenset({"https://example.com/first"}) - assert second.targets == frozenset({"https://example.com/second"}) - assert first.content_hash != second.content_hash - - -def test_scan_source_hash_ignores_reaction_target_order( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post(tmp_path, "example", "

Same content.

") - html = path.read_text(encoding="utf-8") - first_link = 'A' - second_link = 'B' - path.write_text( - html.replace( - '
', - first_link + second_link + '
', - ), - encoding="utf-8", - ) - first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - path.write_text( - path.read_text(encoding="utf-8").replace( - first_link + second_link, second_link + first_link - ), - encoding="utf-8", - ) - second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert first.targets == second.targets - assert first.content_hash == second.content_hash - - -def test_scan_source_falls_back_to_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - """ - - - - - - - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL) - - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - - -def test_scan_source_requires_canonical_or_base_url( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - """ - - - - - - - """, - encoding="utf-8", - ) - - with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"): - scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - -def test_scan_source_extracts_only_content_links( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - f""" - - - - - - - - - - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://example.com/content"}) - - -def test_scan_source_extracts_relative_content_link( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - - Other post - - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) - - -@pytest.mark.parametrize( - "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") -) -def test_scan_source_extracts_reaction_properties( - scanner_module: ModuleType, tmp_path: Path, property_name: str -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - target = f"https://example.com/{property_name}" - - path = directory / "index.html" - path.write_text( - f""" - - - - - - - - - - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({target}) - - -@pytest.mark.parametrize("tag", ("link", "area")) -def test_scan_source_ignores_non_anchor_href_elements( - scanner_module: ModuleType, tmp_path: Path, tag: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - <{tag} href="https://example.com/target"> - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset() - - -def test_scan_source_ignores_self_fragment( - scanner_module: ModuleType, tmp_path: Path -) -> None: - path = write_post( - tmp_path, - "example", - """ - - Section - - - - Target - - """, - ) - - scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) - - assert scanned.targets == frozenset({"https://example.com/target"}) - - def test_scan_creates_source_and_webmentions( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1885,23 +1128,23 @@ def test_unexpected_parser_error_does_not_abort_other_sources( write_post(tmp_path, "good", 'Link') if parser == "bs4": - original_soup = scanner_module.BeautifulSoup + original_soup = source_parser.BeautifulSoup def failing_soup(markup: bytes, features: str) -> BeautifulSoup: if b"/bad/" in markup: raise failure return original_soup(markup, features) - monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup) + monkeypatch.setattr(source_parser, "BeautifulSoup", failing_soup) else: - original_parse = scanner_module.mf2py.parse + original_parse = source_parser.mf2py.parse def failing_parse(*, doc: str, url: str) -> dict[str, Any]: if "/bad/" in doc: raise failure return original_parse(doc=doc, url=url) - monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse) + monkeypatch.setattr(source_parser.mf2py, "parse", failing_parse) queued: list[UUID] = [] monkeypatch.setattr(scanner_module, "send_webmention", queued.append) @@ -1979,46 +1222,6 @@ def test_unexpected_persistence_error_is_not_suppressed( run_scan(app, scanner_module) -def test_scan_source_rejects_symlink_outside_root_before_read( - scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch -) -> None: - root = tmp_path / "site" - root.mkdir() - external = write_post(tmp_path / "outside", "private", "Sensitive content") - linked = root / "escaped" / "index.html" - linked.parent.mkdir() - linked.symlink_to(external) - - read_bytes = Mock(side_effect=AssertionError("External file must not be read")) - monkeypatch.setattr(Path, "read_bytes", read_bytes) - - with pytest.raises( - scanner_module.SourceScanError, match="escapes source directory" - ): - scanner_module.scan_source_file(linked, root=root, base_url=None) - - read_bytes.assert_not_called() - - -def test_scan_source_accepts_symlink_within_root( - scanner_module: ModuleType, tmp_path: Path -) -> None: - root = tmp_path / "site" - root.mkdir() - linked = write_post( - root, "example", 'Target' - ) - shared = root / "shared.html" - linked.rename(shared) - linked.symlink_to(shared) - - scanned = scanner_module.scan_source_file(linked, root=root, base_url=None) - - assert scanned.path == "example/index.html" - assert scanned.url == SOURCE_URL - assert scanned.targets == frozenset({"https://example.com/target"}) - - def test_scan_sources_skips_symlink_outside_root( app: Flask, scanner_module: ModuleType, @@ -2043,16 +1246,6 @@ def test_scan_sources_skips_symlink_outside_root( assert "escapes source directory" in caplog.text -def test_scan_sources_rejects_malformed_runtime_ignored_hostnames( - app: Flask, scanner_module: ModuleType, tmp_path: Path -) -> None: - configure_scanner(app, tmp_path) - app.config["IGNORED_HOSTNAMES"] = "example.com" - - with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"): - run_scan(app, scanner_module) - - def test_scan_sources_accepts_valid_base_url( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -2104,220 +1297,6 @@ def test_scan_sources_rejects_invalid_base_url( run_scan(app, scanner_module) -@pytest.mark.parametrize( - "href", - ( - "https://dennisfink.me/blog/other/", - "https://www.dennisfink.me/blog/other/", - "https://webmentions.dennisfink.me/endpoint", - "https://foo.bar.dennisfink.me/example", - ), -) -def test_scan_source_ignores_matching_hostname( - scanner_module: ModuleType, tmp_path: Path, href: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - - Ignored - - - - External - - """, - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset({"https://example.com/target"}) - - -@pytest.mark.parametrize( - "href", - ( - "https://notdennisfink.me/example", - "https://dennisfink.me.example.com/example", - "https://example.com/example", - ), -) -def test_scan_source_keeps_nonmatching_hostname( - scanner_module: ModuleType, tmp_path: Path, href: str -) -> None: - path = write_post( - tmp_path, - "example", - f""" - - Target - - """, - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset({href}) - - -def test_scan_source_ignores_reaction_to_matching_hostname( - scanner_module: ModuleType, tmp_path: Path -) -> None: - directory = tmp_path / "example" - directory.mkdir() - - path = directory / "index.html" - path.write_text( - f""" - - - - - - - - - - """, - encoding="utf-8", - ) - - scanned = scanner_module.scan_source_file( - path, - root=tmp_path, - base_url=None, - ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), - ) - - assert scanned.targets == frozenset() - - -def test_canonical_url_rejects_non_http_resolved_url( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.return_value = False - - document = parse_html( - """ - - - - - - """ - ) - - with pytest.raises( - scanner_module.SourceScanError, - match="Canonical URL is not a valid HTTP or HTTPS URL", - ): - scanner_module.canonical_url( - document, relative_path=Path("example/index.html"), base_url=BASE_URL - ) - - -def test_property_urls_ignores_non_mapping_properties( - scanner_module: ModuleType, -) -> None: - entry = {"properties": "invalid"} - - assert tuple(scanner_module.property_urls(entry, "url")) == () - - -@pytest.mark.parametrize( - ("parsed", "message"), - [ - pytest.param( - [], - "Microformats parser did not return a document object", - id="non-object-document", - ), - pytest.param( - {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" - ), - pytest.param( - {"items": [{"type": ["h-card"]}]}, - "Parsed microformats item is not an h-entry", - id="wrong-item-type", - ), - ], -) -def test_parse_entry_rejects_invalid_parser_output( - scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str -) -> None: - document = parse_html( - """ -
-

Hello world

-
- """ - ) - - element = document.find(class_="h-entry") - - assert element is not None - - monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed) - - with pytest.raises(scanner_module.SourceScanError, match=message): - scanner_module.parse_entry(element, SOURCE_URL) - - -def test_normalize_target_ignores_empty_target(scanner_module: ModuleType) -> None: - assert ( - scanner_module.normalize_target(" ", base_url=SOURCE_URL, source_url=SOURCE_URL) - is None - ) - - -def test_normalize_target_ignores_url_rejected_by_validator( - scanner_module: ModuleType, -) -> None: - scanner_module.is_http_url.return_value = False - - assert ( - scanner_module.normalize_target( - "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL - ) - is None - ) - - def test_changed_source_url_is_rejected( app: Flask, scanner_module: ModuleType, diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 7ebe083..c2f6e02 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -14,7 +14,6 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source -from webmentions_ssg.url_security import AddressResolutionError SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" @@ -100,25 +99,6 @@ def sender_module(app: Flask) -> ModuleType: return sender -@pytest.mark.parametrize( - ("status", "expected"), - [ - pytest.param(200, False, id="success"), - pytest.param(400, False, id="client-error"), - pytest.param(408, True, id="request-timeout"), - pytest.param(425, True, id="too-early"), - pytest.param(429, True, id="too-many-requests"), - pytest.param(500, True, id="server-error-lower-bound"), - pytest.param(599, True, id="server-error-upper-bound"), - pytest.param(600, False, id="outside-server-error-range"), - ], -) -def test_temporary_http_status( - sender_module: ModuleType, status: int, expected: bool -) -> None: - assert sender_module.temporary_http_status(status) is expected - - @pytest.mark.parametrize( ("desired_revision", "processed_revision", "revision", "expected"), [ @@ -147,671 +127,6 @@ def test_attempt_is_current( assert sender_module.attempt_is_current(webmention, revision) is expected -@pytest.mark.parametrize( - ("value", "expected"), - ( - ( - '; rel="webmention"', - ("https://example.com/webmention", {"webmention"}), - ), - ( - "; rel=webmention", - ("https://example.com/webmention", {"webmention"}), - ), - ( - '; rel="webmention alternate"', - ("https://example.com/webmention", {"webmention", "alternate"}), - ), - ( - '; REL="WebMention Alternate"', - ("https://example.com/webmention", {"webmention", "alternate"}), - ), - ( - ( - '; type="text/html"; ' - 'rel="webmention"; title="Endpoint"' - ), - ("https://example.com/webmention", {"webmention"}), - ), - ( - '; type="text/css"', - ("https://example.com/stylesheet", set()), - ), - ), -) -def test_parse_link_value( - sender_module: ModuleType, value: str, expected: tuple[str, set[str]] -) -> None: - assert sender_module.parse_link_value(value) == expected - - -@pytest.mark.parametrize( - "value", ("", "https://example.com/webmention", " None: - assert sender_module.parse_link_value(value) is None - - -@pytest.mark.parametrize( - ("target", "headers", "html", "expected_endpoint"), - ( - pytest.param( - "https://example.com/test/1", - (("Link", "; rel=webmention"),), - "", - "https://example.com/test/1/webmention?head=true", - id="1-http-link-unquoted-rel-relative-url", - ), - pytest.param( - "https://example.com/test/2", - ( - ( - "Link", - ( - ";" - "rel=webmention" - ), - ), - ), - "", - "https://example.com/test/2/webmention?head=true", - id="2-http-link-unquoted-rel-absolute-url", - ), - pytest.param( - "https://example.com/test/3", - (), - """ - - """, - "https://example.com/test/3/webmention", - id="3-html-link-relative-url", - ), - pytest.param( - "https://example.com/test/4", - (), - """ - - """, - "https://example.com/test/4/webmention", - id="4-html-link-absolute-url", - ), - pytest.param( - "https://example.com/test/5", - (), - """ - - Endpoint - - """, - "https://example.com/test/5/webmention", - id="5-html-a-relative-url", - ), - pytest.param( - "https://example.com/test/6", - (), - """ - - Endpoint - - """, - "https://example.com/test/6/webmention", - id="6-html-a-absolute-url", - ), - pytest.param( - "https://example.com/test/7", - ( - ( - "LinK", - ( - "; " - "rel=webmention" - ), - ), - ), - "", - "https://example.com/test/7/webmention?head=true", - id="7-http-link-strange-casing", - ), - pytest.param( - "https://example.com/test/8", - ( - ( - "Link", - ( - "; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/8/webmention?head=true", - id="8-http-link-quoted-rel", - ), - pytest.param( - "https://example.com/test/9", - (), - """ - - """, - "https://example.com/test/9/webmention", - id="9-multiple-html-rel-values", - ), - pytest.param( - "https://example.com/test/10", - ( - ( - "Link", - ( - "; " - 'rel="webmention somethingelse"' - ), - ), - ), - "", - "https://example.com/test/10/webmention?head=true", - id="10-multiple-link-header-rel-values", - ), - pytest.param( - "https://example.com/test/11", - (("Link", '; rel="webmention"'),), - """ - - - - Wrong endpoint - - """, - "https://example.com/test/11/webmention", - id="11-http-link-precedence", - ), - pytest.param( - "https://example.com/test/12", - (), - """ - - - - Correct endpoint - - """, - "https://example.com/test/12/webmention", - id="12-exact-rel-match", - ), - pytest.param( - "https://example.com/test/13", - (), - """ - - - - Correct endpoint - - """, - "https://example.com/test/13/webmention", - id="13-endpoint-inside-html-comment", - ), - pytest.param( - "https://example.com/test/14", - (), - """ - - <a - rel="webmention" - href="/test/14/webmention/error" - > - False endpoint - </a> - - - - Correct endpoint - - """, - "https://example.com/test/14/webmention", - id="14-endpoint-in-escaped-html", - ), - pytest.param( - "https://example.com/test/15", - (), - """ - - """, - "https://example.com/test/15", - id="15-empty-href", - ), - pytest.param( - "https://example.com/test/16", - (), - """ - - First endpoint - - - - """, - "https://example.com/test/16/webmention", - id="16-a-before-link", - ), - pytest.param( - "https://example.com/test/17", - (), - """ - - - - Second endpoint - - """, - "https://example.com/test/17/webmention", - id="17-link-before-a", - ), - pytest.param( - "https://example.com/test/18", - ( - ("Link", '; rel="other"'), - ( - "Link", - ( - "; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/18/webmention?head=true", - id="18-multiple-http-link-headers", - ), - pytest.param( - "https://example.com/test/19", - ( - ( - "Link", - ( - "; " - 'rel="other", ' - "; " - 'rel="webmention"' - ), - ), - ), - "", - "https://example.com/test/19/webmention?head=true", - id="19-single-header-multiple-values", - ), - pytest.param( - "https://example.com/test/20", - (), - """ - - - - Correct endpoint - - """, - "https://example.com/test/20/webmention", - id="20-link-without-href", - ), - pytest.param( - "https://example.com/test/21", - (), - """ - - """, - "https://example.com/test/21/webmention?query=yes", - id="21-endpoint-query-string", - ), - pytest.param( - "https://example.com/test/22", - (), - """ - - """, - "https://example.com/test/22/webmention", - id="22-endpoint-relative-to-path", - ), - ), -) -def test_discovery( - app: Flask, - sender_module: ModuleType, - target: str, - headers: tuple[tuple[str, str], ...], - html: str, - expected_endpoint: str, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200, headers=headers) - - return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, target) - - assert endpoint == expected_endpoint - - -@pytest.mark.parametrize( - ("target", "responses", "expected_endpoint"), - ( - pytest.param( - "https://example.com/test/23/page", - { - "/test/23/page": (302, {"Location": "page/redirect-key"}), - "/test/23/page/redirect-key": ( - 200, - {"Link": ("; rel=webmention")}, - ), - }, - ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), - id="relative-endpoint-after-redirect", - ), - ), -) -def test_discovery_redirect( - app: Flask, - sender_module: ModuleType, - target: str, - responses: dict[str, tuple[int, dict[str, str]]], - expected_endpoint: str, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - try: - status, headers = responses[request.url.path] - except KeyError: - raise AssertionError(f"Unexpected request: {request.url}") from None - - return httpx.Response(status, headers=headers) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, target) - - assert endpoint == expected_endpoint - - -def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - return httpx.Response(200, headers={"Link": "; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint( - client, "https://example.com/post" - ) - - assert endpoint == "https://example.com/webmention" - assert [request.method for request in requests] == ["HEAD"] - - -def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text=""" - - """, - ) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint( - client, "https://example.com/post" - ) - - assert endpoint == "https://example.com/webmention" - assert [request.method for request in requests] == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - "failure", - [ - pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"), - pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"), - ], -) -def test_discovery_falls_back_to_get_after_head_request_error( - app: Flask, sender_module: ModuleType, failure: httpx.RequestError -) -> None: - methods: list[str] = [] - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - - if request.method == "HEAD": - raise failure - - return httpx.Response(200, headers={"Link": "; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert methods == ["HEAD", "GET"] - - -def test_discovery_falls_back_to_get_after_head_dns_failure( - app: Flask, sender_module: ModuleType -) -> None: - methods: list[str] = [] - - def request_hook(request: httpx.Request) -> None: - if request.method == "HEAD": - raise sender_module.TemporarySenderError("HEAD DNS lookup failed") - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - return httpx.Response(200, headers={"Link": "; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), - event_hooks={"request": [request_hook]}, - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert methods == ["GET"] - - -def test_discovery_get_error_after_head_error_is_not_suppressed( - app: Flask, sender_module: ModuleType -) -> None: - methods: list[str] = [] - - def handler(request: httpx.Request) -> httpx.Response: - methods.append(request.method) - raise httpx.ConnectError(f"{request.method} failed") - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - pytest.raises(httpx.ConnectError, match="GET failed"), - ): - sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert methods == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - "invalid_href", - [ - pytest.param("javascript:alert(1)", id="unsupported-scheme"), - pytest.param("http://[invalid", id="malformed-url"), - ], -) -def test_discovery_skips_invalid_html_candidate( - app: Flask, sender_module: ModuleType, invalid_href: str -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text=( - f'' - 'Valid endpoint' - ), - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - - -def test_discovery_skips_invalid_link_header_candidate( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - headers=[ - ("Link", "; rel=webmention"), - ("Link", "; rel=webmention"), - ("Link", "; rel=webmention"), - ], - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - - -def test_discovery_returns_none_when_all_html_candidates_are_invalid( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200, headers={"Link": "; rel=webmention"}) - - return httpx.Response( - 200, - headers={"Content-Type": "text/html"}, - text='', - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint is None - - def test_send_webmention_persists_success( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1315,267 +630,6 @@ def test_send_webmention_ignores_unknown_identifier( assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text -def test_resolve_endpoint_rejects_invalid_url( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - is_http_url = Mock(return_value=False) - monkeypatch.setattr(sender_module, "is_http_url", is_http_url) - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - with pytest.raises( - sender_module.PermanentSenderError, match="Invalid Webmention endpoint" - ): - sender_module.resolve_endpoint(response, "/webmention") - is_http_url.assert_called_once_with(ENDPOINT_URL) - - -def test_endpoint_from_headers_ignores_malformed_link( - sender_module: ModuleType, -) -> None: - response = httpx.Response( - 200, - headers=[("Link", "not-a-link"), ("Link", "; rel=webmention")], - request=httpx.Request("GET", TARGET_URL), - ) - assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL - - -def test_endpoint_from_html_accepts_string_rel( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - element = Mock() - element.get.side_effect = lambda name: { - "rel": "webmention alternate", - "href": "/webmention", - }[name] - - document = Mock() - document.find_all.return_value = [element] - - beautiful_soup = Mock(return_value=document) - monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) - - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - - assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL - - -def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - missing_rel = Mock() - missing_rel.get.return_value = None - - invalid_href = Mock() - invalid_href.get.side_effect = lambda name: { - "rel": ["webmention"], - "href": ["invalid"], - }[name] - - document = Mock() - document.find_all.return_value = [missing_rel, invalid_href] - - beautiful_soup = Mock(return_value=document) - monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup) - - response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL)) - - assert sender_module.endpoint_from_html(response, b"") is None - - -def test_read_target_body_rejects_declared_oversized_body( - app: Flask, sender_module: ModuleType -) -> None: - app.config["MAX_TARGET_BYTES"] = 10 - - response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11) - - with ( - app.app_context(), - pytest.raises( - sender_module.PermanentSenderError, match="Target document is too large" - ), - ): - sender_module.read_target_body(response) - - -def test_read_target_body_ignores_invalid_content_length( - app: Flask, sender_module: ModuleType -) -> None: - response = httpx.Response( - 200, headers={"Content-Length": "invalid"}, content=b"Document" - ) - - with app.app_context(): - body = sender_module.read_target_body(response) - - assert body == b"Document" - - -def test_read_target_body_rejects_streamed_oversized_body( - app: Flask, sender_module: ModuleType -) -> None: - app.config["MAX_TARGET_BYTES"] = 10 - - class BodyStream(httpx.SyncByteStream): - def __iter__(self): - yield b"x" * 6 - yield b"x" * 6 - - response = httpx.Response(200, stream=BodyStream()) - - with ( - app.app_context(), - pytest.raises( - sender_module.PermanentSenderError, match="Target document is too large" - ), - ): - sender_module.read_target_body(response) - - -def test_discovery_uses_link_header_from_get( - app: Flask, sender_module: ModuleType -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response(200, headers={"Link": "; rel=webmention"}) - - with ( - app.app_context(), - httpx.Client( - transport=httpx.MockTransport(handler), follow_redirects=True - ) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint == ENDPOINT_URL - assert [request.method for request in requests] == ["HEAD", "GET"] - - -@pytest.mark.parametrize( - ("status_code", "exception_name"), - [ - pytest.param(503, "TemporarySenderError", id="temporary"), - pytest.param(404, "PermanentSenderError", id="permanent"), - ], -) -def test_discovery_maps_get_status_to_exception( - app: Flask, sender_module: ModuleType, status_code: int, exception_name: str -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response(status_code) - - exception_type = getattr(sender_module, exception_name) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"), - ): - sender_module.discover_webmention_endpoint(client, TARGET_URL) - - -def test_discovery_returns_none_for_unsupported_media_type( - app: Flask, sender_module: ModuleType -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "HEAD": - return httpx.Response(200) - - return httpx.Response( - 200, headers={"Content-Type": "application/json"}, content=b"{}" - ) - - with ( - app.app_context(), - httpx.Client(transport=httpx.MockTransport(handler)) as client, - ): - endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL) - - assert endpoint is None - - -@pytest.mark.parametrize( - ("status_code", "location", "expected_status_url"), - [ - pytest.param( - 201, "/webmention/status/123", STATUS_URL, id="created-with-location" - ), - pytest.param(202, None, None, id="accepted"), - ], -) -def test_post_webmention( - sender_module: ModuleType, - status_code: int, - location: str | None, - expected_status_url: str | None, -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - - headers = {} - - if location is not None: - headers["Location"] = location - - return httpx.Response(status_code, headers=headers) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - result = sender_module.post_webmention( - client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL - ) - - assert result == (status_code, expected_status_url) - assert len(requests) == 1 - - request = requests[0] - - assert request.method == "POST" - assert str(request.url) == ENDPOINT_URL - - form = httpx.QueryParams(request.content.decode()) - - assert form["source"] == SOURCE_URL - assert form["target"] == TARGET_URL - - -@pytest.mark.parametrize( - ("code", "location", "expected"), - [ - (201, "/webmention/status/123", "https://example.com/webmention/status/123"), - (201, "https://status.example.org/123", "https://status.example.org/123"), - (201, "status/123", "https://example.com/status/123"), - (201, "javascript:alert(1)", None), - (201, "data:text/html,evil", None), - (201, "ftp://example.com/status", None), - (201, "http://[invalid", None), - (202, "https://example.com/status", None), - ], -) -def test_post_webmention_validates_location( - sender_module: ModuleType, code: int, location: str, expected: str | None -) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(code, headers={"Location": location}) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - result = sender_module.post_webmention( - client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL - ) - - assert result == (code, expected) - - @pytest.mark.parametrize( ("status_url", "expected"), [ diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py new file mode 100644 index 0000000..3c3a93e --- /dev/null +++ b/tests/tasks/test_source_parser.py @@ -0,0 +1,1008 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from pytest import MonkeyPatch + +from webmentions_ssg.tasks import source_parser + +from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post + + +@pytest.fixture +def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + _ = app + from webmentions_ssg.tasks import source_parser + + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) + return source_parser + + +def test_parse_entry_returns_microformats_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + entry = source_parser_module.parse_entry(element, SOURCE_URL) + + assert "h-entry" in entry["type"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + element = document.find("article") + + assert element is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one parsed h-entry", + ): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + + """ + ) + + element, entry = source_parser_module.primary_entry(document, SOURCE_URL) + + assert element["id"] == "source" + assert entry["properties"]["name"] == ["Example post"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_primary_entry_rejects_missing_h_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+

No microformats here.

+
+ """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 0", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_multiple_h_entries( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+
+ Hello world +
+
+ """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nonmatching_u_url( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + + """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+
+ Hello world +
+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + content = source_parser_module.content_element(entry) + + assert content["id"] == "content" + + +def test_content_element_rejects_missing_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 0", + ): + source_parser_module.content_element(entry) + + +def test_content_element_rejects_multiple_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+
+ First +
+ +
+ Second +
+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 2", + ): + source_parser_module.content_element(entry) + + +def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + + + + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result == SOURCE_URL + + +def test_canonical_url_returns_none_when_missing( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + + + Example + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result is None + + +def test_canonical_url_resolves_relative_url_with_base_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.side_effect = [False, True] + + document = parse_html( + """ + + + + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + assert result == ("https://dennisfink.me/blog/example/canonical/") + + +def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="relative canonical" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + +def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="empty canonical URL" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_scan_source_uses_canonical_without_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + assert len(scanned.content_hash) == 32 + + +def test_scan_source_ignores_markup_noise_in_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "

Hello world!

" + 'Target', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '

Hello world!

' + ' Target ', + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", 'A photo') + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '
', + 'Like' + '
', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + first_link = 'A' + second_link = 'B' + path.write_text( + html.replace( + '
', + first_link + second_link + '
', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_falls_back_to_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, root=tmp_path, base_url=BASE_URL + ) + + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_requires_canonical_or_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + + + + + + + """, + encoding="utf-8", + ) + + with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"): + source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + +def test_scan_source_extracts_only_content_links( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/content"}) + + +def test_scan_source_extracts_relative_content_link( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Other post + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) + + +@pytest.mark.parametrize( + "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") +) +def test_scan_source_extracts_reaction_properties( + source_parser_module: ModuleType, tmp_path: Path, property_name: str +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + target = f"https://example.com/{property_name}" + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({target}) + + +@pytest.mark.parametrize("tag", ("link", "area")) +def test_scan_source_ignores_non_anchor_href_elements( + source_parser_module: ModuleType, tmp_path: Path, tag: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <{tag} href="https://example.com/target"> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset() + + +def test_scan_source_ignores_self_fragment( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Section + + + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_rejects_symlink_outside_root_before_read( + source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + root = tmp_path / "site" + root.mkdir() + external = write_post(tmp_path / "outside", "private", "Sensitive content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + + read_bytes = Mock(side_effect=AssertionError("External file must not be read")) + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + with pytest.raises( + source_parser_module.SourceScanError, match="escapes source directory" + ): + source_parser_module.scan_source_file(linked, root=root, base_url=None) + + read_bytes.assert_not_called() + + +def test_scan_source_accepts_symlink_within_root( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + root = tmp_path / "site" + root.mkdir() + linked = write_post( + root, "example", 'Target' + ) + shared = root / "shared.html" + linked.rename(shared) + linked.symlink_to(shared) + + scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://dennisfink.me/blog/other/", + "https://www.dennisfink.me/blog/other/", + "https://webmentions.dennisfink.me/endpoint", + "https://foo.bar.dennisfink.me/example", + ), +) +def test_scan_source_ignores_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + + Ignored + + + + External + + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://notdennisfink.me/example", + "https://dennisfink.me.example.com/example", + "https://example.com/example", + ), +) +def test_scan_source_keeps_nonmatching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({href}) + + +def test_scan_source_ignores_reaction_to_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset() + + +def test_canonical_url_rejects_non_http_resolved_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Canonical URL is not a valid HTTP or HTTPS URL", + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_property_urls_ignores_non_mapping_properties( + source_parser_module: ModuleType, +) -> None: + entry = {"properties": "invalid"} + + assert tuple(source_parser_module.property_urls(entry, "url")) == () + + +@pytest.mark.parametrize( + ("parsed", "message"), + [ + pytest.param( + [], + "Microformats parser did not return a document object", + id="non-object-document", + ), + pytest.param( + {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" + ), + pytest.param( + {"items": [{"type": ["h-card"]}]}, + "Parsed microformats item is not an h-entry", + id="wrong-item-type", + ), + ], +) +def test_parse_entry_rejects_invalid_parser_output( + source_parser_module: ModuleType, + monkeypatch: MonkeyPatch, + parsed: object, + message: str, +) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed) + + with pytest.raises(source_parser_module.SourceScanError, match=message): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_normalize_target_ignores_empty_target( + source_parser_module: ModuleType, +) -> None: + assert ( + source_parser_module.normalize_target( + " ", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) + + +def test_normalize_target_ignores_url_rejected_by_validator( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + assert ( + source_parser_module.normalize_target( + "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) -- cgit v1.3.1