aboutsummaryrefslogtreecommitdiff
path: root/tests/tasks
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /tests/tasks
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to '')
-rw-r--r--tests/tasks/scanner_helpers.py72
-rw-r--r--tests/tasks/test_discovery.py951
-rw-r--r--tests/tasks/test_json_serializer.py2
-rw-r--r--tests/tasks/test_mentions.py176
-rw-r--r--tests/tasks/test_receiver.py160
-rw-r--r--tests/tasks/test_scanner.py1035
-rw-r--r--tests/tasks/test_sender.py946
-rw-r--r--tests/tasks/test_source_parser.py1008
8 files changed, 2217 insertions, 2133 deletions
diff --git a/tests/tasks/scanner_helpers.py b/tests/tasks/scanner_helpers.py
new file mode 100644
index 0000000..4100028
--- /dev/null
+++ b/tests/tasks/scanner_helpers.py
@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+from flask import Flask
+
+BASE_URL = "https://dennisfink.me/blog/"
+SOURCE_URL = f"{BASE_URL}example/"
+
+
+def parse_html(html: str) -> BeautifulSoup:
+ return BeautifulSoup(html, "html5lib")
+
+
+def write_post(
+ root: Path,
+ slug: str,
+ content: str,
+ *,
+ canonical: str | None = None,
+ entry_url: str | None = None,
+) -> Path:
+ directory = root / slug
+ directory.mkdir(parents=True, exist_ok=True)
+
+ source_url = f"{BASE_URL}{slug}/"
+
+ if canonical is None:
+ canonical = source_url
+
+ if entry_url is None:
+ entry_url = source_url
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{canonical}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="{entry_url}"
+ >
+ Permalink
+ </a>
+
+ <div class="e-content">
+ {content}
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ return path
+
+
+def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None:
+ app.config["SOURCE_DIRECTORY"] = str(root)
+ app.config["SOURCE_BASE_URL"] = base_url
diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py
new file mode 100644
index 0000000..2fc30b1
--- /dev/null
+++ b/tests/tasks/test_discovery.py
@@ -0,0 +1,951 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import ModuleType
+from unittest.mock import Mock
+
+import httpx
+import pytest
+from flask import Flask
+
+from webmentions_ssg.tasks import discovery
+
+SOURCE_URL = "https://dennisfink.me/blog/example/"
+TARGET_URL = "https://example.com/post"
+ENDPOINT_URL = "https://example.com/webmention"
+STATUS_URL = "https://example.com/webmention/status/123"
+
+
+@pytest.fixture
+def sender_module(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import discovery
+
+ return discovery
+
+
+@pytest.mark.parametrize(
+ ("value", "expected"),
+ (
+ (
+ '<https://example.com/webmention>; rel="webmention"',
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ "<https://example.com/webmention>; rel=webmention",
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ '<https://example.com/webmention>; rel="webmention alternate"',
+ ("https://example.com/webmention", {"webmention", "alternate"}),
+ ),
+ (
+ '<https://example.com/webmention>; REL="WebMention Alternate"',
+ ("https://example.com/webmention", {"webmention", "alternate"}),
+ ),
+ (
+ (
+ '<https://example.com/webmention>; type="text/html"; '
+ 'rel="webmention"; title="Endpoint"'
+ ),
+ ("https://example.com/webmention", {"webmention"}),
+ ),
+ (
+ '<https://example.com/stylesheet>; type="text/css"',
+ ("https://example.com/stylesheet", set()),
+ ),
+ ),
+)
+def test_parse_link_value(
+ sender_module: ModuleType, value: str, expected: tuple[str, set[str]]
+) -> None:
+ assert sender_module.parse_link_value(value) == expected
+
+
+@pytest.mark.parametrize(
+ "value", ("", "https://example.com/webmention", "<https://example.com/webmention")
+)
+def test_parse_link_value_rejects_malformed_value(
+ sender_module: ModuleType, value: str
+) -> None:
+ assert sender_module.parse_link_value(value) is None
+
+
+@pytest.mark.parametrize(
+ ("target", "headers", "html", "expected_endpoint"),
+ (
+ pytest.param(
+ "https://example.com/test/1",
+ (("Link", "</test/1/webmention?head=true>; rel=webmention"),),
+ "",
+ "https://example.com/test/1/webmention?head=true",
+ id="1-http-link-unquoted-rel-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/2",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/2/webmention?head=true>;"
+ "rel=webmention"
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/2/webmention?head=true",
+ id="2-http-link-unquoted-rel-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/3",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/3/webmention"
+ >
+ """,
+ "https://example.com/test/3/webmention",
+ id="3-html-link-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/4",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="https://example.com/test/4/webmention"
+ >
+ """,
+ "https://example.com/test/4/webmention",
+ id="4-html-link-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/5",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="/test/5/webmention"
+ >
+ Endpoint
+ </a>
+ """,
+ "https://example.com/test/5/webmention",
+ id="5-html-a-relative-url",
+ ),
+ pytest.param(
+ "https://example.com/test/6",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="https://example.com/test/6/webmention"
+ >
+ Endpoint
+ </a>
+ """,
+ "https://example.com/test/6/webmention",
+ id="6-html-a-absolute-url",
+ ),
+ pytest.param(
+ "https://example.com/test/7",
+ (
+ (
+ "LinK",
+ (
+ "<https://example.com/test/7/webmention?head=true>; "
+ "rel=webmention"
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/7/webmention?head=true",
+ id="7-http-link-strange-casing",
+ ),
+ pytest.param(
+ "https://example.com/test/8",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/8/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/8/webmention?head=true",
+ id="8-http-link-quoted-rel",
+ ),
+ pytest.param(
+ "https://example.com/test/9",
+ (),
+ """
+ <link
+ rel="webmention somethingelse"
+ href="https://example.com/test/9/webmention"
+ >
+ """,
+ "https://example.com/test/9/webmention",
+ id="9-multiple-html-rel-values",
+ ),
+ pytest.param(
+ "https://example.com/test/10",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/10/webmention?head=true>; "
+ 'rel="webmention somethingelse"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/10/webmention?head=true",
+ id="10-multiple-link-header-rel-values",
+ ),
+ pytest.param(
+ "https://example.com/test/11",
+ (("Link", '</test/11/webmention>; rel="webmention"'),),
+ """
+ <link
+ rel="webmention"
+ href="/test/11/webmention/error"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/11/webmention/error"
+ >
+ Wrong endpoint
+ </a>
+ """,
+ "https://example.com/test/11/webmention",
+ id="11-http-link-precedence",
+ ),
+ pytest.param(
+ "https://example.com/test/12",
+ (),
+ """
+ <link
+ rel="not-webmention"
+ href="/test/12/webmention/error"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/12/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/12/webmention",
+ id="12-exact-rel-match",
+ ),
+ pytest.param(
+ "https://example.com/test/13",
+ (),
+ """
+ <!--
+ <a
+ rel="webmention"
+ href="/test/13/webmention/error"
+ >
+ False endpoint
+ </a>
+ -->
+
+ <a
+ rel="webmention"
+ href="/test/13/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/13/webmention",
+ id="13-endpoint-inside-html-comment",
+ ),
+ pytest.param(
+ "https://example.com/test/14",
+ (),
+ """
+ <code>
+ &lt;a
+ rel="webmention"
+ href="/test/14/webmention/error"
+ &gt;
+ False endpoint
+ &lt;/a&gt;
+ </code>
+
+ <a
+ rel="webmention"
+ href="/test/14/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/14/webmention",
+ id="14-endpoint-in-escaped-html",
+ ),
+ pytest.param(
+ "https://example.com/test/15",
+ (),
+ """
+ <link
+ rel="webmention"
+ href=""
+ >
+ """,
+ "https://example.com/test/15",
+ id="15-empty-href",
+ ),
+ pytest.param(
+ "https://example.com/test/16",
+ (),
+ """
+ <a
+ rel="webmention"
+ href="/test/16/webmention"
+ >
+ First endpoint
+ </a>
+
+ <link
+ rel="webmention"
+ href="/test/16/webmention/error"
+ >
+ """,
+ "https://example.com/test/16/webmention",
+ id="16-a-before-link",
+ ),
+ pytest.param(
+ "https://example.com/test/17",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/17/webmention"
+ >
+
+ <a
+ rel="webmention"
+ href="/test/17/webmention/error"
+ >
+ Second endpoint
+ </a>
+ """,
+ "https://example.com/test/17/webmention",
+ id="17-link-before-a",
+ ),
+ pytest.param(
+ "https://example.com/test/18",
+ (
+ ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'),
+ (
+ "Link",
+ (
+ "<https://example.com/test/18/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/18/webmention?head=true",
+ id="18-multiple-http-link-headers",
+ ),
+ pytest.param(
+ "https://example.com/test/19",
+ (
+ (
+ "Link",
+ (
+ "<https://example.com/test/19/webmention/error>; "
+ 'rel="other", '
+ "<https://example.com/test/19/webmention?head=true>; "
+ 'rel="webmention"'
+ ),
+ ),
+ ),
+ "",
+ "https://example.com/test/19/webmention?head=true",
+ id="19-single-header-multiple-values",
+ ),
+ pytest.param(
+ "https://example.com/test/20",
+ (),
+ """
+ <link rel="webmention">
+
+ <a
+ rel="webmention"
+ href="/test/20/webmention"
+ >
+ Correct endpoint
+ </a>
+ """,
+ "https://example.com/test/20/webmention",
+ id="20-link-without-href",
+ ),
+ pytest.param(
+ "https://example.com/test/21",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="/test/21/webmention?query=yes"
+ >
+ """,
+ "https://example.com/test/21/webmention?query=yes",
+ id="21-endpoint-query-string",
+ ),
+ pytest.param(
+ "https://example.com/test/22",
+ (),
+ """
+ <link
+ rel="webmention"
+ href="22/webmention"
+ >
+ """,
+ "https://example.com/test/22/webmention",
+ id="22-endpoint-relative-to-path",
+ ),
+ ),
+)
+def test_discovery(
+ app: Flask,
+ sender_module: ModuleType,
+ target: str,
+ headers: tuple[tuple[str, str], ...],
+ html: str,
+ expected_endpoint: str,
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200, headers=headers)
+
+ return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html)
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, target)
+
+ assert endpoint == expected_endpoint
+
+
+@pytest.mark.parametrize(
+ ("target", "responses", "expected_endpoint"),
+ (
+ pytest.param(
+ "https://example.com/test/23/page",
+ {
+ "/test/23/page": (302, {"Location": "page/redirect-key"}),
+ "/test/23/page/redirect-key": (
+ 200,
+ {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")},
+ ),
+ },
+ ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"),
+ id="relative-endpoint-after-redirect",
+ ),
+ ),
+)
+def test_discovery_redirect(
+ app: Flask,
+ sender_module: ModuleType,
+ target: str,
+ responses: dict[str, tuple[int, dict[str, str]]],
+ expected_endpoint: str,
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ try:
+ status, headers = responses[request.url.path]
+ except KeyError:
+ raise AssertionError(f"Unexpected request: {request.url}") from None
+
+ return httpx.Response(status, headers=headers)
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, target)
+
+ assert endpoint == expected_endpoint
+
+
+def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(
+ client, "https://example.com/post"
+ )
+
+ assert endpoint == "https://example.com/webmention"
+ assert [request.method for request in requests] == ["HEAD"]
+
+
+def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text="""
+ <link
+ rel="webmention"
+ href="/webmention"
+ >
+ """,
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(
+ client, "https://example.com/post"
+ )
+
+ assert endpoint == "https://example.com/webmention"
+ assert [request.method for request in requests] == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ "failure",
+ [
+ pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"),
+ pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"),
+ ],
+)
+def test_discovery_falls_back_to_get_after_head_request_error(
+ app: Flask, sender_module: ModuleType, failure: httpx.RequestError
+) -> None:
+ methods: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+
+ if request.method == "HEAD":
+ raise failure
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert methods == ["HEAD", "GET"]
+
+
+def test_discovery_falls_back_to_get_after_head_dns_failure(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ methods: list[str] = []
+
+ def request_hook(request: httpx.Request) -> None:
+ if request.method == "HEAD":
+ raise sender_module.TemporarySenderError("HEAD DNS lookup failed")
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler),
+ event_hooks={"request": [request_hook]},
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert methods == ["GET"]
+
+
+def test_discovery_get_error_after_head_error_is_not_suppressed(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ methods: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ methods.append(request.method)
+ raise httpx.ConnectError(f"{request.method} failed")
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ pytest.raises(httpx.ConnectError, match="GET failed"),
+ ):
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert methods == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ "invalid_href",
+ [
+ pytest.param("javascript:alert(1)", id="unsupported-scheme"),
+ pytest.param("http://[invalid", id="malformed-url"),
+ ],
+)
+def test_discovery_skips_invalid_html_candidate(
+ app: Flask, sender_module: ModuleType, invalid_href: str
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text=(
+ f'<link rel="webmention" href="{invalid_href}">'
+ '<a rel="webmention" href="/webmention">Valid endpoint</a>'
+ ),
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+
+
+def test_discovery_skips_invalid_link_header_candidate(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ headers=[
+ ("Link", "<mailto:invalid@example.com>; rel=webmention"),
+ ("Link", "<http://[invalid>; rel=webmention"),
+ ("Link", "</webmention>; rel=webmention"),
+ ],
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+
+
+def test_discovery_returns_none_when_all_html_candidates_are_invalid(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"})
+
+ return httpx.Response(
+ 200,
+ headers={"Content-Type": "text/html"},
+ text='<link rel="webmention" href="javascript:void(0)">',
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint is None
+
+
+def test_resolve_endpoint_rejects_invalid_url(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ is_http_url = Mock(return_value=False)
+ monkeypatch.setattr(discovery, "is_http_url", is_http_url)
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+ with pytest.raises(
+ sender_module.PermanentSenderError, match="Invalid Webmention endpoint"
+ ):
+ sender_module.resolve_endpoint(response, "/webmention")
+ is_http_url.assert_called_once_with(ENDPOINT_URL)
+
+
+def test_endpoint_from_headers_ignores_malformed_link(
+ sender_module: ModuleType,
+) -> None:
+ response = httpx.Response(
+ 200,
+ headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")],
+ request=httpx.Request("GET", TARGET_URL),
+ )
+ assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL
+
+
+def test_endpoint_from_html_accepts_string_rel(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ element = Mock()
+ element.get.side_effect = lambda name: {
+ "rel": "webmention alternate",
+ "href": "/webmention",
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [element]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL
+
+
+def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href(
+ sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ missing_rel = Mock()
+ missing_rel.get.return_value = None
+
+ invalid_href = Mock()
+ invalid_href.get.side_effect = lambda name: {
+ "rel": ["webmention"],
+ "href": ["invalid"],
+ }[name]
+
+ document = Mock()
+ document.find_all.return_value = [missing_rel, invalid_href]
+
+ beautiful_soup = Mock(return_value=document)
+ monkeypatch.setattr(discovery, "BeautifulSoup", beautiful_soup)
+
+ response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
+
+ assert sender_module.endpoint_from_html(response, b"") is None
+
+
+def test_read_target_body_rejects_declared_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["MAX_TARGET_BYTES"] = 10
+
+ response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_read_target_body_ignores_invalid_content_length(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ response = httpx.Response(
+ 200, headers={"Content-Length": "invalid"}, content=b"Document"
+ )
+
+ with app.app_context():
+ body = sender_module.read_target_body(response)
+
+ assert body == b"Document"
+
+
+def test_read_target_body_rejects_streamed_oversized_body(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ app.config["MAX_TARGET_BYTES"] = 10
+
+ class BodyStream(httpx.SyncByteStream):
+ def __iter__(self):
+ yield b"x" * 6
+ yield b"x" * 6
+
+ response = httpx.Response(200, stream=BodyStream())
+
+ with (
+ app.app_context(),
+ pytest.raises(
+ sender_module.PermanentSenderError, match="Target document is too large"
+ ),
+ ):
+ sender_module.read_target_body(response)
+
+
+def test_discovery_uses_link_header_from_get(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
+
+ with (
+ app.app_context(),
+ httpx.Client(
+ transport=httpx.MockTransport(handler), follow_redirects=True
+ ) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint == ENDPOINT_URL
+ assert [request.method for request in requests] == ["HEAD", "GET"]
+
+
+@pytest.mark.parametrize(
+ ("status_code", "exception_name"),
+ [
+ pytest.param(503, "TemporarySenderError", id="temporary"),
+ pytest.param(404, "PermanentSenderError", id="permanent"),
+ ],
+)
+def test_discovery_maps_get_status_to_exception(
+ app: Flask, sender_module: ModuleType, status_code: int, exception_name: str
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(status_code)
+
+ exception_type = getattr(sender_module, exception_name)
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"),
+ ):
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+
+def test_discovery_returns_none_for_unsupported_media_type(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "HEAD":
+ return httpx.Response(200)
+
+ return httpx.Response(
+ 200, headers={"Content-Type": "application/json"}, content=b"{}"
+ )
+
+ with (
+ app.app_context(),
+ httpx.Client(transport=httpx.MockTransport(handler)) as client,
+ ):
+ endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+
+ assert endpoint is None
+
+
+@pytest.mark.parametrize(
+ ("status_code", "location", "expected_status_url"),
+ [
+ pytest.param(
+ 201, "/webmention/status/123", STATUS_URL, id="created-with-location"
+ ),
+ pytest.param(202, None, None, id="accepted"),
+ ],
+)
+def test_post_webmention(
+ sender_module: ModuleType,
+ status_code: int,
+ location: str | None,
+ expected_status_url: str | None,
+) -> None:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+
+ headers = {}
+
+ if location is not None:
+ headers["Location"] = location
+
+ return httpx.Response(status_code, headers=headers)
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ result = sender_module.post_webmention(
+ client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
+ )
+
+ assert result == (status_code, expected_status_url)
+ assert len(requests) == 1
+
+ request = requests[0]
+
+ assert request.method == "POST"
+ assert str(request.url) == ENDPOINT_URL
+
+ form = httpx.QueryParams(request.content.decode())
+
+ assert form["source"] == SOURCE_URL
+ assert form["target"] == TARGET_URL
+
+
+@pytest.mark.parametrize(
+ ("code", "location", "expected"),
+ [
+ (201, "/webmention/status/123", "https://example.com/webmention/status/123"),
+ (201, "https://status.example.org/123", "https://status.example.org/123"),
+ (201, "status/123", "https://example.com/status/123"),
+ (201, "javascript:alert(1)", None),
+ (201, "data:text/html,evil", None),
+ (201, "ftp://example.com/status", None),
+ (201, "http://[invalid", None),
+ (202, "https://example.com/status", None),
+ ],
+)
+def test_post_webmention_validates_location(
+ sender_module: ModuleType, code: int, location: str, expected: str | None
+) -> None:
+ def handler(_request: httpx.Request) -> httpx.Response:
+ return httpx.Response(code, headers={"Location": location})
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ result = sender_module.post_webmention(
+ client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
+ )
+
+ assert result == (code, expected)
diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py
index 108e8a6..c733dce 100644
--- a/tests/tasks/test_json_serializer.py
+++ b/tests/tasks/test_json_serializer.py
@@ -10,7 +10,7 @@ from datetime import UTC, datetime, timedelta
import pytest
from flask import Flask
-from huey.utils import Error, SKIPPED
+from huey.utils import SKIPPED, Error
from webmentions_ssg.tasks.extension import Huey
from webmentions_ssg.tasks.json_serializer import JSONSerializer
diff --git a/tests/tasks/test_mentions.py b/tests/tasks/test_mentions.py
new file mode 100644
index 0000000..a5da56c
--- /dev/null
+++ b/tests/tasks/test_mentions.py
@@ -0,0 +1,176 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import ModuleType
+from unittest.mock import Mock, patch
+
+import pytest
+from flask import Flask
+
+SOURCE_URL = "https://source.example/article"
+TARGET_URL = "https://dennisfink.me/blog/example/"
+
+
+@pytest.fixture
+def receiver(app: Flask) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import mentions
+
+ return mentions
+
+
+@pytest.mark.parametrize(
+ ("body", "expected"),
+ [
+ pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"),
+ pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"),
+ pytest.param(
+ f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href"
+ ),
+ pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"),
+ pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"),
+ pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"),
+ pytest.param(
+ f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src"
+ ),
+ pytest.param(
+ f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src"
+ ),
+ pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"),
+ pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"),
+ pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"),
+ pytest.param(
+ f'<video><track src="{TARGET_URL}"></video>', True, id="track-src"
+ ),
+ pytest.param(
+ f'<input type="image" src="{TARGET_URL}" alt="">',
+ True,
+ id="image-input-src",
+ ),
+ pytest.param(
+ f'<input type="IMAGE" src="{TARGET_URL}" alt="">',
+ True,
+ id="image-input-case-insensitive",
+ ),
+ pytest.param(
+ f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>',
+ True,
+ id="blockquote-cite",
+ ),
+ pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"),
+ pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"),
+ pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"),
+ pytest.param(
+ '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>',
+ True,
+ id="base-url",
+ ),
+ pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"),
+ pytest.param(
+ f'<blockquote src="{TARGET_URL}">Quote</blockquote>',
+ False,
+ id="blockquote-src-invalid",
+ ),
+ pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"),
+ pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"),
+ pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"),
+ pytest.param(
+ f'<input type="text" src="{TARGET_URL}">',
+ False,
+ id="text-input-src-invalid",
+ ),
+ pytest.param(
+ f'<picture><source src="{TARGET_URL}"></picture>',
+ False,
+ id="picture-source-src-invalid",
+ ),
+ pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"),
+ pytest.param(
+ f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url"
+ ),
+ pytest.param(
+ f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">',
+ False,
+ id="invalid-cite-does-not-override-valid-src",
+ ),
+ pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"),
+ pytest.param(
+ '<a href="https://example.com/">Other site</a>', False, id="missing-target"
+ ),
+ ],
+)
+def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None:
+ assert (
+ receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected
+ )
+
+
+def test_html_mentions_relative_target(receiver: ModuleType) -> None:
+ assert receiver.html_mentions_target(
+ b'<a href="../target/">Reply</a>',
+ "https://source.example/posts/article/",
+ "https://source.example/posts/target/",
+ )
+
+
+@pytest.mark.parametrize(
+ ("body", "target_url", "expected"),
+ [
+ (TARGET_URL, TARGET_URL, True),
+ (f"This post replies to {TARGET_URL}", TARGET_URL, True),
+ (
+ f"https://example.com/first {TARGET_URL} https://example.com/last",
+ TARGET_URL,
+ True,
+ ),
+ pytest.param(
+ f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma"
+ ),
+ pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"),
+ pytest.param(
+ f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation"
+ ),
+ pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"),
+ pytest.param(
+ "https://example.com/wiki/Foo_(bar)",
+ "https://example.com/wiki/Foo_(bar)",
+ True,
+ id="balanced-parentheses-in-url",
+ ),
+ pytest.param(
+ "https://example.com/wiki/Foo_(bar)",
+ "https://example.com/wiki/Foo_(bar",
+ False,
+ id="balanced-parentheses-not-trimmed",
+ ),
+ (f"{TARGET_URL}more", TARGET_URL, False),
+ pytest.param(
+ f"{TARGET_URL},another-page",
+ TARGET_URL,
+ False,
+ id="comma-within-longer-url",
+ ),
+ ("https://dennisfink.me/blog/other/", TARGET_URL, False),
+ ("/blog/example/", TARGET_URL, False),
+ ("There are no links here.", TARGET_URL, False),
+ ],
+)
+def test_text_mentions_target(
+ receiver: ModuleType, body: str, target_url: str, expected: bool
+) -> None:
+ assert receiver.text_mentions_target(body, target_url) is expected
+
+
+@patch("webmentions_ssg.tasks.mentions.BeautifulSoup")
+def test_html_mentions_target_ignores_non_string_attribute(
+ beautiful_soup: Mock, receiver: ModuleType
+) -> None:
+ document = beautiful_soup.return_value
+ document.select_one.return_value = None
+
+ element = Mock()
+ element.get.return_value = ["not", "a", "string"]
+ document.select.return_value = [element]
+
+ assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL)
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 44d1cc2..5441eb7 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -15,7 +15,7 @@ from flask import Flask
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import ReceivedWebmention
-from webmentions_ssg.url_security import AddressResolutionError
+from webmentions_ssg.net import AddressResolutionError
SOURCE_URL = "https://source.example/article"
TARGET_URL = "https://dennisfink.me/blog/example/"
@@ -68,148 +68,6 @@ def set_stream_response(httpx_client: Mock, response: httpx.Response) -> Mock:
return client
-@pytest.mark.parametrize(
- ("body", "expected"),
- [
- pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"),
- pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"),
- pytest.param(
- f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href"
- ),
- pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"),
- pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"),
- pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"),
- pytest.param(
- f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src"
- ),
- pytest.param(
- f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src"
- ),
- pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"),
- pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"),
- pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"),
- pytest.param(
- f'<video><track src="{TARGET_URL}"></video>', True, id="track-src"
- ),
- pytest.param(
- f'<input type="image" src="{TARGET_URL}" alt="">',
- True,
- id="image-input-src",
- ),
- pytest.param(
- f'<input type="IMAGE" src="{TARGET_URL}" alt="">',
- True,
- id="image-input-case-insensitive",
- ),
- pytest.param(
- f'<blockquote cite="{TARGET_URL}">Quotation</blockquote>',
- True,
- id="blockquote-cite",
- ),
- pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"),
- pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"),
- pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"),
- pytest.param(
- '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>',
- True,
- id="base-url",
- ),
- pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"),
- pytest.param(
- f'<blockquote src="{TARGET_URL}">Quote</blockquote>',
- False,
- id="blockquote-src-invalid",
- ),
- pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"),
- pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"),
- pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"),
- pytest.param(
- f'<input type="text" src="{TARGET_URL}">',
- False,
- id="text-input-src-invalid",
- ),
- pytest.param(
- f'<picture><source src="{TARGET_URL}"></picture>',
- False,
- id="picture-source-src-invalid",
- ),
- pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"),
- pytest.param(
- f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url"
- ),
- pytest.param(
- f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">',
- False,
- id="invalid-cite-does-not-override-valid-src",
- ),
- pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"),
- pytest.param(
- '<a href="https://example.com/">Other site</a>', False, id="missing-target"
- ),
- ],
-)
-def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None:
- assert (
- receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected
- )
-
-
-def test_html_mentions_relative_target(receiver: ModuleType) -> None:
- assert receiver.html_mentions_target(
- b'<a href="../target/">Reply</a>',
- "https://source.example/posts/article/",
- "https://source.example/posts/target/",
- )
-
-
-@pytest.mark.parametrize(
- ("body", "target_url", "expected"),
- [
- (TARGET_URL, TARGET_URL, True),
- (f"This post replies to {TARGET_URL}", TARGET_URL, True),
- (
- f"https://example.com/first {TARGET_URL} https://example.com/last",
- TARGET_URL,
- True,
- ),
- pytest.param(
- f"See {TARGET_URL}, thanks", TARGET_URL, True, id="trailing-comma"
- ),
- pytest.param(f"({TARGET_URL})", TARGET_URL, True, id="parenthesized-url"),
- pytest.param(
- f"See ({TARGET_URL}), thanks!", TARGET_URL, True, id="mixed-punctuation"
- ),
- pytest.param(f"See {TARGET_URL}. Next", TARGET_URL, True, id="trailing-period"),
- pytest.param(
- "https://example.com/wiki/Foo_(bar)",
- "https://example.com/wiki/Foo_(bar)",
- True,
- id="balanced-parentheses-in-url",
- ),
- pytest.param(
- "https://example.com/wiki/Foo_(bar)",
- "https://example.com/wiki/Foo_(bar",
- False,
- id="balanced-parentheses-not-trimmed",
- ),
- (f"{TARGET_URL}more", TARGET_URL, False),
- pytest.param(
- f"{TARGET_URL},another-page",
- TARGET_URL,
- False,
- id="comma-within-longer-url",
- ),
- ("https://dennisfink.me/blog/other/", TARGET_URL, False),
- ("/blog/example/", TARGET_URL, False),
- ("There are no links here.", TARGET_URL, False),
- ],
-)
-def test_text_mentions_target(
- receiver: ModuleType, body: str, target_url: str, expected: bool
-) -> None:
- assert receiver.text_mentions_target(body, target_url) is expected
-
-
@patch("webmentions_ssg.tasks.receiver.httpx.Client")
def test_fetch_source_returns_response_and_body(
httpx_client: Mock, app: Flask, receiver: ModuleType
@@ -786,7 +644,7 @@ def test_verify_webmention_ignores_unknown_identifier(
def test_fetch_source_rejects_unsafe_destination(
httpx_client: Mock, app: Flask, receiver: ModuleType
) -> None:
- from webmentions_ssg.url_security import UnsafeAddressError
+ from webmentions_ssg.net import UnsafeAddressError
client = httpx_client.return_value.__enter__.return_value
client.stream.side_effect = UnsafeAddressError("Local address")
@@ -810,20 +668,6 @@ def test_fetch_source_treats_dns_failure_as_temporary(
receiver.fetch_source(SOURCE_URL)
-@patch("webmentions_ssg.tasks.receiver.BeautifulSoup")
-def test_html_mentions_target_ignores_non_string_attribute(
- beautiful_soup: Mock, receiver: ModuleType
-) -> None:
- document = beautiful_soup.return_value
- document.select_one.return_value = None
-
- element = Mock()
- element.get.return_value = ["not", "a", "string"]
- document.select.return_value = [element]
-
- assert not receiver.html_mentions_target(b"<html></html>", SOURCE_URL, TARGET_URL)
-
-
@patch("webmentions_ssg.tasks.receiver.httpx.Client")
def test_fetch_source_ignores_invalid_content_length(
httpx_client: Mock, app: Flask, receiver: ModuleType
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 26e81c4..7eb970c 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -18,9 +18,9 @@ from pytest import MonkeyPatch
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
+from webmentions_ssg.tasks import source_parser
-BASE_URL = "https://dennisfink.me/blog/"
-SOURCE_URL = f"{BASE_URL}example/"
+from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post
@pytest.fixture
@@ -29,6 +29,7 @@ def scanner_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
from webmentions_ssg.tasks import scanner
monkeypatch.setattr(scanner, "is_http_url", Mock(return_value=True))
+ monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True))
return scanner
@@ -80,764 +81,6 @@ def test_manual_scan_propagates_unrelated_errors(
scanner_module.manual_scan_sources.call_local()
-def parse_html(html: str) -> BeautifulSoup:
- return BeautifulSoup(html, "html5lib")
-
-
-def write_post(
- root: Path,
- slug: str,
- content: str,
- *,
- canonical: str | None = None,
- entry_url: str | None = None,
-) -> Path:
- directory = root / slug
- directory.mkdir(parents=True, exist_ok=True)
-
- source_url = f"{BASE_URL}{slug}/"
-
- if canonical is None:
- canonical = source_url
-
- if entry_url is None:
- entry_url = source_url
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{canonical}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{entry_url}"
- >
- Permalink
- </a>
-
- <div class="e-content">
- {content}
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- return path
-
-
-def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None:
- app.config["SOURCE_DIRECTORY"] = str(root)
- app.config["SOURCE_BASE_URL"] = base_url
-
-
-def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- entry = scanner_module.parse_entry(element, SOURCE_URL)
-
- assert "h-entry" in entry["type"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article>
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find("article")
-
- assert element is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one parsed h-entry"
- ):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry" id="source">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <p class="p-name">
- Example post
- </p>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element, entry = scanner_module.primary_entry(document, SOURCE_URL)
-
- assert element["id"] == "source"
- assert entry["properties"]["name"] == ["Example post"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <main>
- <p>No microformats here.</p>
- </main>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- First
- </a>
- </article>
-
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Second
- </a>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/nested/"
- >
- Nested
- </a>
- </article>
- </div>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Other
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_content_element_returns_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content" id="content">
- Hello world
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- content = scanner_module.content_element(entry)
-
- assert content["id"] == "content"
-
-
-def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 0"
- ):
- scanner_module.content_element(entry)
-
-
-def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- First
- </div>
-
- <div class="e-content">
- Second
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 2"
- ):
- scanner_module.content_element(entry)
-
-
-def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result == SOURCE_URL
-
-
-def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <title>Example</title>
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result is None
-
-
-def test_canonical_url_resolves_relative_url_with_base_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.side_effect = [False, True]
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
- assert result == ("https://dennisfink.me/blog/example/canonical/")
-
-
-def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="relative canonical"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
-
-def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href=" "
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_scan_source_uses_canonical_without_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
- assert len(scanned.content_hash) == 32
-
-
-def test_scan_source_ignores_markup_noise_in_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- "<p>Hello <strong>world</strong>!</p>"
- '<a href="https://example.com/target">Target</a>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- write_post(
- tmp_path,
- "example",
- '<p class="generated" data-build="123"> Hello world! </p>'
- '<a class="external" href="https://example.com/target"> Target </a>',
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_uses_stable_time_value_inside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_hash_changes_when_image_changes(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_includes_reaction_targets_outside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- path.write_text(
- html.replace(
- '<div class="e-content">',
- '<a class="u-like-of" href="https://example.com/first">Like</a>'
- '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- "https://example.com/first", "https://example.com/second"
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == frozenset({"https://example.com/first"})
- assert second.targets == frozenset({"https://example.com/second"})
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_ignores_reaction_target_order(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
- second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
- path.write_text(
- html.replace(
- '<div class="e-content">',
- first_link + second_link + '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- first_link + second_link, second_link + first_link
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_falls_back_to_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <a href="https://example.com/target">
- Target
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL)
-
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-def test_scan_source_requires_canonical_or_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <p>Hello world.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"):
- scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
-
-def test_scan_source_extracts_only_content_links(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a href="https://example.com/metadata">
- Metadata link
- </a>
-
- <div class="e-content">
- <a href="https://example.com/content">
- Content link
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/content"})
-
-
-def test_scan_source_extracts_relative_content_link(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="../other/">
- Other post
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"})
-
-
-@pytest.mark.parametrize(
- "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-)
-def test_scan_source_extracts_reaction_properties(
- scanner_module: ModuleType, tmp_path: Path, property_name: str
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- target = f"https://example.com/{property_name}"
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a
- class="u-{property_name}"
- href="{target}"
- >
- Reaction target
- </a>
-
- <div class="e-content">
- <p>Post content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({target})
-
-
-@pytest.mark.parametrize("tag", ("link", "area"))
-def test_scan_source_ignores_non_anchor_href_elements(
- scanner_module: ModuleType, tmp_path: Path, tag: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <{tag} href="https://example.com/target">
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset()
-
-
-def test_scan_source_ignores_self_fragment(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="#section">
- Section
- </a>
-
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_creates_source_and_webmentions(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1885,23 +1128,23 @@ def test_unexpected_parser_error_does_not_abort_other_sources(
write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>')
if parser == "bs4":
- original_soup = scanner_module.BeautifulSoup
+ original_soup = source_parser.BeautifulSoup
def failing_soup(markup: bytes, features: str) -> BeautifulSoup:
if b"/bad/" in markup:
raise failure
return original_soup(markup, features)
- monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup)
+ monkeypatch.setattr(source_parser, "BeautifulSoup", failing_soup)
else:
- original_parse = scanner_module.mf2py.parse
+ original_parse = source_parser.mf2py.parse
def failing_parse(*, doc: str, url: str) -> dict[str, Any]:
if "/bad/" in doc:
raise failure
return original_parse(doc=doc, url=url)
- monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse)
+ monkeypatch.setattr(source_parser.mf2py, "parse", failing_parse)
queued: list[UUID] = []
monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
@@ -1979,46 +1222,6 @@ def test_unexpected_persistence_error_is_not_suppressed(
run_scan(app, scanner_module)
-def test_scan_source_rejects_symlink_outside_root_before_read(
- scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- external = write_post(tmp_path / "outside", "private", "Sensitive content")
- linked = root / "escaped" / "index.html"
- linked.parent.mkdir()
- linked.symlink_to(external)
-
- read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
- monkeypatch.setattr(Path, "read_bytes", read_bytes)
-
- with pytest.raises(
- scanner_module.SourceScanError, match="escapes source directory"
- ):
- scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- read_bytes.assert_not_called()
-
-
-def test_scan_source_accepts_symlink_within_root(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- linked = write_post(
- root, "example", '<a href="https://example.com/target">Target</a>'
- )
- shared = root / "shared.html"
- linked.rename(shared)
- linked.symlink_to(shared)
-
- scanned = scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_sources_skips_symlink_outside_root(
app: Flask,
scanner_module: ModuleType,
@@ -2043,16 +1246,6 @@ def test_scan_sources_skips_symlink_outside_root(
assert "escapes source directory" in caplog.text
-def test_scan_sources_rejects_malformed_runtime_ignored_hostnames(
- app: Flask, scanner_module: ModuleType, tmp_path: Path
-) -> None:
- configure_scanner(app, tmp_path)
- app.config["IGNORED_HOSTNAMES"] = "example.com"
-
- with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"):
- run_scan(app, scanner_module)
-
-
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -2104,220 +1297,6 @@ def test_scan_sources_rejects_invalid_base_url(
run_scan(app, scanner_module)
-@pytest.mark.parametrize(
- "href",
- (
- "https://dennisfink.me/blog/other/",
- "https://www.dennisfink.me/blog/other/",
- "https://webmentions.dennisfink.me/endpoint",
- "https://foo.bar.dennisfink.me/example",
- ),
-)
-def test_scan_source_ignores_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Ignored
- </a>
-
- <a href="https://example.com/target">
- External
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-@pytest.mark.parametrize(
- "href",
- (
- "https://notdennisfink.me/example",
- "https://dennisfink.me.example.com/example",
- "https://example.com/example",
- ),
-)
-def test_scan_source_keeps_nonmatching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({href})
-
-
-def test_scan_source_ignores_reaction_to_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{SOURCE_URL}"
- >
- Permalink
- </a>
-
- <a
- class="u-in-reply-to"
- href="https://www.dennisfink.me/blog/other/"
- >
- Reply target
- </a>
-
- <div class="e-content">
- <p>Reply content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset()
-
-
-def test_canonical_url_rejects_non_http_resolved_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="mailto:example@example.com"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError,
- match="Canonical URL is not a valid HTTP or HTTPS URL",
- ):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_property_urls_ignores_non_mapping_properties(
- scanner_module: ModuleType,
-) -> None:
- entry = {"properties": "invalid"}
-
- assert tuple(scanner_module.property_urls(entry, "url")) == ()
-
-
-@pytest.mark.parametrize(
- ("parsed", "message"),
- [
- pytest.param(
- [],
- "Microformats parser did not return a document object",
- id="non-object-document",
- ),
- pytest.param(
- {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
- ),
- pytest.param(
- {"items": [{"type": ["h-card"]}]},
- "Parsed microformats item is not an h-entry",
- id="wrong-item-type",
- ),
- ],
-)
-def test_parse_entry_rejects_invalid_parser_output(
- scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str
-) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed)
-
- with pytest.raises(scanner_module.SourceScanError, match=message):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_normalize_target_ignores_empty_target(scanner_module: ModuleType) -> None:
- assert (
- scanner_module.normalize_target(" ", base_url=SOURCE_URL, source_url=SOURCE_URL)
- is None
- )
-
-
-def test_normalize_target_ignores_url_rejected_by_validator(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- assert (
- scanner_module.normalize_target(
- "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL
- )
- is None
- )
-
-
def test_changed_source_url_is_rejected(
app: Flask,
scanner_module: ModuleType,
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index 7ebe083..c2f6e02 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -14,7 +14,6 @@ from flask import Flask
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
-from webmentions_ssg.url_security import AddressResolutionError
SOURCE_URL = "https://dennisfink.me/blog/example/"
TARGET_URL = "https://example.com/post"
@@ -101,25 +100,6 @@ def sender_module(app: Flask) -> ModuleType:
@pytest.mark.parametrize(
- ("status", "expected"),
- [
- pytest.param(200, False, id="success"),
- pytest.param(400, False, id="client-error"),
- pytest.param(408, True, id="request-timeout"),
- pytest.param(425, True, id="too-early"),
- pytest.param(429, True, id="too-many-requests"),
- pytest.param(500, True, id="server-error-lower-bound"),
- pytest.param(599, True, id="server-error-upper-bound"),
- pytest.param(600, False, id="outside-server-error-range"),
- ],
-)
-def test_temporary_http_status(
- sender_module: ModuleType, status: int, expected: bool
-) -> None:
- assert sender_module.temporary_http_status(status) is expected
-
-
-@pytest.mark.parametrize(
("desired_revision", "processed_revision", "revision", "expected"),
[
pytest.param(2, None, 2, True, id="unprocessed-current"),
@@ -147,671 +127,6 @@ def test_attempt_is_current(
assert sender_module.attempt_is_current(webmention, revision) is expected
-@pytest.mark.parametrize(
- ("value", "expected"),
- (
- (
- '<https://example.com/webmention>; rel="webmention"',
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- "<https://example.com/webmention>; rel=webmention",
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- '<https://example.com/webmention>; rel="webmention alternate"',
- ("https://example.com/webmention", {"webmention", "alternate"}),
- ),
- (
- '<https://example.com/webmention>; REL="WebMention Alternate"',
- ("https://example.com/webmention", {"webmention", "alternate"}),
- ),
- (
- (
- '<https://example.com/webmention>; type="text/html"; '
- 'rel="webmention"; title="Endpoint"'
- ),
- ("https://example.com/webmention", {"webmention"}),
- ),
- (
- '<https://example.com/stylesheet>; type="text/css"',
- ("https://example.com/stylesheet", set()),
- ),
- ),
-)
-def test_parse_link_value(
- sender_module: ModuleType, value: str, expected: tuple[str, set[str]]
-) -> None:
- assert sender_module.parse_link_value(value) == expected
-
-
-@pytest.mark.parametrize(
- "value", ("", "https://example.com/webmention", "<https://example.com/webmention")
-)
-def test_parse_link_value_rejects_malformed_value(
- sender_module: ModuleType, value: str
-) -> None:
- assert sender_module.parse_link_value(value) is None
-
-
-@pytest.mark.parametrize(
- ("target", "headers", "html", "expected_endpoint"),
- (
- pytest.param(
- "https://example.com/test/1",
- (("Link", "</test/1/webmention?head=true>; rel=webmention"),),
- "",
- "https://example.com/test/1/webmention?head=true",
- id="1-http-link-unquoted-rel-relative-url",
- ),
- pytest.param(
- "https://example.com/test/2",
- (
- (
- "Link",
- (
- "<https://example.com/test/2/webmention?head=true>;"
- "rel=webmention"
- ),
- ),
- ),
- "",
- "https://example.com/test/2/webmention?head=true",
- id="2-http-link-unquoted-rel-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/3",
- (),
- """
- <link
- rel="webmention"
- href="/test/3/webmention"
- >
- """,
- "https://example.com/test/3/webmention",
- id="3-html-link-relative-url",
- ),
- pytest.param(
- "https://example.com/test/4",
- (),
- """
- <link
- rel="webmention"
- href="https://example.com/test/4/webmention"
- >
- """,
- "https://example.com/test/4/webmention",
- id="4-html-link-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/5",
- (),
- """
- <a
- rel="webmention"
- href="/test/5/webmention"
- >
- Endpoint
- </a>
- """,
- "https://example.com/test/5/webmention",
- id="5-html-a-relative-url",
- ),
- pytest.param(
- "https://example.com/test/6",
- (),
- """
- <a
- rel="webmention"
- href="https://example.com/test/6/webmention"
- >
- Endpoint
- </a>
- """,
- "https://example.com/test/6/webmention",
- id="6-html-a-absolute-url",
- ),
- pytest.param(
- "https://example.com/test/7",
- (
- (
- "LinK",
- (
- "<https://example.com/test/7/webmention?head=true>; "
- "rel=webmention"
- ),
- ),
- ),
- "",
- "https://example.com/test/7/webmention?head=true",
- id="7-http-link-strange-casing",
- ),
- pytest.param(
- "https://example.com/test/8",
- (
- (
- "Link",
- (
- "<https://example.com/test/8/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/8/webmention?head=true",
- id="8-http-link-quoted-rel",
- ),
- pytest.param(
- "https://example.com/test/9",
- (),
- """
- <link
- rel="webmention somethingelse"
- href="https://example.com/test/9/webmention"
- >
- """,
- "https://example.com/test/9/webmention",
- id="9-multiple-html-rel-values",
- ),
- pytest.param(
- "https://example.com/test/10",
- (
- (
- "Link",
- (
- "<https://example.com/test/10/webmention?head=true>; "
- 'rel="webmention somethingelse"'
- ),
- ),
- ),
- "",
- "https://example.com/test/10/webmention?head=true",
- id="10-multiple-link-header-rel-values",
- ),
- pytest.param(
- "https://example.com/test/11",
- (("Link", '</test/11/webmention>; rel="webmention"'),),
- """
- <link
- rel="webmention"
- href="/test/11/webmention/error"
- >
-
- <a
- rel="webmention"
- href="/test/11/webmention/error"
- >
- Wrong endpoint
- </a>
- """,
- "https://example.com/test/11/webmention",
- id="11-http-link-precedence",
- ),
- pytest.param(
- "https://example.com/test/12",
- (),
- """
- <link
- rel="not-webmention"
- href="/test/12/webmention/error"
- >
-
- <a
- rel="webmention"
- href="/test/12/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/12/webmention",
- id="12-exact-rel-match",
- ),
- pytest.param(
- "https://example.com/test/13",
- (),
- """
- <!--
- <a
- rel="webmention"
- href="/test/13/webmention/error"
- >
- False endpoint
- </a>
- -->
-
- <a
- rel="webmention"
- href="/test/13/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/13/webmention",
- id="13-endpoint-inside-html-comment",
- ),
- pytest.param(
- "https://example.com/test/14",
- (),
- """
- <code>
- &lt;a
- rel="webmention"
- href="/test/14/webmention/error"
- &gt;
- False endpoint
- &lt;/a&gt;
- </code>
-
- <a
- rel="webmention"
- href="/test/14/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/14/webmention",
- id="14-endpoint-in-escaped-html",
- ),
- pytest.param(
- "https://example.com/test/15",
- (),
- """
- <link
- rel="webmention"
- href=""
- >
- """,
- "https://example.com/test/15",
- id="15-empty-href",
- ),
- pytest.param(
- "https://example.com/test/16",
- (),
- """
- <a
- rel="webmention"
- href="/test/16/webmention"
- >
- First endpoint
- </a>
-
- <link
- rel="webmention"
- href="/test/16/webmention/error"
- >
- """,
- "https://example.com/test/16/webmention",
- id="16-a-before-link",
- ),
- pytest.param(
- "https://example.com/test/17",
- (),
- """
- <link
- rel="webmention"
- href="/test/17/webmention"
- >
-
- <a
- rel="webmention"
- href="/test/17/webmention/error"
- >
- Second endpoint
- </a>
- """,
- "https://example.com/test/17/webmention",
- id="17-link-before-a",
- ),
- pytest.param(
- "https://example.com/test/18",
- (
- ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'),
- (
- "Link",
- (
- "<https://example.com/test/18/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/18/webmention?head=true",
- id="18-multiple-http-link-headers",
- ),
- pytest.param(
- "https://example.com/test/19",
- (
- (
- "Link",
- (
- "<https://example.com/test/19/webmention/error>; "
- 'rel="other", '
- "<https://example.com/test/19/webmention?head=true>; "
- 'rel="webmention"'
- ),
- ),
- ),
- "",
- "https://example.com/test/19/webmention?head=true",
- id="19-single-header-multiple-values",
- ),
- pytest.param(
- "https://example.com/test/20",
- (),
- """
- <link rel="webmention">
-
- <a
- rel="webmention"
- href="/test/20/webmention"
- >
- Correct endpoint
- </a>
- """,
- "https://example.com/test/20/webmention",
- id="20-link-without-href",
- ),
- pytest.param(
- "https://example.com/test/21",
- (),
- """
- <link
- rel="webmention"
- href="/test/21/webmention?query=yes"
- >
- """,
- "https://example.com/test/21/webmention?query=yes",
- id="21-endpoint-query-string",
- ),
- pytest.param(
- "https://example.com/test/22",
- (),
- """
- <link
- rel="webmention"
- href="22/webmention"
- >
- """,
- "https://example.com/test/22/webmention",
- id="22-endpoint-relative-to-path",
- ),
- ),
-)
-def test_discovery(
- app: Flask,
- sender_module: ModuleType,
- target: str,
- headers: tuple[tuple[str, str], ...],
- html: str,
- expected_endpoint: str,
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200, headers=headers)
-
- return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html)
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, target)
-
- assert endpoint == expected_endpoint
-
-
-@pytest.mark.parametrize(
- ("target", "responses", "expected_endpoint"),
- (
- pytest.param(
- "https://example.com/test/23/page",
- {
- "/test/23/page": (302, {"Location": "page/redirect-key"}),
- "/test/23/page/redirect-key": (
- 200,
- {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")},
- ),
- },
- ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"),
- id="relative-endpoint-after-redirect",
- ),
- ),
-)
-def test_discovery_redirect(
- app: Flask,
- sender_module: ModuleType,
- target: str,
- responses: dict[str, tuple[int, dict[str, str]]],
- expected_endpoint: str,
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- try:
- status, headers = responses[request.url.path]
- except KeyError:
- raise AssertionError(f"Unexpected request: {request.url}") from None
-
- return httpx.Response(status, headers=headers)
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, target)
-
- assert endpoint == expected_endpoint
-
-
-def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(
- client, "https://example.com/post"
- )
-
- assert endpoint == "https://example.com/webmention"
- assert [request.method for request in requests] == ["HEAD"]
-
-
-def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text="""
- <link
- rel="webmention"
- href="/webmention"
- >
- """,
- )
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(
- client, "https://example.com/post"
- )
-
- assert endpoint == "https://example.com/webmention"
- assert [request.method for request in requests] == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- "failure",
- [
- pytest.param(httpx.ConnectError("HEAD connection failed"), id="connection"),
- pytest.param(httpx.ReadTimeout("HEAD timed out"), id="timeout"),
- ],
-)
-def test_discovery_falls_back_to_get_after_head_request_error(
- app: Flask, sender_module: ModuleType, failure: httpx.RequestError
-) -> None:
- methods: list[str] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
-
- if request.method == "HEAD":
- raise failure
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert methods == ["HEAD", "GET"]
-
-
-def test_discovery_falls_back_to_get_after_head_dns_failure(
- app: Flask, sender_module: ModuleType
-) -> None:
- methods: list[str] = []
-
- def request_hook(request: httpx.Request) -> None:
- if request.method == "HEAD":
- raise sender_module.TemporarySenderError("HEAD DNS lookup failed")
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler),
- event_hooks={"request": [request_hook]},
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert methods == ["GET"]
-
-
-def test_discovery_get_error_after_head_error_is_not_suppressed(
- app: Flask, sender_module: ModuleType
-) -> None:
- methods: list[str] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- methods.append(request.method)
- raise httpx.ConnectError(f"{request.method} failed")
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- pytest.raises(httpx.ConnectError, match="GET failed"),
- ):
- sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert methods == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- "invalid_href",
- [
- pytest.param("javascript:alert(1)", id="unsupported-scheme"),
- pytest.param("http://[invalid", id="malformed-url"),
- ],
-)
-def test_discovery_skips_invalid_html_candidate(
- app: Flask, sender_module: ModuleType, invalid_href: str
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text=(
- f'<link rel="webmention" href="{invalid_href}">'
- '<a rel="webmention" href="/webmention">Valid endpoint</a>'
- ),
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
-
-
-def test_discovery_skips_invalid_link_header_candidate(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- return httpx.Response(
- 200,
- headers=[
- ("Link", "<mailto:invalid@example.com>; rel=webmention"),
- ("Link", "<http://[invalid>; rel=webmention"),
- ("Link", "</webmention>; rel=webmention"),
- ],
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
-
-
-def test_discovery_returns_none_when_all_html_candidates_are_invalid(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200, headers={"Link": "<mailto:bad>; rel=webmention"})
-
- return httpx.Response(
- 200,
- headers={"Content-Type": "text/html"},
- text='<link rel="webmention" href="javascript:void(0)">',
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint is None
-
-
def test_send_webmention_persists_success(
app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -1315,267 +630,6 @@ def test_send_webmention_ignores_unknown_identifier(
assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text
-def test_resolve_endpoint_rejects_invalid_url(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- is_http_url = Mock(return_value=False)
- monkeypatch.setattr(sender_module, "is_http_url", is_http_url)
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
- with pytest.raises(
- sender_module.PermanentSenderError, match="Invalid Webmention endpoint"
- ):
- sender_module.resolve_endpoint(response, "/webmention")
- is_http_url.assert_called_once_with(ENDPOINT_URL)
-
-
-def test_endpoint_from_headers_ignores_malformed_link(
- sender_module: ModuleType,
-) -> None:
- response = httpx.Response(
- 200,
- headers=[("Link", "not-a-link"), ("Link", "</webmention>; rel=webmention")],
- request=httpx.Request("GET", TARGET_URL),
- )
- assert sender_module.endpoint_from_headers(response) == ENDPOINT_URL
-
-
-def test_endpoint_from_html_accepts_string_rel(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- element = Mock()
- element.get.side_effect = lambda name: {
- "rel": "webmention alternate",
- "href": "/webmention",
- }[name]
-
- document = Mock()
- document.find_all.return_value = [element]
-
- beautiful_soup = Mock(return_value=document)
- monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
-
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
-
- assert sender_module.endpoint_from_html(response, b"") == ENDPOINT_URL
-
-
-def test_endpoint_from_html_ignores_elements_without_rel_and_non_string_href(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- missing_rel = Mock()
- missing_rel.get.return_value = None
-
- invalid_href = Mock()
- invalid_href.get.side_effect = lambda name: {
- "rel": ["webmention"],
- "href": ["invalid"],
- }[name]
-
- document = Mock()
- document.find_all.return_value = [missing_rel, invalid_href]
-
- beautiful_soup = Mock(return_value=document)
- monkeypatch.setattr(sender_module, "BeautifulSoup", beautiful_soup)
-
- response = httpx.Response(200, request=httpx.Request("GET", TARGET_URL))
-
- assert sender_module.endpoint_from_html(response, b"") is None
-
-
-def test_read_target_body_rejects_declared_oversized_body(
- app: Flask, sender_module: ModuleType
-) -> None:
- app.config["MAX_TARGET_BYTES"] = 10
-
- response = httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
-
- with (
- app.app_context(),
- pytest.raises(
- sender_module.PermanentSenderError, match="Target document is too large"
- ),
- ):
- sender_module.read_target_body(response)
-
-
-def test_read_target_body_ignores_invalid_content_length(
- app: Flask, sender_module: ModuleType
-) -> None:
- response = httpx.Response(
- 200, headers={"Content-Length": "invalid"}, content=b"Document"
- )
-
- with app.app_context():
- body = sender_module.read_target_body(response)
-
- assert body == b"Document"
-
-
-def test_read_target_body_rejects_streamed_oversized_body(
- app: Flask, sender_module: ModuleType
-) -> None:
- app.config["MAX_TARGET_BYTES"] = 10
-
- class BodyStream(httpx.SyncByteStream):
- def __iter__(self):
- yield b"x" * 6
- yield b"x" * 6
-
- response = httpx.Response(200, stream=BodyStream())
-
- with (
- app.app_context(),
- pytest.raises(
- sender_module.PermanentSenderError, match="Target document is too large"
- ),
- ):
- sender_module.read_target_body(response)
-
-
-def test_discovery_uses_link_header_from_get(
- app: Flask, sender_module: ModuleType
-) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"})
-
- with (
- app.app_context(),
- httpx.Client(
- transport=httpx.MockTransport(handler), follow_redirects=True
- ) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint == ENDPOINT_URL
- assert [request.method for request in requests] == ["HEAD", "GET"]
-
-
-@pytest.mark.parametrize(
- ("status_code", "exception_name"),
- [
- pytest.param(503, "TemporarySenderError", id="temporary"),
- pytest.param(404, "PermanentSenderError", id="permanent"),
- ],
-)
-def test_discovery_maps_get_status_to_exception(
- app: Flask, sender_module: ModuleType, status_code: int, exception_name: str
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(status_code)
-
- exception_type = getattr(sender_module, exception_name)
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- pytest.raises(exception_type, match=f"Target returned HTTP {status_code}"),
- ):
- sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
-
-def test_discovery_returns_none_for_unsupported_media_type(
- app: Flask, sender_module: ModuleType
-) -> None:
- def handler(request: httpx.Request) -> httpx.Response:
- if request.method == "HEAD":
- return httpx.Response(200)
-
- return httpx.Response(
- 200, headers={"Content-Type": "application/json"}, content=b"{}"
- )
-
- with (
- app.app_context(),
- httpx.Client(transport=httpx.MockTransport(handler)) as client,
- ):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
-
- assert endpoint is None
-
-
-@pytest.mark.parametrize(
- ("status_code", "location", "expected_status_url"),
- [
- pytest.param(
- 201, "/webmention/status/123", STATUS_URL, id="created-with-location"
- ),
- pytest.param(202, None, None, id="accepted"),
- ],
-)
-def test_post_webmention(
- sender_module: ModuleType,
- status_code: int,
- location: str | None,
- expected_status_url: str | None,
-) -> None:
- requests: list[httpx.Request] = []
-
- def handler(request: httpx.Request) -> httpx.Response:
- requests.append(request)
-
- headers = {}
-
- if location is not None:
- headers["Location"] = location
-
- return httpx.Response(status_code, headers=headers)
-
- with httpx.Client(transport=httpx.MockTransport(handler)) as client:
- result = sender_module.post_webmention(
- client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
- )
-
- assert result == (status_code, expected_status_url)
- assert len(requests) == 1
-
- request = requests[0]
-
- assert request.method == "POST"
- assert str(request.url) == ENDPOINT_URL
-
- form = httpx.QueryParams(request.content.decode())
-
- assert form["source"] == SOURCE_URL
- assert form["target"] == TARGET_URL
-
-
-@pytest.mark.parametrize(
- ("code", "location", "expected"),
- [
- (201, "/webmention/status/123", "https://example.com/webmention/status/123"),
- (201, "https://status.example.org/123", "https://status.example.org/123"),
- (201, "status/123", "https://example.com/status/123"),
- (201, "javascript:alert(1)", None),
- (201, "data:text/html,evil", None),
- (201, "ftp://example.com/status", None),
- (201, "http://[invalid", None),
- (202, "https://example.com/status", None),
- ],
-)
-def test_post_webmention_validates_location(
- sender_module: ModuleType, code: int, location: str, expected: str | None
-) -> None:
- def handler(_request: httpx.Request) -> httpx.Response:
- return httpx.Response(code, headers={"Location": location})
-
- with httpx.Client(transport=httpx.MockTransport(handler)) as client:
- result = sender_module.post_webmention(
- client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL
- )
-
- assert result == (code, expected)
-
-
@pytest.mark.parametrize(
("status_url", "expected"),
[
diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py
new file mode 100644
index 0000000..3c3a93e
--- /dev/null
+++ b/tests/tasks/test_source_parser.py
@@ -0,0 +1,1008 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from pathlib import Path
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+from flask import Flask
+from pytest import MonkeyPatch
+
+from webmentions_ssg.tasks import source_parser
+
+from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post
+
+
+@pytest.fixture
+def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+ _ = app
+ from webmentions_ssg.tasks import source_parser
+
+ monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True))
+ return source_parser
+
+
+def test_parse_entry_returns_microformats_entry(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ element = document.find(class_="h-entry")
+
+ assert element is not None
+
+ entry = source_parser_module.parse_entry(element, SOURCE_URL)
+
+ assert "h-entry" in entry["type"]
+ assert entry["properties"]["url"] == [SOURCE_URL]
+
+
+def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article>
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ element = document.find("article")
+
+ assert element is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one parsed h-entry",
+ ):
+ source_parser_module.parse_entry(element, SOURCE_URL)
+
+
+def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry" id="source">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <p class="p-name">
+ Example post
+ </p>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ element, entry = source_parser_module.primary_entry(document, SOURCE_URL)
+
+ assert element["id"] == "source"
+ assert entry["properties"]["name"] == ["Example post"]
+ assert entry["properties"]["url"] == [SOURCE_URL]
+
+
+def test_primary_entry_rejects_missing_h_entry(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <main>
+ <p>No microformats here.</p>
+ </main>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 0",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_multiple_h_entries(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ First
+ </a>
+ </article>
+
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/other/"
+ >
+ Second
+ </a>
+ </article>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 2",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/nested/"
+ >
+ Nested
+ </a>
+ </article>
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one h-entry, found 2",
+ ):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="does not match"):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_primary_entry_rejects_nonmatching_u_url(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="https://example.com/other/"
+ >
+ Other
+ </a>
+
+ <div class="e-content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="does not match"):
+ source_parser_module.primary_entry(document, SOURCE_URL)
+
+
+def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content" id="content">
+ Hello world
+ </div>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ content = source_parser_module.content_element(entry)
+
+ assert content["id"] == "content"
+
+
+def test_content_element_rejects_missing_e_content(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one e-content, found 0",
+ ):
+ source_parser_module.content_element(entry)
+
+
+def test_content_element_rejects_multiple_e_content(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <div class="e-content">
+ First
+ </div>
+
+ <div class="e-content">
+ Second
+ </div>
+ </article>
+ """
+ )
+
+ entry = document.find(class_="h-entry")
+
+ assert entry is not None
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Expected exactly one e-content, found 2",
+ ):
+ source_parser_module.content_element(entry)
+
+
+def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ f"""
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+ assert result == SOURCE_URL
+
+
+def test_canonical_url_returns_none_when_missing(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <title>Example</title>
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+ assert result is None
+
+
+def test_canonical_url_resolves_relative_url_with_base_url(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.side_effect = [False, True]
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="./canonical/"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ result = source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+ assert result == ("https://dennisfink.me/blog/example/canonical/")
+
+
+def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="./canonical/"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="relative canonical"
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=None
+ )
+
+
+def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None:
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href=" "
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="empty canonical URL"
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+
+def test_scan_source_uses_canonical_without_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="https://example.com/target">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.path == "example/index.html"
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+ assert len(scanned.content_hash) == 32
+
+
+def test_scan_source_ignores_markup_noise_in_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ "<p>Hello <strong>world</strong>!</p>"
+ '<a href="https://example.com/target">Target</a>',
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ write_post(
+ tmp_path,
+ "example",
+ '<p class="generated" data-build="123"> Hello world! </p>'
+ '<a class="external" href="https://example.com/target"> Target </a>',
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_uses_stable_time_value_inside_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_hash_changes_when_image_changes(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_includes_reaction_targets_outside_content(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<a class="u-like-of" href="https://example.com/first">Like</a>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ "https://example.com/first", "https://example.com/second"
+ ),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == frozenset({"https://example.com/first"})
+ assert second.targets == frozenset({"https://example.com/second"})
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_ignores_reaction_target_order(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
+ second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ first_link + second_link + '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ first_link + second_link, second_link + first_link
+ ),
+ encoding="utf-8",
+ )
+ second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_falls_back_to_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ """
+ <!doctype html>
+ <html>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="./">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <a href="https://example.com/target">
+ Target
+ </a>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path, root=tmp_path, base_url=BASE_URL
+ )
+
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+def test_scan_source_requires_canonical_or_base_url(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ """
+ <!doctype html>
+ <html>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="./">
+ Permalink
+ </a>
+
+ <div class="e-content">
+ <p>Hello world.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"):
+ source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+
+def test_scan_source_extracts_only_content_links(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <a href="https://example.com/metadata">
+ Metadata link
+ </a>
+
+ <div class="e-content">
+ <a href="https://example.com/content">
+ Content link
+ </a>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://example.com/content"})
+
+
+def test_scan_source_extracts_relative_content_link(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="../other/">
+ Other post
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"})
+
+
+@pytest.mark.parametrize(
+ "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+)
+def test_scan_source_extracts_reaction_properties(
+ source_parser_module: ModuleType, tmp_path: Path, property_name: str
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ target = f"https://example.com/{property_name}"
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a class="u-url" href="{SOURCE_URL}">
+ Permalink
+ </a>
+
+ <a
+ class="u-{property_name}"
+ href="{target}"
+ >
+ Reaction target
+ </a>
+
+ <div class="e-content">
+ <p>Post content.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({target})
+
+
+@pytest.mark.parametrize("tag", ("link", "area"))
+def test_scan_source_ignores_non_anchor_href_elements(
+ source_parser_module: ModuleType, tmp_path: Path, tag: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <{tag} href="https://example.com/target">
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset()
+
+
+def test_scan_source_ignores_self_fragment(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ """
+ <a href="#section">
+ Section
+ </a>
+
+ <a href="https://example.com/target">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+def test_scan_source_rejects_symlink_outside_root_before_read(
+ source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ external = write_post(tmp_path / "outside", "private", "Sensitive content")
+ linked = root / "escaped" / "index.html"
+ linked.parent.mkdir()
+ linked.symlink_to(external)
+
+ read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
+ monkeypatch.setattr(Path, "read_bytes", read_bytes)
+
+ with pytest.raises(
+ source_parser_module.SourceScanError, match="escapes source directory"
+ ):
+ source_parser_module.scan_source_file(linked, root=root, base_url=None)
+
+ read_bytes.assert_not_called()
+
+
+def test_scan_source_accepts_symlink_within_root(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ linked = write_post(
+ root, "example", '<a href="https://example.com/target">Target</a>'
+ )
+ shared = root / "shared.html"
+ linked.rename(shared)
+ linked.symlink_to(shared)
+
+ scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None)
+
+ assert scanned.path == "example/index.html"
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+@pytest.mark.parametrize(
+ "href",
+ (
+ "https://dennisfink.me/blog/other/",
+ "https://www.dennisfink.me/blog/other/",
+ "https://webmentions.dennisfink.me/endpoint",
+ "https://foo.bar.dennisfink.me/example",
+ ),
+)
+def test_scan_source_ignores_matching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path, href: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <a href="{href}">
+ Ignored
+ </a>
+
+ <a href="https://example.com/target">
+ External
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+@pytest.mark.parametrize(
+ "href",
+ (
+ "https://notdennisfink.me/example",
+ "https://dennisfink.me.example.com/example",
+ "https://example.com/example",
+ ),
+)
+def test_scan_source_keeps_nonmatching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path, href: str
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ f"""
+ <a href="{href}">
+ Target
+ </a>
+ """,
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset({href})
+
+
+def test_scan_source_ignores_reaction_to_matching_hostname(
+ source_parser_module: ModuleType, tmp_path: Path
+) -> None:
+ directory = tmp_path / "example"
+ directory.mkdir()
+
+ path = directory / "index.html"
+ path.write_text(
+ f"""
+ <!doctype html>
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="{SOURCE_URL}"
+ >
+ </head>
+ <body>
+ <article class="h-entry">
+ <a
+ class="u-url"
+ href="{SOURCE_URL}"
+ >
+ Permalink
+ </a>
+
+ <a
+ class="u-in-reply-to"
+ href="https://www.dennisfink.me/blog/other/"
+ >
+ Reply target
+ </a>
+
+ <div class="e-content">
+ <p>Reply content.</p>
+ </div>
+ </article>
+ </body>
+ </html>
+ """,
+ encoding="utf-8",
+ )
+
+ scanned = source_parser_module.scan_source_file(
+ path,
+ root=tmp_path,
+ base_url=None,
+ ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
+ )
+
+ assert scanned.targets == frozenset()
+
+
+def test_canonical_url_rejects_non_http_resolved_url(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ document = parse_html(
+ """
+ <html>
+ <head>
+ <link
+ rel="canonical"
+ href="mailto:example@example.com"
+ >
+ </head>
+ </html>
+ """
+ )
+
+ with pytest.raises(
+ source_parser_module.SourceScanError,
+ match="Canonical URL is not a valid HTTP or HTTPS URL",
+ ):
+ source_parser_module.canonical_url(
+ document, relative_path=Path("example/index.html"), base_url=BASE_URL
+ )
+
+
+def test_property_urls_ignores_non_mapping_properties(
+ source_parser_module: ModuleType,
+) -> None:
+ entry = {"properties": "invalid"}
+
+ assert tuple(source_parser_module.property_urls(entry, "url")) == ()
+
+
+@pytest.mark.parametrize(
+ ("parsed", "message"),
+ [
+ pytest.param(
+ [],
+ "Microformats parser did not return a document object",
+ id="non-object-document",
+ ),
+ pytest.param(
+ {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
+ ),
+ pytest.param(
+ {"items": [{"type": ["h-card"]}]},
+ "Parsed microformats item is not an h-entry",
+ id="wrong-item-type",
+ ),
+ ],
+)
+def test_parse_entry_rejects_invalid_parser_output(
+ source_parser_module: ModuleType,
+ monkeypatch: MonkeyPatch,
+ parsed: object,
+ message: str,
+) -> None:
+ document = parse_html(
+ """
+ <article class="h-entry">
+ <p>Hello world</p>
+ </article>
+ """
+ )
+
+ element = document.find(class_="h-entry")
+
+ assert element is not None
+
+ monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed)
+
+ with pytest.raises(source_parser_module.SourceScanError, match=message):
+ source_parser_module.parse_entry(element, SOURCE_URL)
+
+
+def test_normalize_target_ignores_empty_target(
+ source_parser_module: ModuleType,
+) -> None:
+ assert (
+ source_parser_module.normalize_target(
+ " ", base_url=SOURCE_URL, source_url=SOURCE_URL
+ )
+ is None
+ )
+
+
+def test_normalize_target_ignores_url_rejected_by_validator(
+ source_parser_module: ModuleType,
+) -> None:
+ source_parser_module.is_http_url.return_value = False
+
+ assert (
+ source_parser_module.normalize_target(
+ "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL
+ )
+ is None
+ )