From 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Thu, 12 Nov 2026 19:57:08 +0100 Subject: refactor(core): split modules by responsibility Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage. --- tests/tasks/test_source_parser.py | 1008 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1008 insertions(+) create mode 100644 tests/tasks/test_source_parser.py (limited to 'tests/tasks/test_source_parser.py') diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py new file mode 100644 index 0000000..3c3a93e --- /dev/null +++ b/tests/tasks/test_source_parser.py @@ -0,0 +1,1008 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from pytest import MonkeyPatch + +from webmentions_ssg.tasks import source_parser + +from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post + + +@pytest.fixture +def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + _ = app + from webmentions_ssg.tasks import source_parser + + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) + return source_parser + + +def test_parse_entry_returns_microformats_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" +
+ + Permalink + + +
+ Hello world +
+
+ """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + entry = source_parser_module.parse_entry(element, SOURCE_URL) + + assert "h-entry" in entry["type"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + element = document.find("article") + + assert element is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one parsed h-entry", + ): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" +
+ + Permalink + + +

+ Example post +

+ +
+ Hello world +
+
+ """ + ) + + element, entry = source_parser_module.primary_entry(document, SOURCE_URL) + + assert element["id"] == "source" + assert entry["properties"]["name"] == ["Example post"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_primary_entry_rejects_missing_h_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+

No microformats here.

+
+ """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 0", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_multiple_h_entries( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" +
+ + First + +
+ +
+ + Second + +
+ """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" +
+ + Permalink + + +
+ +
+
+ """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+
+ Hello world +
+
+ """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nonmatching_u_url( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+ + Other + + +
+ Hello world +
+
+ """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None: + document = parse_html( + """ +
+
+ Hello world +
+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + content = source_parser_module.content_element(entry) + + assert content["id"] == "content" + + +def test_content_element_rejects_missing_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 0", + ): + source_parser_module.content_element(entry) + + +def test_content_element_rejects_multiple_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ +
+
+ First +
+ +
+ Second +
+
+ """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 2", + ): + source_parser_module.content_element(entry) + + +def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + + + + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result == SOURCE_URL + + +def test_canonical_url_returns_none_when_missing( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + + + Example + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result is None + + +def test_canonical_url_resolves_relative_url_with_base_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.side_effect = [False, True] + + document = parse_html( + """ + + + + + + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + assert result == ("https://dennisfink.me/blog/example/canonical/") + + +def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="relative canonical" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + +def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="empty canonical URL" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_scan_source_uses_canonical_without_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + assert len(scanned.content_hash) == 32 + + +def test_scan_source_ignores_markup_noise_in_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "

Hello world!

" + 'Target', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '

Hello world!

' + ' Target ', + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", 'A photo') + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '
', + 'Like' + '
', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + first_link = 'A' + second_link = 'B' + path.write_text( + html.replace( + '
', + first_link + second_link + '
', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_falls_back_to_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, root=tmp_path, base_url=BASE_URL + ) + + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_requires_canonical_or_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + + + + + + + """, + encoding="utf-8", + ) + + with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"): + source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + +def test_scan_source_extracts_only_content_links( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/content"}) + + +def test_scan_source_extracts_relative_content_link( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Other post + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) + + +@pytest.mark.parametrize( + "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") +) +def test_scan_source_extracts_reaction_properties( + source_parser_module: ModuleType, tmp_path: Path, property_name: str +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + target = f"https://example.com/{property_name}" + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({target}) + + +@pytest.mark.parametrize("tag", ("link", "area")) +def test_scan_source_ignores_non_anchor_href_elements( + source_parser_module: ModuleType, tmp_path: Path, tag: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <{tag} href="https://example.com/target"> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset() + + +def test_scan_source_ignores_self_fragment( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + + Section + + + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_rejects_symlink_outside_root_before_read( + source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + root = tmp_path / "site" + root.mkdir() + external = write_post(tmp_path / "outside", "private", "Sensitive content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + + read_bytes = Mock(side_effect=AssertionError("External file must not be read")) + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + with pytest.raises( + source_parser_module.SourceScanError, match="escapes source directory" + ): + source_parser_module.scan_source_file(linked, root=root, base_url=None) + + read_bytes.assert_not_called() + + +def test_scan_source_accepts_symlink_within_root( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + root = tmp_path / "site" + root.mkdir() + linked = write_post( + root, "example", 'Target' + ) + shared = root / "shared.html" + linked.rename(shared) + linked.symlink_to(shared) + + scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://dennisfink.me/blog/other/", + "https://www.dennisfink.me/blog/other/", + "https://webmentions.dennisfink.me/endpoint", + "https://foo.bar.dennisfink.me/example", + ), +) +def test_scan_source_ignores_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + + Ignored + + + + External + + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://notdennisfink.me/example", + "https://dennisfink.me.example.com/example", + "https://example.com/example", + ), +) +def test_scan_source_keeps_nonmatching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + + Target + + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({href}) + + +def test_scan_source_ignores_reaction_to_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + + + + + + + + + + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset() + + +def test_canonical_url_rejects_non_http_resolved_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + + + + + + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Canonical URL is not a valid HTTP or HTTPS URL", + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_property_urls_ignores_non_mapping_properties( + source_parser_module: ModuleType, +) -> None: + entry = {"properties": "invalid"} + + assert tuple(source_parser_module.property_urls(entry, "url")) == () + + +@pytest.mark.parametrize( + ("parsed", "message"), + [ + pytest.param( + [], + "Microformats parser did not return a document object", + id="non-object-document", + ), + pytest.param( + {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" + ), + pytest.param( + {"items": [{"type": ["h-card"]}]}, + "Parsed microformats item is not an h-entry", + id="wrong-item-type", + ), + ], +) +def test_parse_entry_rejects_invalid_parser_output( + source_parser_module: ModuleType, + monkeypatch: MonkeyPatch, + parsed: object, + message: str, +) -> None: + document = parse_html( + """ +
+

Hello world

+
+ """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed) + + with pytest.raises(source_parser_module.SourceScanError, match=message): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_normalize_target_ignores_empty_target( + source_parser_module: ModuleType, +) -> None: + assert ( + source_parser_module.normalize_target( + " ", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) + + +def test_normalize_target_ignores_url_rejected_by_validator( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + assert ( + source_parser_module.normalize_target( + "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) -- cgit v1.3.1