diff options
| author | Dennis Fink | 2026-11-12 19:57:08 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-12 19:57:08 +0100 |
| commit | 4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch) | |
| tree | f7ed41a374a8a4136792039a262dc62aa8cf8c59 /tests/tasks/test_source_parser.py | |
| parent | 7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff) | |
| download | webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip | |
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and
application extension setup so each module owns a focused concern.
Reorganize the test suite around the new boundaries while preserving
existing Webmention behavior and coverage.
Diffstat (limited to '')
| -rw-r--r-- | tests/tasks/test_source_parser.py | 1008 |
1 files changed, 1008 insertions, 0 deletions
diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py new file mode 100644 index 0000000..3c3a93e --- /dev/null +++ b/tests/tasks/test_source_parser.py @@ -0,0 +1,1008 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock + +import pytest +from flask import Flask +from pytest import MonkeyPatch + +from webmentions_ssg.tasks import source_parser + +from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post + + +@pytest.fixture +def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: + _ = app + from webmentions_ssg.tasks import source_parser + + monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) + return source_parser + + +def test_parse_entry_returns_microformats_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + entry = source_parser_module.parse_entry(element, SOURCE_URL) + + assert "h-entry" in entry["type"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_parse_entry_rejects_non_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article> + <p>Hello world</p> + </article> + """ + ) + + element = document.find("article") + + assert element is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one parsed h-entry", + ): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_primary_entry_returns_source_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry" id="source"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <p class="p-name"> + Example post + </p> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element, entry = source_parser_module.primary_entry(document, SOURCE_URL) + + assert element["id"] == "source" + assert entry["properties"]["name"] == ["Example post"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_primary_entry_rejects_missing_h_entry( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <main> + <p>No microformats here.</p> + </main> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 0", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_multiple_h_entries( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + First + </a> + </article> + + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Second + </a> + </article> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nested_h_entry(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/nested/" + > + Nested + </a> + </article> + </div> + </article> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one h-entry, found 2", + ): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_missing_u_url(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nonmatching_u_url( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Other + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(source_parser_module.SourceScanError, match="does not match"): + source_parser_module.primary_entry(document, SOURCE_URL) + + +def test_content_element_returns_e_content(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content" id="content"> + Hello world + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + content = source_parser_module.content_element(entry) + + assert content["id"] == "content" + + +def test_content_element_rejects_missing_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 0", + ): + source_parser_module.content_element(entry) + + +def test_content_element_rejects_multiple_e_content( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + First + </div> + + <div class="e-content"> + Second + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + source_parser_module.SourceScanError, + match="Expected exactly one e-content, found 2", + ): + source_parser_module.content_element(entry) + + +def test_canonical_url_returns_absolute_url(source_parser_module: ModuleType) -> None: + document = parse_html( + f""" + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result == SOURCE_URL + + +def test_canonical_url_returns_none_when_missing( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + """ + <html> + <head> + <title>Example</title> + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result is None + + +def test_canonical_url_resolves_relative_url_with_base_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.side_effect = [False, True] + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + result = source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + assert result == ("https://dennisfink.me/blog/example/canonical/") + + +def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="relative canonical" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + +def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href=" " + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, match="empty canonical URL" + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_scan_source_uses_canonical_without_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + assert len(scanned.content_hash) == 32 + + +def test_scan_source_ignores_markup_noise_in_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "<p>Hello <strong>world</strong>!</p>" + '<a href="https://example.com/target">Target</a>', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '<p class="generated" data-build="123"> Hello world! </p>' + '<a class="external" href="https://example.com/target"> Target </a>', + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>', + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">') + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '<div class="e-content">', + '<a class="u-like-of" href="https://example.com/first">Like</a>' + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + first_link = '<a class="u-like-of" href="https://example.com/a">A</a>' + second_link = '<a class="u-like-of" href="https://example.com/b">B</a>' + path.write_text( + html.replace( + '<div class="e-content">', + first_link + second_link + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_falls_back_to_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <a href="https://example.com/target"> + Target + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, root=tmp_path, base_url=BASE_URL + ) + + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_requires_canonical_or_base_url( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <p>Hello world.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"): + source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + +def test_scan_source_extracts_only_content_links( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a href="https://example.com/metadata"> + Metadata link + </a> + + <div class="e-content"> + <a href="https://example.com/content"> + Content link + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/content"}) + + +def test_scan_source_extracts_relative_content_link( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="../other/"> + Other post + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) + + +@pytest.mark.parametrize( + "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") +) +def test_scan_source_extracts_reaction_properties( + source_parser_module: ModuleType, tmp_path: Path, property_name: str +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + target = f"https://example.com/{property_name}" + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a + class="u-{property_name}" + href="{target}" + > + Reaction target + </a> + + <div class="e-content"> + <p>Post content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({target}) + + +@pytest.mark.parametrize("tag", ("link", "area")) +def test_scan_source_ignores_non_anchor_href_elements( + source_parser_module: ModuleType, tmp_path: Path, tag: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <{tag} href="https://example.com/target"> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset() + + +def test_scan_source_ignores_self_fragment( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="#section"> + Section + </a> + + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_rejects_symlink_outside_root_before_read( + source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + root = tmp_path / "site" + root.mkdir() + external = write_post(tmp_path / "outside", "private", "Sensitive content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + + read_bytes = Mock(side_effect=AssertionError("External file must not be read")) + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + with pytest.raises( + source_parser_module.SourceScanError, match="escapes source directory" + ): + source_parser_module.scan_source_file(linked, root=root, base_url=None) + + read_bytes.assert_not_called() + + +def test_scan_source_accepts_symlink_within_root( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + root = tmp_path / "site" + root.mkdir() + linked = write_post( + root, "example", '<a href="https://example.com/target">Target</a>' + ) + shared = root / "shared.html" + linked.rename(shared) + linked.symlink_to(shared) + + scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://dennisfink.me/blog/other/", + "https://www.dennisfink.me/blog/other/", + "https://webmentions.dennisfink.me/endpoint", + "https://foo.bar.dennisfink.me/example", + ), +) +def test_scan_source_ignores_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Ignored + </a> + + <a href="https://example.com/target"> + External + </a> + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://notdennisfink.me/example", + "https://dennisfink.me.example.com/example", + "https://example.com/example", + ), +) +def test_scan_source_keeps_nonmatching_hostname( + source_parser_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Target + </a> + """, + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({href}) + + +def test_scan_source_ignores_reaction_to_matching_hostname( + source_parser_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a + class="u-url" + href="{SOURCE_URL}" + > + Permalink + </a> + + <a + class="u-in-reply-to" + href="https://www.dennisfink.me/blog/other/" + > + Reply target + </a> + + <div class="e-content"> + <p>Reply content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = source_parser_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset() + + +def test_canonical_url_rejects_non_http_resolved_url( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="mailto:example@example.com" + > + </head> + </html> + """ + ) + + with pytest.raises( + source_parser_module.SourceScanError, + match="Canonical URL is not a valid HTTP or HTTPS URL", + ): + source_parser_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +def test_property_urls_ignores_non_mapping_properties( + source_parser_module: ModuleType, +) -> None: + entry = {"properties": "invalid"} + + assert tuple(source_parser_module.property_urls(entry, "url")) == () + + +@pytest.mark.parametrize( + ("parsed", "message"), + [ + pytest.param( + [], + "Microformats parser did not return a document object", + id="non-object-document", + ), + pytest.param( + {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" + ), + pytest.param( + {"items": [{"type": ["h-card"]}]}, + "Parsed microformats item is not an h-entry", + id="wrong-item-type", + ), + ], +) +def test_parse_entry_rejects_invalid_parser_output( + source_parser_module: ModuleType, + monkeypatch: MonkeyPatch, + parsed: object, + message: str, +) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed) + + with pytest.raises(source_parser_module.SourceScanError, match=message): + source_parser_module.parse_entry(element, SOURCE_URL) + + +def test_normalize_target_ignores_empty_target( + source_parser_module: ModuleType, +) -> None: + assert ( + source_parser_module.normalize_target( + " ", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) + + +def test_normalize_target_ignores_url_rejected_by_validator( + source_parser_module: ModuleType, +) -> None: + source_parser_module.is_http_url.return_value = False + + assert ( + source_parser_module.normalize_target( + "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL + ) + is None + ) |
