# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause from pathlib import Path from types import ModuleType from unittest.mock import Mock import pytest from flask import Flask from pytest import MonkeyPatch from webmentions_ssg.tasks import source_parser from .scanner_helpers import BASE_URL, SOURCE_URL, parse_html, write_post @pytest.fixture def source_parser_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType: _ = app from webmentions_ssg.tasks import source_parser monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True)) return source_parser def test_parse_h_entry_returns_microformats_entry( source_parser_module: ModuleType, ) -> None: document = parse_html( f"""
Permalink
Hello world
""" ) element = document.find(class_="h-entry") assert element is not None entry = source_parser_module.parse_h_entry(element, SOURCE_URL) assert "h-entry" in entry["type"] assert entry["properties"]["url"] == [SOURCE_URL] def test_parse_h_entry_rejects_non_entry(source_parser_module: ModuleType) -> None: document = parse_html( """

Hello world

""" ) element = document.find("article") assert element is not None with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one parsed h-entry", ): source_parser_module.parse_h_entry(element, SOURCE_URL) def test_find_primary_h_entry_returns_source_entry( source_parser_module: ModuleType, ) -> None: document = parse_html( f"""
Permalink

Example post

Hello world
""" ) element, entry = source_parser_module.find_primary_h_entry(document, SOURCE_URL) assert element["id"] == "source" assert entry["properties"]["name"] == ["Example post"] assert entry["properties"]["url"] == [SOURCE_URL] def test_find_primary_h_entry_rejects_missing_h_entry( source_parser_module: ModuleType, ) -> None: document = parse_html( """

No microformats here.

""" ) with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one h-entry, found 0", ): source_parser_module.find_primary_h_entry(document, SOURCE_URL) def test_find_primary_h_entry_rejects_multiple_h_entries( source_parser_module: ModuleType, ) -> None: document = parse_html( f"""
First
Second
""" ) with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one h-entry, found 2", ): source_parser_module.find_primary_h_entry(document, SOURCE_URL) def test_find_primary_h_entry_rejects_nested_h_entry( source_parser_module: ModuleType, ) -> None: document = parse_html( f"""
Permalink
""" ) with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one h-entry, found 2", ): source_parser_module.find_primary_h_entry(document, SOURCE_URL) def test_find_primary_h_entry_rejects_missing_u_url( source_parser_module: ModuleType, ) -> None: document = parse_html( """
Hello world
""" ) with pytest.raises(source_parser_module.SourceScanError, match="does not match"): source_parser_module.find_primary_h_entry(document, SOURCE_URL) def test_find_primary_h_entry_rejects_nonmatching_u_url( source_parser_module: ModuleType, ) -> None: document = parse_html( """
Other
Hello world
""" ) with pytest.raises(source_parser_module.SourceScanError, match="does not match"): source_parser_module.find_primary_h_entry(document, SOURCE_URL) def test_find_e_content_returns_e_content(source_parser_module: ModuleType) -> None: document = parse_html( """
Hello world
""" ) entry = document.find(class_="h-entry") assert entry is not None content = source_parser_module.find_e_content(entry) assert content["id"] == "content" def test_find_e_content_rejects_missing_e_content( source_parser_module: ModuleType, ) -> None: document = parse_html( """

Hello world

""" ) entry = document.find(class_="h-entry") assert entry is not None with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one e-content, found 0", ): source_parser_module.find_e_content(entry) def test_find_e_content_rejects_multiple_e_content( source_parser_module: ModuleType, ) -> None: document = parse_html( """
First
Second
""" ) entry = document.find(class_="h-entry") assert entry is not None with pytest.raises( source_parser_module.SourceScanError, match="Expected exactly one e-content, found 2", ): source_parser_module.find_e_content(entry) def test_declared_canonical_url_returns_absolute_url( source_parser_module: ModuleType, ) -> None: document = parse_html( f""" """ ) result = source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=None ) assert result == SOURCE_URL def test_declared_canonical_url_returns_none_when_missing( source_parser_module: ModuleType, ) -> None: document = parse_html( """ Example """ ) result = source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=None ) assert result is None def test_declared_canonical_url_resolves_relative_url_with_base_url( source_parser_module: ModuleType, ) -> None: source_parser_module.is_http_url.side_effect = [False, True] document = parse_html( """ """ ) result = source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=BASE_URL ) assert result == ("https://dennisfink.me/blog/example/canonical/") def test_relative_canonical_requires_base_url(source_parser_module: ModuleType) -> None: source_parser_module.is_http_url.return_value = False document = parse_html( """ """ ) with pytest.raises( source_parser_module.SourceScanError, match="relative canonical" ): source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=None ) def test_empty_canonical_is_rejected(source_parser_module: ModuleType) -> None: document = parse_html( """ """ ) with pytest.raises( source_parser_module.SourceScanError, match="empty canonical URL" ): source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=BASE_URL ) def test_scan_source_uses_canonical_without_base_url( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post( tmp_path, "example", """ Target """, ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.path == "example/index.html" assert scanned.url == SOURCE_URL assert scanned.targets == frozenset({"https://example.com/target"}) assert len(scanned.content_hash) == 32 def test_scan_source_ignores_markup_noise_in_content( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post( tmp_path, "example", "

Hello world!

" 'Target', ) first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) write_post( tmp_path, "example", '

Hello world!

' ' Target ', ) second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert first.targets == second.targets assert first.content_hash == second.content_hash def test_scan_source_uses_stable_time_value_inside_content( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post( tmp_path, "example", '', ) first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) path.write_text( path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), encoding="utf-8", ) second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert first.content_hash == second.content_hash def test_scan_source_hash_changes_when_image_changes( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post(tmp_path, "example", 'A photo') first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) path.write_text( path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), encoding="utf-8", ) second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert first.content_hash != second.content_hash def test_scan_source_hash_includes_reaction_targets_outside_content( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post(tmp_path, "example", "

Same content.

") html = path.read_text(encoding="utf-8") path.write_text( html.replace( '
', 'Like' '
', ), encoding="utf-8", ) first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) path.write_text( path.read_text(encoding="utf-8").replace( "https://example.com/first", "https://example.com/second" ), encoding="utf-8", ) second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert first.targets == frozenset({"https://example.com/first"}) assert second.targets == frozenset({"https://example.com/second"}) assert first.content_hash != second.content_hash def test_scan_source_hash_ignores_reaction_target_order( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post(tmp_path, "example", "

Same content.

") html = path.read_text(encoding="utf-8") first_link = 'A' second_link = 'B' path.write_text( html.replace( '
', first_link + second_link + '
', ), encoding="utf-8", ) first = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) path.write_text( path.read_text(encoding="utf-8").replace( first_link + second_link, second_link + first_link ), encoding="utf-8", ) second = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert first.targets == second.targets assert first.content_hash == second.content_hash def test_scan_source_falls_back_to_base_url( source_parser_module: ModuleType, tmp_path: Path ) -> None: directory = tmp_path / "example" directory.mkdir() path = directory / "index.html" path.write_text( """ """, encoding="utf-8", ) scanned = source_parser_module.scan_source_file( path, root=tmp_path, base_url=BASE_URL ) assert scanned.url == SOURCE_URL assert scanned.targets == frozenset({"https://example.com/target"}) def test_scan_source_requires_canonical_or_base_url( source_parser_module: ModuleType, tmp_path: Path ) -> None: directory = tmp_path / "example" directory.mkdir() path = directory / "index.html" path.write_text( """ """, encoding="utf-8", ) with pytest.raises(source_parser_module.SourceScanError, match="no canonical URL"): source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) def test_scan_source_extracts_only_content_links( source_parser_module: ModuleType, tmp_path: Path ) -> None: directory = tmp_path / "example" directory.mkdir() path = directory / "index.html" path.write_text( f""" """, encoding="utf-8", ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.targets == frozenset({"https://example.com/content"}) def test_scan_source_extracts_relative_content_link( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post( tmp_path, "example", """ Other post """, ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) @pytest.mark.parametrize( "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") ) def test_scan_source_extracts_reaction_properties( source_parser_module: ModuleType, tmp_path: Path, property_name: str ) -> None: directory = tmp_path / "example" directory.mkdir() target = f"https://example.com/{property_name}" path = directory / "index.html" path.write_text( f""" """, encoding="utf-8", ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.targets == frozenset({target}) @pytest.mark.parametrize("tag", ("link", "area")) def test_scan_source_ignores_non_anchor_href_elements( source_parser_module: ModuleType, tmp_path: Path, tag: str ) -> None: path = write_post( tmp_path, "example", f""" <{tag} href="https://example.com/target"> """, ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.targets == frozenset() def test_scan_source_ignores_self_fragment( source_parser_module: ModuleType, tmp_path: Path ) -> None: path = write_post( tmp_path, "example", """ Section Target """, ) scanned = source_parser_module.scan_source_file(path, root=tmp_path, base_url=None) assert scanned.targets == frozenset({"https://example.com/target"}) def test_scan_source_rejects_symlink_outside_root_before_read( source_parser_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: root = tmp_path / "site" root.mkdir() external = write_post(tmp_path / "outside", "private", "Sensitive content") linked = root / "escaped" / "index.html" linked.parent.mkdir() linked.symlink_to(external) read_bytes = Mock(side_effect=AssertionError("External file must not be read")) monkeypatch.setattr(Path, "read_bytes", read_bytes) with pytest.raises( source_parser_module.SourceScanError, match="escapes source directory" ): source_parser_module.scan_source_file(linked, root=root, base_url=None) read_bytes.assert_not_called() def test_scan_source_accepts_symlink_within_root( source_parser_module: ModuleType, tmp_path: Path ) -> None: root = tmp_path / "site" root.mkdir() linked = write_post( root, "example", 'Target' ) shared = root / "shared.html" linked.rename(shared) linked.symlink_to(shared) scanned = source_parser_module.scan_source_file(linked, root=root, base_url=None) assert scanned.path == "example/index.html" assert scanned.url == SOURCE_URL assert scanned.targets == frozenset({"https://example.com/target"}) @pytest.mark.parametrize( "href", ( "https://dennisfink.me/blog/other/", "https://www.dennisfink.me/blog/other/", "https://webmentions.dennisfink.me/endpoint", "https://foo.bar.dennisfink.me/example", ), ) def test_scan_source_ignores_matching_hostname( source_parser_module: ModuleType, tmp_path: Path, href: str ) -> None: path = write_post( tmp_path, "example", f""" Ignored External """, ) scanned = source_parser_module.scan_source_file( path, root=tmp_path, base_url=None, ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), ) assert scanned.targets == frozenset({"https://example.com/target"}) @pytest.mark.parametrize( "href", ( "https://notdennisfink.me/example", "https://dennisfink.me.example.com/example", "https://example.com/example", ), ) def test_scan_source_keeps_nonmatching_hostname( source_parser_module: ModuleType, tmp_path: Path, href: str ) -> None: path = write_post( tmp_path, "example", f""" Target """, ) scanned = source_parser_module.scan_source_file( path, root=tmp_path, base_url=None, ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), ) assert scanned.targets == frozenset({href}) def test_scan_source_ignores_reaction_to_matching_hostname( source_parser_module: ModuleType, tmp_path: Path ) -> None: directory = tmp_path / "example" directory.mkdir() path = directory / "index.html" path.write_text( f""" """, encoding="utf-8", ) scanned = source_parser_module.scan_source_file( path, root=tmp_path, base_url=None, ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), ) assert scanned.targets == frozenset() def test_declared_canonical_url_rejects_non_http_resolved_url( source_parser_module: ModuleType, ) -> None: source_parser_module.is_http_url.return_value = False document = parse_html( """ """ ) with pytest.raises( source_parser_module.SourceScanError, match="Canonical URL is not a valid HTTP or HTTPS URL", ): source_parser_module.declared_canonical_url( document, relative_path=Path("example/index.html"), base_url=BASE_URL ) def test_mf2_string_values_ignores_non_mapping_properties( source_parser_module: ModuleType, ) -> None: entry = {"properties": "invalid"} assert tuple(source_parser_module.mf2_string_values(entry, "url")) == () @pytest.mark.parametrize( ("parsed", "message"), [ pytest.param( [], "Microformats parser did not return a document object", id="non-object-document", ), pytest.param( {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry" ), pytest.param( {"items": [{"type": ["h-card"]}]}, "Parsed microformats item is not an h-entry", id="wrong-item-type", ), ], ) def test_parse_h_entry_rejects_invalid_parser_output( source_parser_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str, ) -> None: document = parse_html( """

Hello world

""" ) element = document.find(class_="h-entry") assert element is not None monkeypatch.setattr(source_parser.mf2py, "parse", lambda **kwargs: parsed) with pytest.raises(source_parser_module.SourceScanError, match=message): source_parser_module.parse_h_entry(element, SOURCE_URL) def test_normalize_target_url_ignores_empty_target( source_parser_module: ModuleType, ) -> None: assert ( source_parser_module.normalize_target_url( " ", base_url=SOURCE_URL, source_url=SOURCE_URL ) is None ) def test_normalize_target_url_ignores_url_rejected_by_validator( source_parser_module: ModuleType, ) -> None: source_parser_module.is_http_url.return_value = False assert ( source_parser_module.normalize_target_url( "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL ) is None ) def test_outgoing_targets_skip_urls_with_unparseable_hostname( source_parser_module: ModuleType, monkeypatch: MonkeyPatch ) -> None: document = parse_html( '' ) content = document.find("div") assert content is not None original = source_parser_module.normalize_hostname def invalid_hostname(url: str, *, from_url: bool = False) -> str: if "bad.example" in url: raise ValueError("Malformed URL hostname") return original(url, from_url=from_url) monkeypatch.setattr(source_parser_module, "normalize_hostname", invalid_hostname) assert list( source_parser_module.iter_outgoing_targets( content, {}, base_url=SOURCE_URL, source_url=SOURCE_URL, ) ) == ["https://good.example/post"] def test_stable_content_text_excludes_hidden_markup( source_parser_module: ModuleType, ) -> None: document = parse_html( "
Hello" "world
" ) content = document.find("div") assert content is not None normalized = source_parser_module.stable_content_text(content) assert "Hello" in normalized and "world" in normalized assert "secret-" not in normalized