aboutsummaryrefslogtreecommitdiff
path: root/tests/tasks/test_scanner.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /tests/tasks/test_scanner.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to 'tests/tasks/test_scanner.py')
-rw-r--r--tests/tasks/test_scanner.py1035
1 files changed, 7 insertions, 1028 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 26e81c4..7eb970c 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -18,9 +18,9 @@ from pytest import MonkeyPatch
from webmentions_ssg import DATABASE as db
from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source
+from webmentions_ssg.tasks import source_parser
-BASE_URL = "https://dennisfink.me/blog/"
-SOURCE_URL = f"{BASE_URL}example/"
+from .scanner_helpers import BASE_URL, SOURCE_URL, configure_scanner, write_post
@pytest.fixture
@@ -29,6 +29,7 @@ def scanner_module(app: Flask, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
from webmentions_ssg.tasks import scanner
monkeypatch.setattr(scanner, "is_http_url", Mock(return_value=True))
+ monkeypatch.setattr(source_parser, "is_http_url", Mock(return_value=True))
return scanner
@@ -80,764 +81,6 @@ def test_manual_scan_propagates_unrelated_errors(
scanner_module.manual_scan_sources.call_local()
-def parse_html(html: str) -> BeautifulSoup:
- return BeautifulSoup(html, "html5lib")
-
-
-def write_post(
- root: Path,
- slug: str,
- content: str,
- *,
- canonical: str | None = None,
- entry_url: str | None = None,
-) -> Path:
- directory = root / slug
- directory.mkdir(parents=True, exist_ok=True)
-
- source_url = f"{BASE_URL}{slug}/"
-
- if canonical is None:
- canonical = source_url
-
- if entry_url is None:
- entry_url = source_url
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{canonical}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{entry_url}"
- >
- Permalink
- </a>
-
- <div class="e-content">
- {content}
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- return path
-
-
-def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None:
- app.config["SOURCE_DIRECTORY"] = str(root)
- app.config["SOURCE_BASE_URL"] = base_url
-
-
-def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- entry = scanner_module.parse_entry(element, SOURCE_URL)
-
- assert "h-entry" in entry["type"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article>
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find("article")
-
- assert element is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one parsed h-entry"
- ):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry" id="source">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <p class="p-name">
- Example post
- </p>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- element, entry = scanner_module.primary_entry(document, SOURCE_URL)
-
- assert element["id"] == "source"
- assert entry["properties"]["name"] == ["Example post"]
- assert entry["properties"]["url"] == [SOURCE_URL]
-
-
-def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <main>
- <p>No microformats here.</p>
- </main>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- First
- </a>
- </article>
-
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Second
- </a>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <div class="e-content">
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/nested/"
- >
- Nested
- </a>
- </article>
- </div>
- </article>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2"
- ):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <a
- class="u-url"
- href="https://example.com/other/"
- >
- Other
- </a>
-
- <div class="e-content">
- Hello world
- </div>
- </article>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="does not match"):
- scanner_module.primary_entry(document, SOURCE_URL)
-
-
-def test_content_element_returns_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content" id="content">
- Hello world
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- content = scanner_module.content_element(entry)
-
- assert content["id"] == "content"
-
-
-def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 0"
- ):
- scanner_module.content_element(entry)
-
-
-def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <div class="e-content">
- First
- </div>
-
- <div class="e-content">
- Second
- </div>
- </article>
- """
- )
-
- entry = document.find(class_="h-entry")
-
- assert entry is not None
-
- with pytest.raises(
- scanner_module.SourceScanError, match="Expected exactly one e-content, found 2"
- ):
- scanner_module.content_element(entry)
-
-
-def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None:
- document = parse_html(
- f"""
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result == SOURCE_URL
-
-
-def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <title>Example</title>
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
- assert result is None
-
-
-def test_canonical_url_resolves_relative_url_with_base_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.side_effect = [False, True]
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- result = scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
- assert result == ("https://dennisfink.me/blog/example/canonical/")
-
-
-def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="./canonical/"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="relative canonical"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=None
- )
-
-
-def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None:
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href=" "
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_scan_source_uses_canonical_without_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
- assert len(scanned.content_hash) == 32
-
-
-def test_scan_source_ignores_markup_noise_in_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- "<p>Hello <strong>world</strong>!</p>"
- '<a href="https://example.com/target">Target</a>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- write_post(
- tmp_path,
- "example",
- '<p class="generated" data-build="123"> Hello world! </p>'
- '<a class="external" href="https://example.com/target"> Target </a>',
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_uses_stable_time_value_inside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_hash_changes_when_image_changes(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_includes_reaction_targets_outside_content(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- path.write_text(
- html.replace(
- '<div class="e-content">',
- '<a class="u-like-of" href="https://example.com/first">Like</a>'
- '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- "https://example.com/first", "https://example.com/second"
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == frozenset({"https://example.com/first"})
- assert second.targets == frozenset({"https://example.com/second"})
- assert first.content_hash != second.content_hash
-
-
-def test_scan_source_hash_ignores_reaction_target_order(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(tmp_path, "example", "<p>Same content.</p>")
- html = path.read_text(encoding="utf-8")
- first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
- second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
- path.write_text(
- html.replace(
- '<div class="e-content">',
- first_link + second_link + '<div class="e-content">',
- ),
- encoding="utf-8",
- )
- first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- path.write_text(
- path.read_text(encoding="utf-8").replace(
- first_link + second_link, second_link + first_link
- ),
- encoding="utf-8",
- )
- second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert first.targets == second.targets
- assert first.content_hash == second.content_hash
-
-
-def test_scan_source_falls_back_to_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <a href="https://example.com/target">
- Target
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL)
-
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-def test_scan_source_requires_canonical_or_base_url(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- """
- <!doctype html>
- <html>
- <body>
- <article class="h-entry">
- <a class="u-url" href="./">
- Permalink
- </a>
-
- <div class="e-content">
- <p>Hello world.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"):
- scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
-
-def test_scan_source_extracts_only_content_links(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a href="https://example.com/metadata">
- Metadata link
- </a>
-
- <div class="e-content">
- <a href="https://example.com/content">
- Content link
- </a>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/content"})
-
-
-def test_scan_source_extracts_relative_content_link(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="../other/">
- Other post
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"})
-
-
-@pytest.mark.parametrize(
- "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-)
-def test_scan_source_extracts_reaction_properties(
- scanner_module: ModuleType, tmp_path: Path, property_name: str
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- target = f"https://example.com/{property_name}"
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a class="u-url" href="{SOURCE_URL}">
- Permalink
- </a>
-
- <a
- class="u-{property_name}"
- href="{target}"
- >
- Reaction target
- </a>
-
- <div class="e-content">
- <p>Post content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({target})
-
-
-@pytest.mark.parametrize("tag", ("link", "area"))
-def test_scan_source_ignores_non_anchor_href_elements(
- scanner_module: ModuleType, tmp_path: Path, tag: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <{tag} href="https://example.com/target">
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset()
-
-
-def test_scan_source_ignores_self_fragment(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- """
- <a href="#section">
- Section
- </a>
-
- <a href="https://example.com/target">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_creates_source_and_webmentions(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1885,23 +1128,23 @@ def test_unexpected_parser_error_does_not_abort_other_sources(
write_post(tmp_path, "good", '<a href="https://example.com/target">Link</a>')
if parser == "bs4":
- original_soup = scanner_module.BeautifulSoup
+ original_soup = source_parser.BeautifulSoup
def failing_soup(markup: bytes, features: str) -> BeautifulSoup:
if b"/bad/" in markup:
raise failure
return original_soup(markup, features)
- monkeypatch.setattr(scanner_module, "BeautifulSoup", failing_soup)
+ monkeypatch.setattr(source_parser, "BeautifulSoup", failing_soup)
else:
- original_parse = scanner_module.mf2py.parse
+ original_parse = source_parser.mf2py.parse
def failing_parse(*, doc: str, url: str) -> dict[str, Any]:
if "/bad/" in doc:
raise failure
return original_parse(doc=doc, url=url)
- monkeypatch.setattr(scanner_module.mf2py, "parse", failing_parse)
+ monkeypatch.setattr(source_parser.mf2py, "parse", failing_parse)
queued: list[UUID] = []
monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
@@ -1979,46 +1222,6 @@ def test_unexpected_persistence_error_is_not_suppressed(
run_scan(app, scanner_module)
-def test_scan_source_rejects_symlink_outside_root_before_read(
- scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- external = write_post(tmp_path / "outside", "private", "Sensitive content")
- linked = root / "escaped" / "index.html"
- linked.parent.mkdir()
- linked.symlink_to(external)
-
- read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
- monkeypatch.setattr(Path, "read_bytes", read_bytes)
-
- with pytest.raises(
- scanner_module.SourceScanError, match="escapes source directory"
- ):
- scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- read_bytes.assert_not_called()
-
-
-def test_scan_source_accepts_symlink_within_root(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- root = tmp_path / "site"
- root.mkdir()
- linked = write_post(
- root, "example", '<a href="https://example.com/target">Target</a>'
- )
- shared = root / "shared.html"
- linked.rename(shared)
- linked.symlink_to(shared)
-
- scanned = scanner_module.scan_source_file(linked, root=root, base_url=None)
-
- assert scanned.path == "example/index.html"
- assert scanned.url == SOURCE_URL
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
def test_scan_sources_skips_symlink_outside_root(
app: Flask,
scanner_module: ModuleType,
@@ -2043,16 +1246,6 @@ def test_scan_sources_skips_symlink_outside_root(
assert "escapes source directory" in caplog.text
-def test_scan_sources_rejects_malformed_runtime_ignored_hostnames(
- app: Flask, scanner_module: ModuleType, tmp_path: Path
-) -> None:
- configure_scanner(app, tmp_path)
- app.config["IGNORED_HOSTNAMES"] = "example.com"
-
- with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"):
- run_scan(app, scanner_module)
-
-
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -2104,220 +1297,6 @@ def test_scan_sources_rejects_invalid_base_url(
run_scan(app, scanner_module)
-@pytest.mark.parametrize(
- "href",
- (
- "https://dennisfink.me/blog/other/",
- "https://www.dennisfink.me/blog/other/",
- "https://webmentions.dennisfink.me/endpoint",
- "https://foo.bar.dennisfink.me/example",
- ),
-)
-def test_scan_source_ignores_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Ignored
- </a>
-
- <a href="https://example.com/target">
- External
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({"https://example.com/target"})
-
-
-@pytest.mark.parametrize(
- "href",
- (
- "https://notdennisfink.me/example",
- "https://dennisfink.me.example.com/example",
- "https://example.com/example",
- ),
-)
-def test_scan_source_keeps_nonmatching_hostname(
- scanner_module: ModuleType, tmp_path: Path, href: str
-) -> None:
- path = write_post(
- tmp_path,
- "example",
- f"""
- <a href="{href}">
- Target
- </a>
- """,
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset({href})
-
-
-def test_scan_source_ignores_reaction_to_matching_hostname(
- scanner_module: ModuleType, tmp_path: Path
-) -> None:
- directory = tmp_path / "example"
- directory.mkdir()
-
- path = directory / "index.html"
- path.write_text(
- f"""
- <!doctype html>
- <html>
- <head>
- <link
- rel="canonical"
- href="{SOURCE_URL}"
- >
- </head>
- <body>
- <article class="h-entry">
- <a
- class="u-url"
- href="{SOURCE_URL}"
- >
- Permalink
- </a>
-
- <a
- class="u-in-reply-to"
- href="https://www.dennisfink.me/blog/other/"
- >
- Reply target
- </a>
-
- <div class="e-content">
- <p>Reply content.</p>
- </div>
- </article>
- </body>
- </html>
- """,
- encoding="utf-8",
- )
-
- scanned = scanner_module.scan_source_file(
- path,
- root=tmp_path,
- base_url=None,
- ignored_hostnames=("dennisfink.me", "*.dennisfink.me"),
- )
-
- assert scanned.targets == frozenset()
-
-
-def test_canonical_url_rejects_non_http_resolved_url(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- document = parse_html(
- """
- <html>
- <head>
- <link
- rel="canonical"
- href="mailto:example@example.com"
- >
- </head>
- </html>
- """
- )
-
- with pytest.raises(
- scanner_module.SourceScanError,
- match="Canonical URL is not a valid HTTP or HTTPS URL",
- ):
- scanner_module.canonical_url(
- document, relative_path=Path("example/index.html"), base_url=BASE_URL
- )
-
-
-def test_property_urls_ignores_non_mapping_properties(
- scanner_module: ModuleType,
-) -> None:
- entry = {"properties": "invalid"}
-
- assert tuple(scanner_module.property_urls(entry, "url")) == ()
-
-
-@pytest.mark.parametrize(
- ("parsed", "message"),
- [
- pytest.param(
- [],
- "Microformats parser did not return a document object",
- id="non-object-document",
- ),
- pytest.param(
- {"items": [None]}, "Parsed h-entry is not an object", id="non-object-entry"
- ),
- pytest.param(
- {"items": [{"type": ["h-card"]}]},
- "Parsed microformats item is not an h-entry",
- id="wrong-item-type",
- ),
- ],
-)
-def test_parse_entry_rejects_invalid_parser_output(
- scanner_module: ModuleType, monkeypatch: MonkeyPatch, parsed: object, message: str
-) -> None:
- document = parse_html(
- """
- <article class="h-entry">
- <p>Hello world</p>
- </article>
- """
- )
-
- element = document.find(class_="h-entry")
-
- assert element is not None
-
- monkeypatch.setattr(scanner_module.mf2py, "parse", lambda **kwargs: parsed)
-
- with pytest.raises(scanner_module.SourceScanError, match=message):
- scanner_module.parse_entry(element, SOURCE_URL)
-
-
-def test_normalize_target_ignores_empty_target(scanner_module: ModuleType) -> None:
- assert (
- scanner_module.normalize_target(" ", base_url=SOURCE_URL, source_url=SOURCE_URL)
- is None
- )
-
-
-def test_normalize_target_ignores_url_rejected_by_validator(
- scanner_module: ModuleType,
-) -> None:
- scanner_module.is_http_url.return_value = False
-
- assert (
- scanner_module.normalize_target(
- "https://example.com/target", base_url=SOURCE_URL, source_url=SOURCE_URL
- )
- is None
- )
-
-
def test_changed_source_url_is_rejected(
app: Flask,
scanner_module: ModuleType,