diff options
| -rw-r--r-- | tests/tasks/test_scanner.py | 167 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 96 |
2 files changed, 250 insertions, 13 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 21a13c6..7aefff4 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -508,6 +508,119 @@ def test_scan_source_uses_canonical_without_base_url( assert len(scanned.content_hash) == 32 +def test_scan_source_ignores_markup_noise_in_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "<p>Hello <strong>world</strong>!</p>" + '<a href="https://example.com/target">Target</a>', + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '<p class="generated" data-build="123"> Hello world! </p>' + '<a class="external" href="https://example.com/target"> Target </a>', + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>', + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">') + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '<div class="e-content">', + '<a class="u-like-of" href="https://example.com/first">Like</a>' + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "<p>Same content.</p>") + html = path.read_text(encoding="utf-8") + first_link = '<a class="u-like-of" href="https://example.com/a">A</a>' + second_link = '<a class="u-like-of" href="https://example.com/b">B</a>' + path.write_text( + html.replace( + '<div class="e-content">', + first_link + second_link + '<div class="e-content">', + ), + encoding="utf-8", + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + def test_scan_source_falls_back_to_base_url( scanner_module: ModuleType, tmp_path: Path ) -> None: @@ -976,6 +1089,56 @@ def test_unchanged_source_does_not_increment_revision( assert queued == [] +def test_changing_relative_timestamp_does_not_resend( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + path = write_post( + tmp_path, "example", '<a href="https://example.com/target">Target</a>' + ) + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '<div class="e-content">', + '<time class="dt-published" datetime="2026-10-30">5 minutes ago</time>' + '<div class="e-content">', + ), + encoding="utf-8", + ) + + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + assert source is not None and webmention is not None + original_hash = source.content_hash + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + db.session.commit() + + queued.clear() + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + assert source is not None and webmention is not None + assert source.content_hash == original_hash + assert source.revision == 1 + assert webmention.desired_revision == 1 + assert webmention.processed_revision == 1 + + assert queued == [] + + def test_updated_source_increments_revision( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1597,7 +1760,6 @@ def test_pending_webmention_is_requeued_on_next_scan( assert queued == [identifier] - def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1631,7 +1793,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget queued.clear() # Changing the h-entry creates a new desired revision with a fresh budget. - path.write_text(path.read_text().replace('>A</a>', '>A updated</a>')) + path.write_text(path.read_text().replace(">A</a>", ">A updated</a>")) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None @@ -1647,6 +1809,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget assert webmention.attempt_count == 0 assert webmention.next_retry_at is None + def test_invalid_known_source_is_not_deleted( app: Flask, scanner_module: ModuleType, diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 298fed7..68c35be 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -4,6 +4,7 @@ """Scan generated site pages to track outgoing Webmention targets.""" +import json from collections.abc import Iterator from dataclasses import dataclass from datetime import UTC, datetime @@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit import mf2py import sqlalchemy as sa import xxhash -from bs4 import BeautifulSoup, Tag +from bs4 import BeautifulSoup, NavigableString, Tag +from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload @@ -29,6 +31,42 @@ from .sender import send_webmention REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + class SourceScanError(Exception): """A source document cannot safely be processed.""" @@ -267,6 +305,30 @@ def iter_targets( yield target +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + def scan_source_file( path: Path, *, @@ -304,19 +366,31 @@ def scan_source_file( content = content_element(entry_element) + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + # Fingerprint the meaningful post text and its outgoing targets rather than + # the full h-entry markup. Relative timestamps, classes, and other volatile + # metadata outside e-content must not cause repeat Webmentions. + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + return ScannedSource( path=relative_path.as_posix(), url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")), - targets=frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ), + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, ) |
