aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tests/tasks/test_scanner.py167
-rw-r--r--webmentions_ssg/tasks/scanner.py96
2 files changed, 250 insertions, 13 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 21a13c6..7aefff4 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -508,6 +508,119 @@ def test_scan_source_uses_canonical_without_base_url(
assert len(scanned.content_hash) == 32
+def test_scan_source_ignores_markup_noise_in_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ "<p>Hello <strong>world</strong>!</p>"
+ '<a href="https://example.com/target">Target</a>',
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ write_post(
+ tmp_path,
+ "example",
+ '<p class="generated" data-build="123"> Hello world! </p>'
+ '<a class="external" href="https://example.com/target"> Target </a>',
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_uses_stable_time_value_inside_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_hash_changes_when_image_changes(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_includes_reaction_targets_outside_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<a class="u-like-of" href="https://example.com/first">Like</a>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ "https://example.com/first", "https://example.com/second"
+ ),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == frozenset({"https://example.com/first"})
+ assert second.targets == frozenset({"https://example.com/second"})
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_ignores_reaction_target_order(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
+ second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ first_link + second_link + '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ first_link + second_link, second_link + first_link
+ ),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
def test_scan_source_falls_back_to_base_url(
scanner_module: ModuleType, tmp_path: Path
) -> None:
@@ -976,6 +1089,56 @@ def test_unchanged_source_does_not_increment_revision(
assert queued == []
+def test_changing_relative_timestamp_does_not_resend(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ configure_scanner(app, tmp_path)
+ path = write_post(
+ tmp_path, "example", '<a href="https://example.com/target">Target</a>'
+ )
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<time class="dt-published" datetime="2026-10-30">5 minutes ago</time>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ source = db.session.scalar(sa.select(Source))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+ assert source is not None and webmention is not None
+ original_hash = source.content_hash
+ webmention.processed_revision = 1
+ webmention.sent_revision = 1
+ webmention.status = SentWebmentionStatus.SENT
+ db.session.commit()
+
+ queued.clear()
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ source = db.session.scalar(sa.select(Source))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+ assert source is not None and webmention is not None
+ assert source.content_hash == original_hash
+ assert source.revision == 1
+ assert webmention.desired_revision == 1
+ assert webmention.processed_revision == 1
+
+ assert queued == []
+
+
def test_updated_source_increments_revision(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1597,7 +1760,6 @@ def test_pending_webmention_is_requeued_on_next_scan(
assert queued == [identifier]
-
def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1631,7 +1793,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget
queued.clear()
# Changing the h-entry creates a new desired revision with a fresh budget.
- path.write_text(path.read_text().replace('>A</a>', '>A updated</a>'))
+ path.write_text(path.read_text().replace(">A</a>", ">A updated</a>"))
with app.app_context():
webmention = db.session.get(SentWebmention, identifier)
assert webmention is not None
@@ -1647,6 +1809,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget
assert webmention.attempt_count == 0
assert webmention.next_retry_at is None
+
def test_invalid_known_source_is_not_deleted(
app: Flask,
scanner_module: ModuleType,
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 298fed7..68c35be 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Scan generated site pages to track outgoing Webmention targets."""
+import json
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit
import mf2py
import sqlalchemy as sa
import xxhash
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup, NavigableString, Tag
+from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
@@ -29,6 +31,42 @@ from .sender import send_webmention
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
class SourceScanError(Exception):
"""A source document cannot safely be processed."""
@@ -267,6 +305,30 @@ def iter_targets(
yield target
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
def scan_source_file(
path: Path,
*,
@@ -304,19 +366,31 @@ def scan_source_file(
content = content_element(entry_element)
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ # Fingerprint the meaningful post text and its outgoing targets rather than
+ # the full h-entry markup. Relative timestamps, classes, and other volatile
+ # metadata outside e-content must not cause repeat Webmentions.
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
return ScannedSource(
path=relative_path.as_posix(),
url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")),
- targets=frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- ),
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
)