aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 19:23:17 +0100
committerDennis Fink2026-10-30 19:23:17 +0100
commitdcbbfdb38d4e18e377d4e326d82f35341840c6cc (patch)
tree110d3dad8fcd0c875d86fbc630d8626f1a01de7b
parent1a9e7a49240533bb7b7f7b8fd0527f12d2a029bf (diff)
downloadwebmentions-ssg-dcbbfdb38d4e18e377d4e326d82f35341840c6cc.tar.gz
webmentions-ssg-dcbbfdb38d4e18e377d4e326d82f35341840c6cc.zip
fix(sender): stabilize source content fingerprints
Hash normalized reader-facing content and outgoing targets instead of raw h-entry markup so generated attributes and relative timestamps do not trigger spurious source revisions or repeat Webmentions. Preserve meaningful changes such as image updates and reaction target changes while ignoring target ordering and other markup noise.
-rw-r--r--tests/tasks/test_scanner.py167
-rw-r--r--webmentions_ssg/tasks/scanner.py96
2 files changed, 250 insertions, 13 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 21a13c6..7aefff4 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -508,6 +508,119 @@ def test_scan_source_uses_canonical_without_base_url(
assert len(scanned.content_hash) == 32
+def test_scan_source_ignores_markup_noise_in_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ "<p>Hello <strong>world</strong>!</p>"
+ '<a href="https://example.com/target">Target</a>',
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ write_post(
+ tmp_path,
+ "example",
+ '<p class="generated" data-build="123"> Hello world! </p>'
+ '<a class="external" href="https://example.com/target"> Target </a>',
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_uses_stable_time_value_inside_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(
+ tmp_path,
+ "example",
+ '<time datetime="2026-10-30T12:00:00Z">5 minutes ago</time>',
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash == second.content_hash
+
+
+def test_scan_source_hash_changes_when_image_changes(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", '<img src="/images/a.jpg" alt="A photo">')
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_includes_reaction_targets_outside_content(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<a class="u-like-of" href="https://example.com/first">Like</a>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ "https://example.com/first", "https://example.com/second"
+ ),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == frozenset({"https://example.com/first"})
+ assert second.targets == frozenset({"https://example.com/second"})
+ assert first.content_hash != second.content_hash
+
+
+def test_scan_source_hash_ignores_reaction_target_order(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ path = write_post(tmp_path, "example", "<p>Same content.</p>")
+ html = path.read_text(encoding="utf-8")
+ first_link = '<a class="u-like-of" href="https://example.com/a">A</a>'
+ second_link = '<a class="u-like-of" href="https://example.com/b">B</a>'
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ first_link + second_link + '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+ first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ first_link + second_link, second_link + first_link
+ ),
+ encoding="utf-8",
+ )
+ second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None)
+
+ assert first.targets == second.targets
+ assert first.content_hash == second.content_hash
+
+
def test_scan_source_falls_back_to_base_url(
scanner_module: ModuleType, tmp_path: Path
) -> None:
@@ -976,6 +1089,56 @@ def test_unchanged_source_does_not_increment_revision(
assert queued == []
+def test_changing_relative_timestamp_does_not_resend(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ configure_scanner(app, tmp_path)
+ path = write_post(
+ tmp_path, "example", '<a href="https://example.com/target">Target</a>'
+ )
+ html = path.read_text(encoding="utf-8")
+ path.write_text(
+ html.replace(
+ '<div class="e-content">',
+ '<time class="dt-published" datetime="2026-10-30">5 minutes ago</time>'
+ '<div class="e-content">',
+ ),
+ encoding="utf-8",
+ )
+
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ source = db.session.scalar(sa.select(Source))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+ assert source is not None and webmention is not None
+ original_hash = source.content_hash
+ webmention.processed_revision = 1
+ webmention.sent_revision = 1
+ webmention.status = SentWebmentionStatus.SENT
+ db.session.commit()
+
+ queued.clear()
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"),
+ encoding="utf-8",
+ )
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ source = db.session.scalar(sa.select(Source))
+ webmention = db.session.scalar(sa.select(SentWebmention))
+ assert source is not None and webmention is not None
+ assert source.content_hash == original_hash
+ assert source.revision == 1
+ assert webmention.desired_revision == 1
+ assert webmention.processed_revision == 1
+
+ assert queued == []
+
+
def test_updated_source_increments_revision(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1597,7 +1760,6 @@ def test_pending_webmention_is_requeued_on_next_scan(
assert queued == [identifier]
-
def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
@@ -1631,7 +1793,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget
queued.clear()
# Changing the h-entry creates a new desired revision with a fresh budget.
- path.write_text(path.read_text().replace('>A</a>', '>A updated</a>'))
+ path.write_text(path.read_text().replace(">A</a>", ">A updated</a>"))
with app.app_context():
webmention = db.session.get(SentWebmention, identifier)
assert webmention is not None
@@ -1647,6 +1809,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget
assert webmention.attempt_count == 0
assert webmention.next_retry_at is None
+
def test_invalid_known_source_is_not_deleted(
app: Flask,
scanner_module: ModuleType,
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 298fed7..68c35be 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Scan generated site pages to track outgoing Webmention targets."""
+import json
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit
import mf2py
import sqlalchemy as sa
import xxhash
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup, NavigableString, Tag
+from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
@@ -29,6 +31,42 @@ from .sender import send_webmention
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
class SourceScanError(Exception):
"""A source document cannot safely be processed."""
@@ -267,6 +305,30 @@ def iter_targets(
yield target
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
def scan_source_file(
path: Path,
*,
@@ -304,19 +366,31 @@ def scan_source_file(
content = content_element(entry_element)
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ # Fingerprint the meaningful post text and its outgoing targets rather than
+ # the full h-entry markup. Relative timestamps, classes, and other volatile
+ # metadata outside e-content must not cause repeat Webmentions.
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
return ScannedSource(
path=relative_path.as_posix(),
url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")),
- targets=frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- ),
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
)