From dcbbfdb38d4e18e377d4e326d82f35341840c6cc Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 19:23:17 +0100 Subject: fix(sender): stabilize source content fingerprints Hash normalized reader-facing content and outgoing targets instead of raw h-entry markup so generated attributes and relative timestamps do not trigger spurious source revisions or repeat Webmentions. Preserve meaningful changes such as image updates and reaction target changes while ignoring target ordering and other markup noise. --- webmentions_ssg/tasks/scanner.py | 96 +++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 11 deletions(-) (limited to 'webmentions_ssg/tasks/scanner.py') diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 298fed7..68c35be 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -4,6 +4,7 @@ """Scan generated site pages to track outgoing Webmention targets.""" +import json from collections.abc import Iterator from dataclasses import dataclass from datetime import UTC, datetime @@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit import mf2py import sqlalchemy as sa import xxhash -from bs4 import BeautifulSoup, Tag +from bs4 import BeautifulSoup, NavigableString, Tag +from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload @@ -29,6 +31,42 @@ from .sender import send_webmention REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + class SourceScanError(Exception): """A source document cannot safely be processed.""" @@ -267,6 +305,30 @@ def iter_targets( yield target +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + def scan_source_file( path: Path, *, @@ -304,19 +366,31 @@ def scan_source_file( content = content_element(entry_element) + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + # Fingerprint the meaningful post text and its outgoing targets rather than + # the full h-entry markup. Relative timestamps, classes, and other volatile + # metadata outside e-content must not cause repeat Webmentions. + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + return ScannedSource( path=relative_path.as_posix(), url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")), - targets=frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ), + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, ) -- cgit v1.3.1