aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 19:23:17 +0100
committerDennis Fink2026-10-30 19:23:17 +0100
commitdcbbfdb38d4e18e377d4e326d82f35341840c6cc (patch)
tree110d3dad8fcd0c875d86fbc630d8626f1a01de7b /webmentions_ssg/tasks/scanner.py
parent1a9e7a49240533bb7b7f7b8fd0527f12d2a029bf (diff)
downloadwebmentions-ssg-dcbbfdb38d4e18e377d4e326d82f35341840c6cc.tar.gz
webmentions-ssg-dcbbfdb38d4e18e377d4e326d82f35341840c6cc.zip
fix(sender): stabilize source content fingerprints
Hash normalized reader-facing content and outgoing targets instead of raw h-entry markup so generated attributes and relative timestamps do not trigger spurious source revisions or repeat Webmentions. Preserve meaningful changes such as image updates and reaction target changes while ignoring target ordering and other markup noise.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/scanner.py96
1 files changed, 85 insertions, 11 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 298fed7..68c35be 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Scan generated site pages to track outgoing Webmention targets."""
+import json
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit
import mf2py
import sqlalchemy as sa
import xxhash
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup, NavigableString, Tag
+from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
@@ -29,6 +31,42 @@ from .sender import send_webmention
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
class SourceScanError(Exception):
"""A source document cannot safely be processed."""
@@ -267,6 +305,30 @@ def iter_targets(
yield target
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
def scan_source_file(
path: Path,
*,
@@ -304,19 +366,31 @@ def scan_source_file(
content = content_element(entry_element)
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ # Fingerprint the meaningful post text and its outgoing targets rather than
+ # the full h-entry markup. Relative timestamps, classes, and other volatile
+ # metadata outside e-content must not cause repeat Webmentions.
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
return ScannedSource(
path=relative_path.as_posix(),
url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")),
- targets=frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- ),
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
)