diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 96 |
1 files changed, 85 insertions, 11 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 298fed7..68c35be 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -4,6 +4,7 @@ """Scan generated site pages to track outgoing Webmention targets.""" +import json from collections.abc import Iterator from dataclasses import dataclass from datetime import UTC, datetime @@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit import mf2py import sqlalchemy as sa import xxhash -from bs4 import BeautifulSoup, Tag +from bs4 import BeautifulSoup, NavigableString, Tag +from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload @@ -29,6 +31,42 @@ from .sender import send_webmention REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") +CONTENT_BLOCK_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "ol", + "p", + "pre", + "section", + "table", + "td", + "th", + "tr", + "ul", + } +) + class SourceScanError(Exception): """A source document cannot safely be processed.""" @@ -267,6 +305,30 @@ def iter_targets( yield target +def normalized_content(content: Tag) -> str: + """Extract stable reader-facing content without generated HTML attributes.""" + + def extract(node: PageElement) -> str: + if isinstance(node, NavigableString): + return str(node) + if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: + return "" + + # A relative-time label may change on every site build even though its + # machine-readable date stays the same. + if node.name == "time" and isinstance(value := node.get("datetime"), str): + return value.strip() + + # Image-only posts also have meaningful content, despite having no text. + if node.name == "img": + return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " + + text = "".join(extract(child) for child in node.children) + return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + + return " ".join(extract(content).split()) + + def scan_source_file( path: Path, *, @@ -304,19 +366,31 @@ def scan_source_file( content = content_element(entry_element) + targets = frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ) + + # Fingerprint the meaningful post text and its outgoing targets rather than + # the full h-entry markup. Relative timestamps, classes, and other volatile + # metadata outside e-content must not cause repeat Webmentions. + stable_content = normalized_content(content) + fingerprint = json.dumps( + {"content": stable_content, "targets": sorted(targets)}, + ensure_ascii=False, + separators=(",", ":"), + ) + return ScannedSource( path=relative_path.as_posix(), url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")), - targets=frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ), + content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), + targets=targets, ) |
