aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/tasks/scanner.py96
1 files changed, 85 insertions, 11 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 298fed7..68c35be 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Scan generated site pages to track outgoing Webmention targets."""
+import json
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -16,7 +17,8 @@ from urllib.parse import quote, urldefrag, urljoin, urlsplit
import mf2py
import sqlalchemy as sa
import xxhash
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup, NavigableString, Tag
+from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
@@ -29,6 +31,42 @@ from .sender import send_webmention
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
class SourceScanError(Exception):
"""A source document cannot safely be processed."""
@@ -267,6 +305,30 @@ def iter_targets(
yield target
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
def scan_source_file(
path: Path,
*,
@@ -304,19 +366,31 @@ def scan_source_file(
content = content_element(entry_element)
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ # Fingerprint the meaningful post text and its outgoing targets rather than
+ # the full h-entry markup. Relative timestamps, classes, and other volatile
+ # metadata outside e-content must not cause repeat Webmentions.
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
return ScannedSource(
path=relative_path.as_posix(),
url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")),
- targets=frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- ),
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
)