aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/source_parser.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/source_parser.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg/tasks/source_parser.py')
-rw-r--r--webmentions_ssg/tasks/source_parser.py390
1 files changed, 390 insertions, 0 deletions
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
new file mode 100644
index 0000000..cb15ab6
--- /dev/null
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -0,0 +1,390 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Extract canonical source URLs, content fingerprints, and outgoing links."""
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from itertools import chain
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urldefrag, urljoin, urlsplit
+
+import mf2py
+import xxhash
+from bs4 import BeautifulSoup, Tag
+from bs4.element import NavigableString, PageElement
+
+from ..net import is_http_url
+
+REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
+
+CONTENT_BLOCK_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "br",
+ "dd",
+ "div",
+ "dl",
+ "dt",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hr",
+ "li",
+ "main",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "td",
+ "th",
+ "tr",
+ "ul",
+ }
+)
+
+
+class SourceScanError(Exception):
+ """A source document cannot safely be processed."""
+
+
+@dataclass(frozen=True)
+class ScannedSource:
+ path: str
+ url: str
+ content_hash: str
+ targets: frozenset[str]
+
+
+def source_url_for_path(relative_path: Path, base_url: str) -> str:
+ """
+ Derive the public source URL from a relative filesystem path.
+
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Base URL under which source documents are published.
+ :return: Public URL corresponding to the source document.
+ """
+ directory = relative_path.parent.as_posix()
+
+ return urljoin(base_url, f"{quote(directory, safe='/')}/")
+
+
+def canonical_url(
+ document: BeautifulSoup, *, relative_path: Path, base_url: str | None
+) -> str | None:
+ """
+ Return the canonical URL declared by a source document.
+
+ Relative canonical URLs are resolved against the configured source base URL.
+
+ :param document: Parsed HTML source document.
+ :param relative_path: Path of the source document relative to the source root.
+ :param base_url: Configured source base URL, if available.
+ :return: Canonical URL, or ``None`` if none is declared.
+ :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
+ or does not resolve to an HTTP or HTTPS URL.
+ """
+ if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
+ href = link.get("href")
+
+ if not isinstance(href, str) or not (href := href.strip()):
+ raise SourceScanError("Document contains an empty canonical URL")
+
+ if is_http_url(href):
+ return href
+
+ if base_url is None:
+ raise SourceScanError(
+ "Document contains a relative canonical URL, but "
+ "SOURCE_BASE_URL is not configured"
+ )
+
+ resolved = urljoin(source_url_for_path(relative_path, base_url), href)
+
+ if not is_http_url(resolved):
+ raise SourceScanError(
+ f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
+ )
+
+ return resolved
+
+
+def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+ """
+ Yield URL values from a microformats property.
+
+ :param entry: Parsed microformats entry.
+ :param property_name: Name of the property to inspect.
+ :return: Iterator over string values of the property.
+ """
+ properties = entry.get("properties")
+
+ if not isinstance(properties, dict):
+ return
+
+ values = properties.get(property_name)
+
+ if not isinstance(values, list):
+ return
+
+ yield from (value for value in values if isinstance(value, str))
+
+
+def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+ """
+ Parse and validate an h-entry with mf2py.
+
+ :param element: HTML element containing the h-entry.
+ :param base_url: Base URL used when parsing the microformats data.
+ :return: Parsed h-entry.
+ :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
+ """
+ parsed = mf2py.parse(doc=str(element), url=base_url)
+
+ if not isinstance(parsed, dict):
+ raise SourceScanError("Microformats parser did not return a document object")
+
+ items = parsed.get("items")
+
+ if not isinstance(items, list) or len(items) != 1:
+ raise SourceScanError("Expected exactly one parsed h-entry")
+
+ entry = items[0]
+
+ if not isinstance(entry, dict):
+ raise SourceScanError("Parsed h-entry is not an object")
+
+ types = entry.get("type")
+
+ if not isinstance(types, list) or "h-entry" not in types:
+ raise SourceScanError("Parsed microformats item is not an h-entry")
+
+ return entry
+
+
+def primary_entry(
+ document: BeautifulSoup, source_url: str
+) -> tuple[Tag, dict[str, Any]]:
+ """
+ Return and validate the primary h-entry of a source document.
+
+ The document must contain exactly one h-entry whose ``u-url`` matches the
+ source URL.
+
+ :param document: Parsed HTML source document.
+ :param source_url: Public URL of the source document.
+ :return: h-entry element and its parsed microformats representation.
+ :raises SourceScanError: If the document does not contain exactly one h-entry
+ or its ``u-url`` does not match the source URL.
+ """
+ entries = document.find_all(class_="h-entry")
+
+ if len(entries) != 1:
+ raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
+
+ entry_element = entries[0]
+ mf2_entry = parse_entry(entry_element, source_url)
+
+ urls = tuple(property_urls(mf2_entry, "url"))
+
+ if source_url not in urls:
+ raise SourceScanError(
+ f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
+ )
+
+ return entry_element, mf2_entry
+
+
+def content_element(entry: Tag) -> Tag:
+ """
+ Return the e-content element of an h-entry.
+
+ :param entry: HTML element containing the h-entry.
+ :return: The h-entry's e-content element.
+ :raises SourceScanError: If the h-entry does not contain exactly one
+ e-content element.
+ """
+ contents = entry.find_all(class_="e-content")
+
+ if len(contents) != 1:
+ raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
+
+ return contents[0]
+
+
+def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+ """
+ Resolve and validate a possible Webmention target URL.
+
+ Empty values, non-HTTP URLs, and URLs referring to the source itself are
+ discarded.
+
+ :param value: URL reference to normalize.
+ :param base_url: Base URL against which relative references are resolved.
+ :param source_url: URL of the source document.
+ :return: Normalized target URL, or ``None`` if the value is not a valid
+ Webmention target.
+ """
+ value = value.strip()
+
+ if not value:
+ return None
+
+ target = urljoin(base_url, value)
+
+ if not is_http_url(target):
+ return None
+
+ if urldefrag(target)[0] == urldefrag(source_url)[0]:
+ return None
+
+ return target
+
+
+def iter_targets(
+ content: Tag,
+ mf2_entry: dict[str, Any],
+ *,
+ base_url: str,
+ source_url: str,
+ ignored_hostnames: Iterable[str] = (),
+) -> Iterator[str]:
+ """
+ Yield outgoing Webmention targets from an h-entry.
+
+ Targets are collected from links in the entry content and supported
+ microformats reaction properties. Invalid, self-referencing, and ignored
+ targets are excluded.
+
+ :param content: e-content element of the h-entry.
+ :param mf2_entry: Parsed microformats representation of the h-entry.
+ :param base_url: Base URL against which relative targets are resolved.
+ :param source_url: URL of the source document.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Iterator over outgoing Webmention target URLs.
+ """
+ hrefs = (
+ href
+ for element in content.find_all("a", href=True)
+ if isinstance(href := element.get("href"), str)
+ )
+
+ reactions = chain.from_iterable(
+ property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ )
+
+ for value in chain(hrefs, reactions):
+ target = normalize_target(value, base_url=base_url, source_url=source_url)
+ if target is not None:
+ hostname = (urlsplit(target).hostname or "").rstrip(".")
+ if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
+ yield target
+
+
+def normalized_content(content: Tag) -> str:
+ """Extract stable reader-facing content without generated HTML attributes."""
+
+ def extract(node: PageElement) -> str:
+ if isinstance(node, NavigableString):
+ return str(node)
+
+ if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
+ return ""
+
+ # A relative-time label may change on every site build even though its
+ # machine-readable date stays the same.
+ if node.name == "time" and isinstance(value := node.get("datetime"), str):
+ return value.strip()
+
+ # Image-only posts also have meaningful content, despite having no text.
+ if node.name == "img":
+ return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
+
+ text = "".join(extract(child) for child in node.children)
+ return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+
+ return " ".join(extract(content).split())
+
+
+def scan_source_file(
+ path: Path,
+ *,
+ root: Path,
+ base_url: str | None,
+ ignored_hostnames: Iterable[str] = (),
+) -> ScannedSource:
+ """
+ Scan one generated source document for outgoing Webmentions.
+
+ :param path: Path of the generated HTML document.
+ :param root: Root directory containing generated source documents.
+ :param base_url: Configured source base URL, if available.
+ :param ignored_hostnames: Hostname patterns whose targets should be ignored.
+ :return: Scanned source metadata and discovered targets.
+ :raises SourceScanError: If the document cannot be safely interpreted as a
+ Webmention source.
+ :raises OSError: If the source document cannot be read.
+ """
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
+
+ relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
+
+ source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+
+ if source_url is None:
+ if base_url is None:
+ raise SourceScanError(
+ "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ )
+
+ source_url = source_url_for_path(relative_path, base_url)
+
+ entry_element, mf2_entry = primary_entry(document, source_url)
+
+ content = content_element(entry_element)
+
+ targets = frozenset(
+ iter_targets(
+ content,
+ mf2_entry,
+ base_url=source_url,
+ source_url=source_url,
+ ignored_hostnames=ignored_hostnames,
+ )
+ )
+
+ stable_content = normalized_content(content)
+ fingerprint = json.dumps(
+ {"content": stable_content, "targets": sorted(targets)},
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+ return ScannedSource(
+ path=relative_path.as_posix(),
+ url=source_url,
+ content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
+ targets=targets,
+ )