diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 414 |
1 files changed, 10 insertions, 404 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 666eba7..b2d6def 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -2,404 +2,22 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Scan generated site pages to track outgoing Webmention targets.""" +"""Reconcile scanned static-site sources with the database.""" -import json -from collections.abc import Iterator -from dataclasses import dataclass from datetime import UTC, datetime -from fnmatch import fnmatchcase -from itertools import chain from pathlib import Path -from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit -import mf2py import sqlalchemy as sa -import xxhash -from bs4 import BeautifulSoup, NavigableString, Tag -from bs4.element import PageElement from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload -from .. import DATABASE as db -from .. import HUEY as huey -from ..config import normalize_ignored_hostnames +from ..extensions import DATABASE as db +from ..extensions import HUEY as huey from ..models import SentWebmention, Source -from ..url_security import is_http_url +from ..net import is_http_url from .sender import send_webmention - -REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") - -CONTENT_BLOCK_TAGS = frozenset( - { - "address", - "article", - "aside", - "blockquote", - "br", - "dd", - "div", - "dl", - "dt", - "figcaption", - "figure", - "footer", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "header", - "hr", - "li", - "main", - "ol", - "p", - "pre", - "section", - "table", - "td", - "th", - "tr", - "ul", - } -) - - -class SourceScanError(Exception): - """A source document cannot safely be processed.""" - - -@dataclass(frozen=True) -class ScannedSource: - path: str - url: str - content_hash: str - targets: frozenset[str] - - -def source_url_for_path(relative_path: Path, base_url: str) -> str: - """ - Derive the public source URL from a relative filesystem path. - - :param relative_path: Path of the source document relative to the source root. - :param base_url: Base URL under which source documents are published. - :return: Public URL corresponding to the source document. - """ - directory = relative_path.parent.as_posix() - - return urljoin(base_url, f"{quote(directory, safe='/')}/") - - -def canonical_url( - document: BeautifulSoup, *, relative_path: Path, base_url: str | None -) -> str | None: - """ - Return the canonical URL declared by a source document. - - Relative canonical URLs are resolved against the configured source base URL. - - :param document: Parsed HTML source document. - :param relative_path: Path of the source document relative to the source root. - :param base_url: Configured source base URL, if available. - :return: Canonical URL, or ``None`` if none is declared. - :raises SourceScanError: If the canonical URL is empty, cannot be resolved, - or does not resolve to an HTTP or HTTPS URL. - """ - if (link := document.select_one('link[rel~="canonical"][href]')) is not None: - href = link.get("href") - - if not isinstance(href, str) or not (href := href.strip()): - raise SourceScanError("Document contains an empty canonical URL") - - if is_http_url(href): - return href - - if base_url is None: - raise SourceScanError( - "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" - ) - - resolved = urljoin(source_url_for_path(relative_path, base_url), href) - - if not is_http_url(resolved): - raise SourceScanError( - f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" - ) - - return resolved - - -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """ - Yield URL values from a microformats property. - - :param entry: Parsed microformats entry. - :param property_name: Name of the property to inspect. - :return: Iterator over string values of the property. - """ - properties = entry.get("properties") - - if not isinstance(properties, dict): - return - - values = properties.get(property_name) - - if not isinstance(values, list): - return - - yield from (value for value in values if isinstance(value, str)) - - -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """ - Parse and validate an h-entry with mf2py. - - :param element: HTML element containing the h-entry. - :param base_url: Base URL used when parsing the microformats data. - :return: Parsed h-entry. - :raises SourceScanError: If parsing does not produce exactly one valid h-entry. - """ - parsed = mf2py.parse(doc=str(element), url=base_url) - - if not isinstance(parsed, dict): - raise SourceScanError("Microformats parser did not return a document object") - - items = parsed.get("items") - - if not isinstance(items, list) or len(items) != 1: - raise SourceScanError("Expected exactly one parsed h-entry") - - entry = items[0] - - if not isinstance(entry, dict): - raise SourceScanError("Parsed h-entry is not an object") - - types = entry.get("type") - - if not isinstance(types, list) or "h-entry" not in types: - raise SourceScanError("Parsed microformats item is not an h-entry") - - return entry - - -def primary_entry( - document: BeautifulSoup, source_url: str -) -> tuple[Tag, dict[str, Any]]: - """ - Return and validate the primary h-entry of a source document. - - The document must contain exactly one h-entry whose ``u-url`` matches the - source URL. - - :param document: Parsed HTML source document. - :param source_url: Public URL of the source document. - :return: h-entry element and its parsed microformats representation. - :raises SourceScanError: If the document does not contain exactly one h-entry - or its ``u-url`` does not match the source URL. - """ - entries = document.find_all(class_="h-entry") - - if len(entries) != 1: - raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") - - entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) - - urls = tuple(property_urls(mf2_entry, "url")) - - if source_url not in urls: - raise SourceScanError( - f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" - ) - - return entry_element, mf2_entry - - -def content_element(entry: Tag) -> Tag: - """ - Return the e-content element of an h-entry. - - :param entry: HTML element containing the h-entry. - :return: The h-entry's e-content element. - :raises SourceScanError: If the h-entry does not contain exactly one - e-content element. - """ - contents = entry.find_all(class_="e-content") - - if len(contents) != 1: - raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") - - return contents[0] - - -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """ - Resolve and validate a possible Webmention target URL. - - Empty values, non-HTTP URLs, and URLs referring to the source itself are - discarded. - - :param value: URL reference to normalize. - :param base_url: Base URL against which relative references are resolved. - :param source_url: URL of the source document. - :return: Normalized target URL, or ``None`` if the value is not a valid - Webmention target. - """ - value = value.strip() - - if not value: - return None - - target = urljoin(base_url, value) - - if not is_http_url(target): - return None - - if urldefrag(target)[0] == urldefrag(source_url)[0]: - return None - - return target - - -def iter_targets( - content: Tag, - mf2_entry: dict[str, Any], - *, - base_url: str, - source_url: str, - ignored_hostnames: tuple[str, ...] = (), -) -> Iterator[str]: - """ - Yield outgoing Webmention targets from an h-entry. - - Targets are collected from links in the entry content and supported - microformats reaction properties. Invalid, self-referencing, and ignored - targets are excluded. - - :param content: e-content element of the h-entry. - :param mf2_entry: Parsed microformats representation of the h-entry. - :param base_url: Base URL against which relative targets are resolved. - :param source_url: URL of the source document. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Iterator over outgoing Webmention target URLs. - """ - hrefs = ( - href - for element in content.find_all("a", href=True) - if isinstance(href := element.get("href"), str) - ) - - reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES - ) - - for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) - if target is not None: - hostname = urlsplit(target).hostname or "" - if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): - yield target - - -def normalized_content(content: Tag) -> str: - """Extract stable reader-facing content without generated HTML attributes.""" - - def extract(node: PageElement) -> str: - if isinstance(node, NavigableString): - return str(node) - if not isinstance(node, Tag) or node.name in {"script", "style", "template"}: - return "" - - # A relative-time label may change on every site build even though its - # machine-readable date stays the same. - if node.name == "time" and isinstance(value := node.get("datetime"), str): - return value.strip() - - # Image-only posts also have meaningful content, despite having no text. - if node.name == "img": - return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " - - text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text - - return " ".join(extract(content).split()) - - -def scan_source_file( - path: Path, - *, - root: Path, - base_url: str | None, - ignored_hostnames: tuple[str, ...] = (), -) -> ScannedSource: - """ - Scan one generated source document for outgoing Webmentions. - - :param path: Path of the generated HTML document. - :param root: Root directory containing generated source documents. - :param base_url: Configured source base URL, if available. - :param ignored_hostnames: Hostname patterns whose targets should be ignored. - :return: Scanned source metadata and discovered targets. - :raises SourceScanError: If the document cannot be safely interpreted as a - Webmention source. - :raises OSError: If the source document cannot be read. - """ - # Use the path as published for its URL, but read only files whose - # resolved target stays inside the configured source directory. Resolving - # before reading also protects against symlinked index.html files. - resolved_root = root.resolve() - resolved_path = path.resolve() - if not resolved_path.is_relative_to(resolved_root): - raise SourceScanError(f"Source document escapes source directory: {path}") - - relative_path = path.relative_to(root) - document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) - - if source_url is None: - if base_url is None: - raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" - ) - - source_url = source_url_for_path(relative_path, base_url) - - entry_element, mf2_entry = primary_entry(document, source_url) - - content = content_element(entry_element) - - targets = frozenset( - iter_targets( - content, - mf2_entry, - base_url=source_url, - source_url=source_url, - ignored_hostnames=ignored_hostnames, - ) - ) - - # Fingerprint the meaningful post text and its outgoing targets rather than - # the full h-entry markup. Relative timestamps, classes, and other volatile - # metadata outside e-content must not cause repeat Webmentions. - stable_content = normalized_content(content) - fingerprint = json.dumps( - {"content": stable_content, "targets": sorted(targets)}, - ensure_ascii=False, - separators=(",", ":"), - ) - - return ScannedSource( - path=relative_path.as_posix(), - url=source_url, - content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")), - targets=targets, - ) +from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: @@ -466,10 +84,9 @@ def update_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.active = webmention.target in scanned.targets - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen( + source.revision, webmention.target in scanned.targets + ) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -523,10 +140,6 @@ def scan_sources_task() -> None: raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" - ignored_hostnames = normalize_ignored_hostnames( - current_app.config["IGNORED_HOSTNAMES"] - ) - sources_by_path = { source.path: source for source in db.session.scalars( @@ -555,7 +168,7 @@ def scan_sources_task() -> None: index_path, root=root, base_url=base_url, - ignored_hostnames=ignored_hostnames, + ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -592,10 +205,6 @@ def scan_sources_task() -> None: except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - # A static-site generator can temporarily empty its output directory while - # rebuilding. With no successfully scanned pages, absence is not evidence - # of deletion; do not generate removals or send pending Webmentions that - # might be verified while the site is unavailable. if scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " @@ -612,10 +221,7 @@ def scan_sources_task() -> None: source.deleted_at = scan_time for webmention in source.sent_webmentions: - webmention.active = False - webmention.desired_revision = source.revision - webmention.attempt_count = 0 - webmention.next_retry_at = None + webmention = webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision |
