aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/scanner.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to 'webmentions_ssg/tasks/scanner.py')
-rw-r--r--webmentions_ssg/tasks/scanner.py414
1 files changed, 10 insertions, 404 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 666eba7..b2d6def 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -2,404 +2,22 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Scan generated site pages to track outgoing Webmention targets."""
+"""Reconcile scanned static-site sources with the database."""
-import json
-from collections.abc import Iterator
-from dataclasses import dataclass
from datetime import UTC, datetime
-from fnmatch import fnmatchcase
-from itertools import chain
from pathlib import Path
-from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
-import mf2py
import sqlalchemy as sa
-import xxhash
-from bs4 import BeautifulSoup, NavigableString, Tag
-from bs4.element import PageElement
from flask import current_app
from huey.exceptions import RetryTask, TaskLockedException
from sqlalchemy.orm import selectinload
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..config import normalize_ignored_hostnames
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
from ..models import SentWebmention, Source
-from ..url_security import is_http_url
+from ..net import is_http_url
from .sender import send_webmention
-
-REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-
-CONTENT_BLOCK_TAGS = frozenset(
- {
- "address",
- "article",
- "aside",
- "blockquote",
- "br",
- "dd",
- "div",
- "dl",
- "dt",
- "figcaption",
- "figure",
- "footer",
- "h1",
- "h2",
- "h3",
- "h4",
- "h5",
- "h6",
- "header",
- "hr",
- "li",
- "main",
- "ol",
- "p",
- "pre",
- "section",
- "table",
- "td",
- "th",
- "tr",
- "ul",
- }
-)
-
-
-class SourceScanError(Exception):
- """A source document cannot safely be processed."""
-
-
-@dataclass(frozen=True)
-class ScannedSource:
- path: str
- url: str
- content_hash: str
- targets: frozenset[str]
-
-
-def source_url_for_path(relative_path: Path, base_url: str) -> str:
- """
- Derive the public source URL from a relative filesystem path.
-
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Base URL under which source documents are published.
- :return: Public URL corresponding to the source document.
- """
- directory = relative_path.parent.as_posix()
-
- return urljoin(base_url, f"{quote(directory, safe='/')}/")
-
-
-def canonical_url(
- document: BeautifulSoup, *, relative_path: Path, base_url: str | None
-) -> str | None:
- """
- Return the canonical URL declared by a source document.
-
- Relative canonical URLs are resolved against the configured source base URL.
-
- :param document: Parsed HTML source document.
- :param relative_path: Path of the source document relative to the source root.
- :param base_url: Configured source base URL, if available.
- :return: Canonical URL, or ``None`` if none is declared.
- :raises SourceScanError: If the canonical URL is empty, cannot be resolved,
- or does not resolve to an HTTP or HTTPS URL.
- """
- if (link := document.select_one('link[rel~="canonical"][href]')) is not None:
- href = link.get("href")
-
- if not isinstance(href, str) or not (href := href.strip()):
- raise SourceScanError("Document contains an empty canonical URL")
-
- if is_http_url(href):
- return href
-
- if base_url is None:
- raise SourceScanError(
- "Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
- )
-
- resolved = urljoin(source_url_for_path(relative_path, base_url), href)
-
- if not is_http_url(resolved):
- raise SourceScanError(
- f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}"
- )
-
- return resolved
-
-
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
- """
- Yield URL values from a microformats property.
-
- :param entry: Parsed microformats entry.
- :param property_name: Name of the property to inspect.
- :return: Iterator over string values of the property.
- """
- properties = entry.get("properties")
-
- if not isinstance(properties, dict):
- return
-
- values = properties.get(property_name)
-
- if not isinstance(values, list):
- return
-
- yield from (value for value in values if isinstance(value, str))
-
-
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
- """
- Parse and validate an h-entry with mf2py.
-
- :param element: HTML element containing the h-entry.
- :param base_url: Base URL used when parsing the microformats data.
- :return: Parsed h-entry.
- :raises SourceScanError: If parsing does not produce exactly one valid h-entry.
- """
- parsed = mf2py.parse(doc=str(element), url=base_url)
-
- if not isinstance(parsed, dict):
- raise SourceScanError("Microformats parser did not return a document object")
-
- items = parsed.get("items")
-
- if not isinstance(items, list) or len(items) != 1:
- raise SourceScanError("Expected exactly one parsed h-entry")
-
- entry = items[0]
-
- if not isinstance(entry, dict):
- raise SourceScanError("Parsed h-entry is not an object")
-
- types = entry.get("type")
-
- if not isinstance(types, list) or "h-entry" not in types:
- raise SourceScanError("Parsed microformats item is not an h-entry")
-
- return entry
-
-
-def primary_entry(
- document: BeautifulSoup, source_url: str
-) -> tuple[Tag, dict[str, Any]]:
- """
- Return and validate the primary h-entry of a source document.
-
- The document must contain exactly one h-entry whose ``u-url`` matches the
- source URL.
-
- :param document: Parsed HTML source document.
- :param source_url: Public URL of the source document.
- :return: h-entry element and its parsed microformats representation.
- :raises SourceScanError: If the document does not contain exactly one h-entry
- or its ``u-url`` does not match the source URL.
- """
- entries = document.find_all(class_="h-entry")
-
- if len(entries) != 1:
- raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
-
- entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
-
- urls = tuple(property_urls(mf2_entry, "url"))
-
- if source_url not in urls:
- raise SourceScanError(
- f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}"
- )
-
- return entry_element, mf2_entry
-
-
-def content_element(entry: Tag) -> Tag:
- """
- Return the e-content element of an h-entry.
-
- :param entry: HTML element containing the h-entry.
- :return: The h-entry's e-content element.
- :raises SourceScanError: If the h-entry does not contain exactly one
- e-content element.
- """
- contents = entry.find_all(class_="e-content")
-
- if len(contents) != 1:
- raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
-
- return contents[0]
-
-
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
- """
- Resolve and validate a possible Webmention target URL.
-
- Empty values, non-HTTP URLs, and URLs referring to the source itself are
- discarded.
-
- :param value: URL reference to normalize.
- :param base_url: Base URL against which relative references are resolved.
- :param source_url: URL of the source document.
- :return: Normalized target URL, or ``None`` if the value is not a valid
- Webmention target.
- """
- value = value.strip()
-
- if not value:
- return None
-
- target = urljoin(base_url, value)
-
- if not is_http_url(target):
- return None
-
- if urldefrag(target)[0] == urldefrag(source_url)[0]:
- return None
-
- return target
-
-
-def iter_targets(
- content: Tag,
- mf2_entry: dict[str, Any],
- *,
- base_url: str,
- source_url: str,
- ignored_hostnames: tuple[str, ...] = (),
-) -> Iterator[str]:
- """
- Yield outgoing Webmention targets from an h-entry.
-
- Targets are collected from links in the entry content and supported
- microformats reaction properties. Invalid, self-referencing, and ignored
- targets are excluded.
-
- :param content: e-content element of the h-entry.
- :param mf2_entry: Parsed microformats representation of the h-entry.
- :param base_url: Base URL against which relative targets are resolved.
- :param source_url: URL of the source document.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Iterator over outgoing Webmention target URLs.
- """
- hrefs = (
- href
- for element in content.find_all("a", href=True)
- if isinstance(href := element.get("href"), str)
- )
-
- reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
- )
-
- for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
- if target is not None:
- hostname = urlsplit(target).hostname or ""
- if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
- yield target
-
-
-def normalized_content(content: Tag) -> str:
- """Extract stable reader-facing content without generated HTML attributes."""
-
- def extract(node: PageElement) -> str:
- if isinstance(node, NavigableString):
- return str(node)
- if not isinstance(node, Tag) or node.name in {"script", "style", "template"}:
- return ""
-
- # A relative-time label may change on every site build even though its
- # machine-readable date stays the same.
- if node.name == "time" and isinstance(value := node.get("datetime"), str):
- return value.strip()
-
- # Image-only posts also have meaningful content, despite having no text.
- if node.name == "img":
- return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
-
- text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
-
- return " ".join(extract(content).split())
-
-
-def scan_source_file(
- path: Path,
- *,
- root: Path,
- base_url: str | None,
- ignored_hostnames: tuple[str, ...] = (),
-) -> ScannedSource:
- """
- Scan one generated source document for outgoing Webmentions.
-
- :param path: Path of the generated HTML document.
- :param root: Root directory containing generated source documents.
- :param base_url: Configured source base URL, if available.
- :param ignored_hostnames: Hostname patterns whose targets should be ignored.
- :return: Scanned source metadata and discovered targets.
- :raises SourceScanError: If the document cannot be safely interpreted as a
- Webmention source.
- :raises OSError: If the source document cannot be read.
- """
- # Use the path as published for its URL, but read only files whose
- # resolved target stays inside the configured source directory. Resolving
- # before reading also protects against symlinked index.html files.
- resolved_root = root.resolve()
- resolved_path = path.resolve()
- if not resolved_path.is_relative_to(resolved_root):
- raise SourceScanError(f"Source document escapes source directory: {path}")
-
- relative_path = path.relative_to(root)
- document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
-
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
-
- if source_url is None:
- if base_url is None:
- raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
- )
-
- source_url = source_url_for_path(relative_path, base_url)
-
- entry_element, mf2_entry = primary_entry(document, source_url)
-
- content = content_element(entry_element)
-
- targets = frozenset(
- iter_targets(
- content,
- mf2_entry,
- base_url=source_url,
- source_url=source_url,
- ignored_hostnames=ignored_hostnames,
- )
- )
-
- # Fingerprint the meaningful post text and its outgoing targets rather than
- # the full h-entry markup. Relative timestamps, classes, and other volatile
- # metadata outside e-content must not cause repeat Webmentions.
- stable_content = normalized_content(content)
- fingerprint = json.dumps(
- {"content": stable_content, "targets": sorted(targets)},
- ensure_ascii=False,
- separators=(",", ":"),
- )
-
- return ScannedSource(
- path=relative_path.as_posix(),
- url=source_url,
- content_hash=xxhash.xxh3_128_hexdigest(fingerprint.encode("utf-8")),
- targets=targets,
- )
+from .source_parser import ScannedSource, SourceScanError, scan_source_file
def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
@@ -466,10 +84,9 @@ def update_source(
existing = {webmention.target: webmention for webmention in source.sent_webmentions}
for webmention in existing.values():
- webmention.active = webmention.target in scanned.targets
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(
+ source.revision, webmention.target in scanned.targets
+ )
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -523,10 +140,6 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = normalize_ignored_hostnames(
- current_app.config["IGNORED_HOSTNAMES"]
- )
-
sources_by_path = {
source.path: source
for source in db.session.scalars(
@@ -555,7 +168,7 @@ def scan_sources_task() -> None:
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=ignored_hostnames,
+ ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -592,10 +205,6 @@ def scan_sources_task() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- # A static-site generator can temporarily empty its output directory while
- # rebuilding. With no successfully scanned pages, absence is not evidence
- # of deletion; do not generate removals or send pending Webmentions that
- # might be verified while the site is unavailable.
if scanned_count == 0:
current_app.logger.warning(
"No source documents successfully scanned; "
@@ -612,10 +221,7 @@ def scan_sources_task() -> None:
source.deleted_at = scan_time
for webmention in source.sent_webmentions:
- webmention.active = False
- webmention.desired_revision = source.revision
- webmention.attempt_count = 0
- webmention.next_retry_at = None
+ webmention = webmention.reopen(source.revision, False)
if webmention.sent_revision is None:
webmention.processed_revision = source.revision