# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause """Reconcile scanned static-site sources with the database.""" from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path import sqlalchemy as sa from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey from ..models import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file @dataclass(frozen=True) class TreeScanResult: """Count and discovered paths from a traversal of the source tree.""" scanned_count: int seen_paths: frozenset[str] def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. A sent Webmention is built for each discovered target at the initial source revision. :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. :returns: Newly built source model. """ source = SourceDocument( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, revision=1, last_seen_at=scan_time, revised_at=scan_time, deleted_at=None, ) for target in scanned.targets: source.sent_webmentions.append( SentWebmention(target=target, linked=True, desired_revision=1) ) return source def apply_scan_to_source( source: SourceDocument, scanned: ScannedSource, scan_time: datetime ) -> SourceDocument: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent Webmentions are updated to reflect the current targets, and newly discovered targets are added. :param source: Existing source model to update. :param scanned: Newly scanned source data. :param scan_time: Time at which the source was scanned. :returns: Updated source model. :raises SourceScanError: If the public URL of the source has changed. """ if source.url != scanned.url: raise SourceScanError( f"Source path {source.path!r} changed public URL " f"from {source.url!r} to {scanned.url!r}" ) source.last_seen_at = scan_time if source.deleted_at is None and source.content_hash == scanned.content_hash: return source source.revision += 1 source.content_hash = scanned.content_hash source.revised_at = scan_time source.deleted_at = None existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): webmention.reset_for_revision( source.revision, webmention.target in scanned.targets ) if not webmention.linked and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( target=target, linked=True, desired_revision=source.revision ) ) return source def scan_sources() -> None: """Scan generated source documents and queue pending Webmentions. New and changed sources are persisted, missing sources are marked as deleted, and Webmentions requiring processing are queued for sending. A scan that successfully processes no sources is treated as inconclusive: it neither marks sources deleted nor queues Webmentions. Require a configured source root and public base URL before scanning generated pages. :raises RuntimeError: If the configured source directory or base URL is invalid. """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): _scan_and_enqueue() def reconcile_source_tree( root: Path, base_url: str | None, sources_by_path: dict[str, SourceDocument], paths_by_url: dict[str, str], scan_time: datetime, ) -> TreeScanResult: """Reconcile found pages and return the paths seen, even on parse failures. :param root: Root of the generated static-site output tree. :param base_url: Optional public base URL used for source URL resolution. :param sources_by_path: Known source records indexed by relative path. :param paths_by_url: Known relative paths indexed by public source URL. :param scan_time: Timestamp associated with the current scan. :returns: Scan result containing discovered paths and processing counts. """ seen_paths: set[str] = set() scanned_count = 0 for index_path in root.rglob("index.html"): relative_path = index_path.relative_to(root) if relative_path.parent == Path(): continue # A failed parse still proves the path exists; do not mark it deleted. seen_paths.add(relative_path.as_posix()) try: scanned = scan_source_file( index_path, root=root, base_url=base_url, ignored_hostnames=current_app.config["IGNORED_TARGET_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) continue except Exception: # A bad page must not prevent other sources being saved. current_app.logger.exception( "Unexpected error scanning source %s", index_path ) continue if (owner_path := paths_by_url.get(scanned.url)) is not None and ( owner_path != scanned.path ): current_app.logger.warning( "Skipping source %s: duplicate canonical URL %r (already used by %s)", index_path, scanned.url, owner_path, ) continue try: if (source := sources_by_path.get(scanned.path)) is None: source = build_source(scanned, scan_time) db.session.add(source) else: source = apply_scan_to_source(source, scanned, scan_time) sources_by_path[source.path] = source paths_by_url[source.url] = source.path scanned_count += 1 except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) return TreeScanResult(scanned_count, frozenset(seen_paths)) def mark_missing_sources_deleted( sources_by_path: dict[str, SourceDocument], seen_paths: frozenset[str], scan_time: datetime, ) -> int: """Mark absent, previously present sources deleted and return their count. :param sources_by_path: Known source records indexed by relative path. :param seen_paths: Paths found during this scan, including failed parses. :param scan_time: Timestamp to record for sources newly marked missing. :returns: Number of previously linked sources marked as deleted. """ marked = 0 for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: continue source.revision += 1 source.revised_at = scan_time source.deleted_at = scan_time marked += 1 for webmention in source.sent_webmentions: webmention.reset_for_revision(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision return marked def enqueue_due_sends(scan_time: datetime) -> int: """Enqueue due Webmentions and return the number of tasks submitted. :param scan_time: Current scan timestamp used to evaluate retry deadlines. :returns: Number of due Webmention tasks enqueued. """ queued = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, ), ) .order_by(SentWebmention.uuid) ): send_webmention(identifier) queued += 1 return queued def _scan_and_enqueue() -> None: """Scan sources while the caller holds the shared scan lock. Reconcile the generated source tree, persist changes, and enqueue eligible outgoing mentions. :raises RuntimeError: If the source directory or configured base URL is invalid. """ directory = current_app.config.get("SITE_OUTPUT_DIRECTORY") if directory is None: return root = Path(directory).expanduser().resolve() if not root.is_dir(): raise RuntimeError(f"Source directory is not a directory: {root}") if (base_url := current_app.config.get("SITE_BASE_URL")) is not None: if not is_http_url(base_url): raise RuntimeError("SITE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" sources_by_path = { source.path: source for source in db.session.scalars( sa.select(SourceDocument).options( selectinload(SourceDocument.sent_webmentions) ) ) } # Even deleted sources reserve their canonical URL, so a later alias cannot # take ownership merely by appearing earlier in the traversal. paths_by_url = {source.url: source.path for source in sources_by_path.values()} scan_time = datetime.now(UTC) scan = reconcile_source_tree( root, base_url, sources_by_path, paths_by_url, scan_time ) if scan.scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " "skipping deletion detection and Webmention sends" ) return missing_count = mark_missing_sources_deleted( sources_by_path, scan.seen_paths, scan_time ) db.session.commit() pending_count = enqueue_due_sends(scan_time) current_app.logger.info( "Webmention source scan complete: %d sources scanned, " "%d missing sources, %d pending sends", scan.scanned_count, missing_count, pending_count, ) @huey.task() def manual_scan_sources() -> None: """Run a source scan, deferring it while another scan holds the lock. If another scan holds the lock, arrange for the scan to run later instead of overlapping it. :raises RetryTask: If another worker currently holds the scan lock. """ try: scan_sources() except TaskLockedException as exc: # Unlike a periodic scan, a requested manual scan must not be discarded # merely because another scan is running. RetryTask schedules another # attempt without consuming a finite retry budget. current_app.logger.info("Source scan already running; deferring manual scan") raise RetryTask(delay=30) from exc