# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause """Reconcile scanned static-site sources with the database.""" from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path import sqlalchemy as sa from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey from ..models import SentWebmention, Source from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: """Create a source model from a newly discovered document. A sent Webmention is created for each discovered target at the initial source revision. :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. :return: Newly created source model. """ source = Source( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, revision=1, last_seen_at=scan_time, revised_at=scan_time, deleted_at=None, ) for target in scanned.targets: source.sent_webmentions.append( SentWebmention(target=target, active=True, desired_revision=1) ) return source def update_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent Webmentions are updated to reflect the current targets, and newly discovered targets are added. :param source: Existing source model to update. :param scanned: Newly scanned source data. :param scan_time: Time at which the source was scanned. :return: Updated source model. :raises SourceScanError: If the public URL of the source has changed. """ if source.url != scanned.url: raise SourceScanError( f"Source path {source.path!r} changed public URL " f"from {source.url!r} to {scanned.url!r}" ) source.last_seen_at = scan_time if source.deleted_at is None and source.content_hash == scanned.content_hash: return source source.revision += 1 source.content_hash = scanned.content_hash source.revised_at = scan_time source.deleted_at = None existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): webmention.reopen(source.revision, webmention.target in scanned.targets) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( target=target, active=True, desired_revision=source.revision ) ) return source def scan_sources() -> None: """Scan generated source documents and queue pending Webmentions. New and changed sources are persisted, missing sources are marked as deleted, and Webmentions requiring processing are queued for sending. A scan that successfully processes no sources is treated as inconclusive: it neither marks sources deleted nor queues Webmentions. :raises RuntimeError: If the configured source directory or base URL is invalid. """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): _scan_locked() @dataclass(frozen=True) class ScanResult: """Count and discovered paths from a traversal of the source tree.""" scanned_count: int seen_paths: frozenset[str] def scan_tree( root: Path, base_url: str | None, sources_by_path: dict[str, Source], paths_by_url: dict[str, str], scan_time: datetime, ) -> ScanResult: """Reconcile found pages and return the paths seen, even on parse failures.""" seen_paths: set[str] = set() scanned_count = 0 for index_path in root.rglob("index.html"): relative_path = index_path.relative_to(root) if relative_path.parent == Path(): continue # A failed parse still proves the path exists; do not mark it deleted. seen_paths.add(relative_path.as_posix()) try: scanned = scan_source_file( index_path, root=root, base_url=base_url, ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) continue except Exception: # A bad page must not prevent other sources being saved. current_app.logger.exception( "Unexpected error scanning source %s", index_path ) continue if (owner_path := paths_by_url.get(scanned.url)) is not None and ( owner_path != scanned.path ): current_app.logger.warning( "Skipping source %s: duplicate canonical URL %r (already used by %s)", index_path, scanned.url, owner_path, ) continue try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) db.session.add(source) else: source = update_source(source, scanned, scan_time) sources_by_path[source.path] = source paths_by_url[source.url] = source.path scanned_count += 1 except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) return ScanResult(scanned_count, frozenset(seen_paths)) def mark_missing( sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime ) -> int: """Mark absent, previously active sources deleted and return their count.""" marked = 0 for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: continue source.revision += 1 source.revised_at = scan_time source.deleted_at = scan_time marked += 1 for webmention in source.sent_webmentions: webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision return marked def queue_pending(scan_time: datetime) -> int: """Enqueue due Webmentions and return the number of tasks submitted.""" queued = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( SentWebmention.pending, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, ), ) .order_by(SentWebmention.uuid) ): send_webmention(identifier) queued += 1 return queued def _scan_locked() -> None: """Scan sources while the caller holds the shared scan lock.""" directory = current_app.config.get("SOURCE_DIRECTORY") if directory is None: return root = Path(directory).expanduser().resolve() if not root.is_dir(): raise RuntimeError(f"Source directory is not a directory: {root}") if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: if not is_http_url(base_url): raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" sources_by_path = { source.path: source for source in db.session.scalars( sa.select(Source).options(selectinload(Source.sent_webmentions)) ) } # Even deleted sources reserve their canonical URL, so a later alias cannot # take ownership merely by appearing earlier in the traversal. paths_by_url = {source.url: source.path for source in sources_by_path.values()} scan_time = datetime.now(UTC) scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time) if scan.scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " "skipping deletion detection and Webmention sends" ) return missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time) db.session.commit() pending_count = queue_pending(scan_time) current_app.logger.info( "Webmention source scan complete: %d sources scanned, " "%d missing sources, %d pending sends", scan.scanned_count, missing_count, pending_count, ) @huey.task() def manual_scan_sources() -> None: """Run a source scan, deferring it while another scan holds the lock.""" try: scan_sources() except TaskLockedException as exc: # Unlike a periodic scan, a requested manual scan must not be discarded # merely because another scan is running. RetryTask schedules another # attempt without consuming a finite retry budget. current_app.logger.info("Source scan already running; deferring manual scan") raise RetryTask(delay=30) from exc