# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause """Reconcile scanned static-site sources with the database.""" from datetime import UTC, datetime from pathlib import Path import sqlalchemy as sa from flask import current_app from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import DATABASE as db from ..extensions import HUEY as huey from ..models import SentWebmention, Source from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: """Create a source model from a newly discovered document. A sent Webmention is created for each discovered target at the initial source revision. :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. :return: Newly created source model. """ source = Source( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, revision=1, last_seen_at=scan_time, revised_at=scan_time, deleted_at=None, ) for target in scanned.targets: source.sent_webmentions.append( SentWebmention(target=target, active=True, desired_revision=1) ) return source def update_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent Webmentions are updated to reflect the current targets, and newly discovered targets are added. :param source: Existing source model to update. :param scanned: Newly scanned source data. :param scan_time: Time at which the source was scanned. :return: Updated source model. :raises SourceScanError: If the public URL of the source has changed. """ if source.url != scanned.url: raise SourceScanError( f"Source path {source.path!r} changed public URL " f"from {source.url!r} to {scanned.url!r}" ) source.last_seen_at = scan_time if source.deleted_at is None and source.content_hash == scanned.content_hash: return source source.revision += 1 source.content_hash = scanned.content_hash source.revised_at = scan_time source.deleted_at = None existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): webmention = webmention.reopen( source.revision, webmention.target in scanned.targets ) if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( target=target, active=True, desired_revision=source.revision ) ) return source def scan_sources() -> None: """Scan generated source documents and queue pending Webmentions. New and changed sources are persisted, missing sources are marked as deleted, and Webmentions requiring processing are queued for sending. A scan that successfully processes no sources is treated as inconclusive: it neither marks sources deleted nor queues Webmentions. :raises RuntimeError: If the configured source directory or base URL is invalid. """ # Resolve the lock at execution time. The global Huey extension can be # reinitialized by Flask app factories (notably in the test suite), so an # import-time lock decorator may point at an obsolete storage backend. with huey.lock_task("scan-webmention-sources"): scan_sources_task() def scan_sources_task() -> None: """Scan the sources while the caller holds the shared scan lock.""" directory = current_app.config.get("SOURCE_DIRECTORY") if directory is None: return root = Path(directory).expanduser().resolve() if not root.is_dir(): raise RuntimeError(f"Source directory is not a directory: {root}") if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: if not is_http_url(base_url): raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" sources_by_path = { source.path: source for source in db.session.scalars( sa.select(Source).options(selectinload(Source.sent_webmentions)) ) } # Existing sources retain their canonical URLs even when marked deleted. # Reserve these URLs before scanning so an alias cannot claim one just # because it is discovered earlier than its persisted owner. paths_by_url = {source.url: source.path for source in sources_by_path.values()} seen_paths: set[str] = set() scanned_count = 0 scan_time = datetime.now(UTC) for index_path in root.rglob("index.html"): relative_path = index_path.relative_to(root) if relative_path.parent == Path(): continue seen_paths.add(relative_path.as_posix()) try: scanned = scan_source_file( index_path, root=root, base_url=base_url, ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) continue except Exception: # Third-party HTML parsers can raise unexpected exceptions (including # RecursionError). A bad page must not prevent other sources being saved. current_app.logger.exception( "Unexpected error scanning source %s", index_path ) continue if (owner_path := paths_by_url.get(scanned.url)) is not None and ( owner_path != scanned.path ): current_app.logger.warning( "Skipping source %s: duplicate canonical URL %r (already used by %s)", index_path, scanned.url, owner_path, ) continue try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) db.session.add(source) else: source = update_source(source, scanned, scan_time) sources_by_path[source.path] = source paths_by_url[source.url] = source.path scanned_count += 1 except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) if scanned_count == 0: current_app.logger.warning( "No source documents successfully scanned; " "skipping deletion detection and Webmention sends" ) return for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: continue source.revision += 1 source.revised_at = scan_time source.deleted_at = scan_time for webmention in source.sent_webmentions: webmention = webmention.reopen(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision db.session.commit() pending_count = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( sa.or_( SentWebmention.processed_revision.is_(None), SentWebmention.processed_revision < SentWebmention.desired_revision, ), sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, ), ) .order_by(SentWebmention.uuid) ): send_webmention(identifier) pending_count += 1 current_app.logger.info( "Webmention source scan complete: %d sources scanned, %d pending sends", scanned_count, pending_count, ) @huey.task() def manual_scan_sources() -> None: """Run a source scan, deferring it while another scan holds the lock.""" try: scan_sources() except TaskLockedException as exc: # Unlike a periodic scan, a requested manual scan must not be discarded # merely because another scan is running. RetryTask schedules another # attempt without consuming a finite retry budget. current_app.logger.info("Source scan already running; deferring manual scan") raise RetryTask(delay=30) from exc