diff options
| author | Dennis Fink | 2026-11-13 22:12:02 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-13 22:12:02 +0100 |
| commit | 2366d455c7d1ae16c84e0bcbffb94ed762c16832 (patch) | |
| tree | 51de07737ebc12fdcbf29a0a405ff2cd5aa9eb87 /webmentions_ssg/tasks/scanner.py | |
| parent | 64f328d86bba9caa9d0a60aa89e06f70403942b9 (diff) | |
| download | webmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.tar.gz webmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.zip | |
refactor(core): split task workflows into focused helpers
Decompose receiver verification, source scanning, and Webmention
delivery into explicit claim, check, reconciliation, and
outcome-recording stages.
Extract Huey backend parsing helpers and add focused tests for the new
task boundaries while preserving existing processing semantics.
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 115 |
1 files changed, 75 insertions, 40 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 5ee2d9e..ce03185 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -4,6 +4,7 @@ """Reconcile scanned static-site sources with the database.""" +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -113,38 +114,24 @@ def scan_sources() -> None: _scan_locked() -def _scan_locked() -> None: - """Scan the sources while the caller holds the shared scan lock.""" - directory = current_app.config.get("SOURCE_DIRECTORY") - - if directory is None: - return - - root = Path(directory).expanduser().resolve() - - if not root.is_dir(): - raise RuntimeError(f"Source directory is not a directory: {root}") - - if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: - if not is_http_url(base_url): - raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") - base_url = f"{base_url.rstrip('/')}/" +@dataclass(frozen=True) +class ScanResult: + """Count and discovered paths from a traversal of the source tree.""" - sources_by_path = { - source.path: source - for source in db.session.scalars( - sa.select(Source).options(selectinload(Source.sent_webmentions)) - ) - } + scanned_count: int + seen_paths: frozenset[str] - # Existing sources retain their canonical URLs even when marked deleted. - # Reserve these URLs before scanning so an alias cannot claim one just - # because it is discovered earlier than its persisted owner. - paths_by_url = {source.url: source.path for source in sources_by_path.values()} +def scan_tree( + root: Path, + base_url: str | None, + sources_by_path: dict[str, Source], + paths_by_url: dict[str, str], + scan_time: datetime, +) -> ScanResult: + """Reconcile found pages and return the paths seen, even on parse failures.""" seen_paths: set[str] = set() scanned_count = 0 - scan_time = datetime.now(UTC) for index_path in root.rglob("index.html"): relative_path = index_path.relative_to(root) @@ -152,6 +139,7 @@ def _scan_locked() -> None: if relative_path.parent == Path(): continue + # A failed parse still proves the path exists; do not mark it deleted. seen_paths.add(relative_path.as_posix()) try: @@ -165,8 +153,7 @@ def _scan_locked() -> None: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) continue except Exception: - # Third-party HTML parsers can raise unexpected exceptions (including - # RecursionError). A bad page must not prevent other sources being saved. + # A bad page must not prevent other sources being saved. current_app.logger.exception( "Unexpected error scanning source %s", index_path ) @@ -196,13 +183,14 @@ def _scan_locked() -> None: except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - if scanned_count == 0: - current_app.logger.warning( - "No source documents successfully scanned; " - "skipping deletion detection and Webmention sends" - ) - return + return ScanResult(scanned_count, frozenset(seen_paths)) + +def mark_missing( + sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime +) -> int: + """Mark absent, previously active sources deleted and return their count.""" + marked = 0 for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: continue @@ -210,6 +198,7 @@ def _scan_locked() -> None: source.revision += 1 source.revised_at = scan_time source.deleted_at = scan_time + marked += 1 for webmention in source.sent_webmentions: webmention.reopen(source.revision, False) @@ -217,10 +206,12 @@ def _scan_locked() -> None: if webmention.sent_revision is None: webmention.processed_revision = source.revision - db.session.commit() + return marked - pending_count = 0 +def queue_pending(scan_time: datetime) -> int: + """Enqueue due Webmentions and return the number of tasks submitted.""" + queued = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( @@ -233,11 +224,55 @@ def _scan_locked() -> None: .order_by(SentWebmention.uuid) ): send_webmention(identifier) - pending_count += 1 + queued += 1 + + return queued + + +def _scan_locked() -> None: + """Scan sources while the caller holds the shared scan lock.""" + directory = current_app.config.get("SOURCE_DIRECTORY") + if directory is None: + return + + root = Path(directory).expanduser().resolve() + if not root.is_dir(): + raise RuntimeError(f"Source directory is not a directory: {root}") + + if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: + if not is_http_url(base_url): + raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") + base_url = f"{base_url.rstrip('/')}/" + + sources_by_path = { + source.path: source + for source in db.session.scalars( + sa.select(Source).options(selectinload(Source.sent_webmentions)) + ) + } + + # Even deleted sources reserve their canonical URL, so a later alias cannot + # take ownership merely by appearing earlier in the traversal. + paths_by_url = {source.url: source.path for source in sources_by_path.values()} + scan_time = datetime.now(UTC) + scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time) + + if scan.scanned_count == 0: + current_app.logger.warning( + "No source documents successfully scanned; " + "skipping deletion detection and Webmention sends" + ) + return + + missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time) + db.session.commit() + pending_count = queue_pending(scan_time) current_app.logger.info( - "Webmention source scan complete: %d sources scanned, %d pending sends", - scanned_count, + "Webmention source scan complete: %d sources scanned, " + "%d missing sources, %d pending sends", + scan.scanned_count, + missing_count, pending_count, ) |
