aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-13 22:12:02 +0100
committerDennis Fink2026-11-13 22:12:02 +0100
commit2366d455c7d1ae16c84e0bcbffb94ed762c16832 (patch)
tree51de07737ebc12fdcbf29a0a405ff2cd5aa9eb87 /webmentions_ssg/tasks/scanner.py
parent64f328d86bba9caa9d0a60aa89e06f70403942b9 (diff)
downloadwebmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.tar.gz
webmentions-ssg-2366d455c7d1ae16c84e0bcbffb94ed762c16832.zip
refactor(core): split task workflows into focused helpers
Decompose receiver verification, source scanning, and Webmention delivery into explicit claim, check, reconciliation, and outcome-recording stages. Extract Huey backend parsing helpers and add focused tests for the new task boundaries while preserving existing processing semantics.
Diffstat (limited to 'webmentions_ssg/tasks/scanner.py')
-rw-r--r--webmentions_ssg/tasks/scanner.py115
1 files changed, 75 insertions, 40 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 5ee2d9e..ce03185 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -4,6 +4,7 @@
"""Reconcile scanned static-site sources with the database."""
+from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
@@ -113,38 +114,24 @@ def scan_sources() -> None:
_scan_locked()
-def _scan_locked() -> None:
- """Scan the sources while the caller holds the shared scan lock."""
- directory = current_app.config.get("SOURCE_DIRECTORY")
-
- if directory is None:
- return
-
- root = Path(directory).expanduser().resolve()
-
- if not root.is_dir():
- raise RuntimeError(f"Source directory is not a directory: {root}")
-
- if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
- if not is_http_url(base_url):
- raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
- base_url = f"{base_url.rstrip('/')}/"
+@dataclass(frozen=True)
+class ScanResult:
+ """Count and discovered paths from a traversal of the source tree."""
- sources_by_path = {
- source.path: source
- for source in db.session.scalars(
- sa.select(Source).options(selectinload(Source.sent_webmentions))
- )
- }
+ scanned_count: int
+ seen_paths: frozenset[str]
- # Existing sources retain their canonical URLs even when marked deleted.
- # Reserve these URLs before scanning so an alias cannot claim one just
- # because it is discovered earlier than its persisted owner.
- paths_by_url = {source.url: source.path for source in sources_by_path.values()}
+def scan_tree(
+ root: Path,
+ base_url: str | None,
+ sources_by_path: dict[str, Source],
+ paths_by_url: dict[str, str],
+ scan_time: datetime,
+) -> ScanResult:
+ """Reconcile found pages and return the paths seen, even on parse failures."""
seen_paths: set[str] = set()
scanned_count = 0
- scan_time = datetime.now(UTC)
for index_path in root.rglob("index.html"):
relative_path = index_path.relative_to(root)
@@ -152,6 +139,7 @@ def _scan_locked() -> None:
if relative_path.parent == Path():
continue
+ # A failed parse still proves the path exists; do not mark it deleted.
seen_paths.add(relative_path.as_posix())
try:
@@ -165,8 +153,7 @@ def _scan_locked() -> None:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
continue
except Exception:
- # Third-party HTML parsers can raise unexpected exceptions (including
- # RecursionError). A bad page must not prevent other sources being saved.
+ # A bad page must not prevent other sources being saved.
current_app.logger.exception(
"Unexpected error scanning source %s", index_path
)
@@ -196,13 +183,14 @@ def _scan_locked() -> None:
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- if scanned_count == 0:
- current_app.logger.warning(
- "No source documents successfully scanned; "
- "skipping deletion detection and Webmention sends"
- )
- return
+ return ScanResult(scanned_count, frozenset(seen_paths))
+
+def mark_missing(
+ sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
+) -> int:
+ """Mark absent, previously active sources deleted and return their count."""
+ marked = 0
for path, source in sources_by_path.items():
if source.deleted_at is not None or path in seen_paths:
continue
@@ -210,6 +198,7 @@ def _scan_locked() -> None:
source.revision += 1
source.revised_at = scan_time
source.deleted_at = scan_time
+ marked += 1
for webmention in source.sent_webmentions:
webmention.reopen(source.revision, False)
@@ -217,10 +206,12 @@ def _scan_locked() -> None:
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
- db.session.commit()
+ return marked
- pending_count = 0
+def queue_pending(scan_time: datetime) -> int:
+ """Enqueue due Webmentions and return the number of tasks submitted."""
+ queued = 0
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
.where(
@@ -233,11 +224,55 @@ def _scan_locked() -> None:
.order_by(SentWebmention.uuid)
):
send_webmention(identifier)
- pending_count += 1
+ queued += 1
+
+ return queued
+
+
+def _scan_locked() -> None:
+ """Scan sources while the caller holds the shared scan lock."""
+ directory = current_app.config.get("SOURCE_DIRECTORY")
+ if directory is None:
+ return
+
+ root = Path(directory).expanduser().resolve()
+ if not root.is_dir():
+ raise RuntimeError(f"Source directory is not a directory: {root}")
+
+ if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
+ if not is_http_url(base_url):
+ raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
+ base_url = f"{base_url.rstrip('/')}/"
+
+ sources_by_path = {
+ source.path: source
+ for source in db.session.scalars(
+ sa.select(Source).options(selectinload(Source.sent_webmentions))
+ )
+ }
+
+ # Even deleted sources reserve their canonical URL, so a later alias cannot
+ # take ownership merely by appearing earlier in the traversal.
+ paths_by_url = {source.url: source.path for source in sources_by_path.values()}
+ scan_time = datetime.now(UTC)
+ scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time)
+
+ if scan.scanned_count == 0:
+ current_app.logger.warning(
+ "No source documents successfully scanned; "
+ "skipping deletion detection and Webmention sends"
+ )
+ return
+
+ missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time)
+ db.session.commit()
+ pending_count = queue_pending(scan_time)
current_app.logger.info(
- "Webmention source scan complete: %d sources scanned, %d pending sends",
- scanned_count,
+ "Webmention source scan complete: %d sources scanned, "
+ "%d missing sources, %d pending sends",
+ scan.scanned_count,
+ missing_count,
pending_count,
)