diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 48 |
1 files changed, 26 insertions, 22 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 13d2ceb..034b2cf 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -20,15 +20,15 @@ from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file -def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: - """Create a source model from a newly discovered document. +def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: + """Build a source model from a newly discovered document. - A sent Webmention is created for each discovered target at the initial source + A sent Webmention is built for each discovered target at the initial source revision. :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. - :returns: Newly created source model. + :returns: Newly built source model. """ source = Source( path=scanned.path, @@ -48,7 +48,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: return source -def update_source( +def apply_scan_to_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: """Update an existing source from newly scanned data. @@ -113,24 +113,24 @@ def scan_sources() -> None: """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): - _scan_locked() + _scan_and_enqueue() @dataclass(frozen=True) -class ScanResult: +class TreeScanResult: """Count and discovered paths from a traversal of the source tree.""" scanned_count: int seen_paths: frozenset[str] -def scan_tree( +def reconcile_source_tree( root: Path, base_url: str | None, sources_by_path: dict[str, Source], paths_by_url: dict[str, str], scan_time: datetime, -) -> ScanResult: +) -> TreeScanResult: """Reconcile found pages and return the paths seen, even on parse failures. :param root: Root of the generated static-site output tree. @@ -157,7 +157,7 @@ def scan_tree( index_path, root=root, base_url=base_url, - ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], + ignored_hostnames=current_app.config["IGNORED_TARGET_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -182,10 +182,10 @@ def scan_tree( try: if (source := sources_by_path.get(scanned.path)) is None: - source = create_source(scanned, scan_time) + source = build_source(scanned, scan_time) db.session.add(source) else: - source = update_source(source, scanned, scan_time) + source = apply_scan_to_source(source, scanned, scan_time) sources_by_path[source.path] = source paths_by_url[source.url] = source.path @@ -193,10 +193,10 @@ def scan_tree( except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - return ScanResult(scanned_count, frozenset(seen_paths)) + return TreeScanResult(scanned_count, frozenset(seen_paths)) -def mark_missing( +def mark_missing_sources_deleted( sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime ) -> int: """Mark absent, previously active sources deleted and return their count. @@ -225,7 +225,7 @@ def mark_missing( return marked -def queue_pending(scan_time: datetime) -> int: +def enqueue_due_sends(scan_time: datetime) -> int: """Enqueue due Webmentions and return the number of tasks submitted. :param scan_time: Current scan timestamp used to evaluate retry deadlines. @@ -249,7 +249,7 @@ def queue_pending(scan_time: datetime) -> int: return queued -def _scan_locked() -> None: +def _scan_and_enqueue() -> None: """Scan sources while the caller holds the shared scan lock. Reconcile the generated source tree, persist changes, and enqueue eligible outgoing @@ -257,7 +257,7 @@ def _scan_locked() -> None: :raises RuntimeError: If the source directory or configured base URL is invalid. """ - directory = current_app.config.get("SOURCE_DIRECTORY") + directory = current_app.config.get("SITE_OUTPUT_DIRECTORY") if directory is None: return @@ -265,9 +265,9 @@ def _scan_locked() -> None: if not root.is_dir(): raise RuntimeError(f"Source directory is not a directory: {root}") - if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: + if (base_url := current_app.config.get("SITE_BASE_URL")) is not None: if not is_http_url(base_url): - raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") + raise RuntimeError("SITE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" sources_by_path = { @@ -281,7 +281,9 @@ def _scan_locked() -> None: # take ownership merely by appearing earlier in the traversal. paths_by_url = {source.url: source.path for source in sources_by_path.values()} scan_time = datetime.now(UTC) - scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time) + scan = reconcile_source_tree( + root, base_url, sources_by_path, paths_by_url, scan_time + ) if scan.scanned_count == 0: current_app.logger.warning( @@ -290,9 +292,11 @@ def _scan_locked() -> None: ) return - missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time) + missing_count = mark_missing_sources_deleted( + sources_by_path, scan.seen_paths, scan_time + ) db.session.commit() - pending_count = queue_pending(scan_time) + pending_count = enqueue_due_sends(scan_time) current_app.logger.info( "Webmention source scan complete: %d sources scanned, " |
