diff options
Diffstat (limited to 'webmentions_ssg/tasks/scanner.py')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 50 |
1 files changed, 41 insertions, 9 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index ce03185..13d2ceb 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -28,7 +28,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. - :return: Newly created source model. + :returns: Newly created source model. """ source = Source( path=scanned.path, @@ -60,7 +60,7 @@ def update_source( :param source: Existing source model to update. :param scanned: Newly scanned source data. :param scan_time: Time at which the source was scanned. - :return: Updated source model. + :returns: Updated source model. :raises SourceScanError: If the public URL of the source has changed. """ if source.url != scanned.url: @@ -106,8 +106,10 @@ def scan_sources() -> None: successfully processes no sources is treated as inconclusive: it neither marks sources deleted nor queues Webmentions. - :raises RuntimeError: If the configured source directory or base URL is - invalid. + Require a configured source root and public base URL before scanning generated + pages. + + :raises RuntimeError: If the configured source directory or base URL is invalid. """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): @@ -129,7 +131,15 @@ def scan_tree( paths_by_url: dict[str, str], scan_time: datetime, ) -> ScanResult: - """Reconcile found pages and return the paths seen, even on parse failures.""" + """Reconcile found pages and return the paths seen, even on parse failures. + + :param root: Root of the generated static-site output tree. + :param base_url: Optional public base URL used for source URL resolution. + :param sources_by_path: Known source records indexed by relative path. + :param paths_by_url: Known relative paths indexed by public source URL. + :param scan_time: Timestamp associated with the current scan. + :returns: Scan result containing discovered paths and processing counts. + """ seen_paths: set[str] = set() scanned_count = 0 @@ -189,7 +199,13 @@ def scan_tree( def mark_missing( sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime ) -> int: - """Mark absent, previously active sources deleted and return their count.""" + """Mark absent, previously active sources deleted and return their count. + + :param sources_by_path: Known source records indexed by relative path. + :param seen_paths: Paths found during this scan, including failed parses. + :param scan_time: Timestamp to record for sources newly marked missing. + :returns: Number of previously active sources marked as deleted. + """ marked = 0 for path, source in sources_by_path.items(): if source.deleted_at is not None or path in seen_paths: @@ -210,7 +226,11 @@ def mark_missing( def queue_pending(scan_time: datetime) -> int: - """Enqueue due Webmentions and return the number of tasks submitted.""" + """Enqueue due Webmentions and return the number of tasks submitted. + + :param scan_time: Current scan timestamp used to evaluate retry deadlines. + :returns: Number of due Webmention tasks enqueued. + """ queued = 0 for identifier in db.session.scalars( sa.select(SentWebmention.uuid) @@ -230,7 +250,13 @@ def queue_pending(scan_time: datetime) -> int: def _scan_locked() -> None: - """Scan sources while the caller holds the shared scan lock.""" + """Scan sources while the caller holds the shared scan lock. + + Reconcile the generated source tree, persist changes, and enqueue eligible outgoing + mentions. + + :raises RuntimeError: If the source directory or configured base URL is invalid. + """ directory = current_app.config.get("SOURCE_DIRECTORY") if directory is None: return @@ -279,7 +305,13 @@ def _scan_locked() -> None: @huey.task() def manual_scan_sources() -> None: - """Run a source scan, deferring it while another scan holds the lock.""" + """Run a source scan, deferring it while another scan holds the lock. + + If another scan holds the lock, arrange for the scan to run later instead of + overlapping it. + + :raises RetryTask: If another worker currently holds the scan lock. + """ try: scan_sources() except TaskLockedException as exc: |
