aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/scanner.py50
1 files changed, 41 insertions, 9 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index ce03185..13d2ceb 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -28,7 +28,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
:param scanned: Scanned source data.
:param scan_time: Time at which the source was discovered.
- :return: Newly created source model.
+ :returns: Newly created source model.
"""
source = Source(
path=scanned.path,
@@ -60,7 +60,7 @@ def update_source(
:param source: Existing source model to update.
:param scanned: Newly scanned source data.
:param scan_time: Time at which the source was scanned.
- :return: Updated source model.
+ :returns: Updated source model.
:raises SourceScanError: If the public URL of the source has changed.
"""
if source.url != scanned.url:
@@ -106,8 +106,10 @@ def scan_sources() -> None:
successfully processes no sources is treated as inconclusive: it neither
marks sources deleted nor queues Webmentions.
- :raises RuntimeError: If the configured source directory or base URL is
- invalid.
+ Require a configured source root and public base URL before scanning generated
+ pages.
+
+ :raises RuntimeError: If the configured source directory or base URL is invalid.
"""
# Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
@@ -129,7 +131,15 @@ def scan_tree(
paths_by_url: dict[str, str],
scan_time: datetime,
) -> ScanResult:
- """Reconcile found pages and return the paths seen, even on parse failures."""
+ """Reconcile found pages and return the paths seen, even on parse failures.
+
+ :param root: Root of the generated static-site output tree.
+ :param base_url: Optional public base URL used for source URL resolution.
+ :param sources_by_path: Known source records indexed by relative path.
+ :param paths_by_url: Known relative paths indexed by public source URL.
+ :param scan_time: Timestamp associated with the current scan.
+ :returns: Scan result containing discovered paths and processing counts.
+ """
seen_paths: set[str] = set()
scanned_count = 0
@@ -189,7 +199,13 @@ def scan_tree(
def mark_missing(
sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
) -> int:
- """Mark absent, previously active sources deleted and return their count."""
+ """Mark absent, previously active sources deleted and return their count.
+
+ :param sources_by_path: Known source records indexed by relative path.
+ :param seen_paths: Paths found during this scan, including failed parses.
+ :param scan_time: Timestamp to record for sources newly marked missing.
+ :returns: Number of previously active sources marked as deleted.
+ """
marked = 0
for path, source in sources_by_path.items():
if source.deleted_at is not None or path in seen_paths:
@@ -210,7 +226,11 @@ def mark_missing(
def queue_pending(scan_time: datetime) -> int:
- """Enqueue due Webmentions and return the number of tasks submitted."""
+ """Enqueue due Webmentions and return the number of tasks submitted.
+
+ :param scan_time: Current scan timestamp used to evaluate retry deadlines.
+ :returns: Number of due Webmention tasks enqueued.
+ """
queued = 0
for identifier in db.session.scalars(
sa.select(SentWebmention.uuid)
@@ -230,7 +250,13 @@ def queue_pending(scan_time: datetime) -> int:
def _scan_locked() -> None:
- """Scan sources while the caller holds the shared scan lock."""
+ """Scan sources while the caller holds the shared scan lock.
+
+ Reconcile the generated source tree, persist changes, and enqueue eligible outgoing
+ mentions.
+
+ :raises RuntimeError: If the source directory or configured base URL is invalid.
+ """
directory = current_app.config.get("SOURCE_DIRECTORY")
if directory is None:
return
@@ -279,7 +305,13 @@ def _scan_locked() -> None:
@huey.task()
def manual_scan_sources() -> None:
- """Run a source scan, deferring it while another scan holds the lock."""
+ """Run a source scan, deferring it while another scan holds the lock.
+
+ If another scan holds the lock, arrange for the scan to run later instead of
+ overlapping it.
+
+ :raises RetryTask: If another worker currently holds the scan lock.
+ """
try:
scan_sources()
except TaskLockedException as exc: