aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/scanner.py48
1 files changed, 26 insertions, 22 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 13d2ceb..034b2cf 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -20,15 +20,15 @@ from .sender import send_webmention
from .source_parser import ScannedSource, SourceScanError, scan_source_file
-def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
- """Create a source model from a newly discovered document.
+def build_source(scanned: ScannedSource, scan_time: datetime) -> Source:
+ """Build a source model from a newly discovered document.
- A sent Webmention is created for each discovered target at the initial source
+ A sent Webmention is built for each discovered target at the initial source
revision.
:param scanned: Scanned source data.
:param scan_time: Time at which the source was discovered.
- :returns: Newly created source model.
+ :returns: Newly built source model.
"""
source = Source(
path=scanned.path,
@@ -48,7 +48,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source:
return source
-def update_source(
+def apply_scan_to_source(
source: Source, scanned: ScannedSource, scan_time: datetime
) -> Source:
"""Update an existing source from newly scanned data.
@@ -113,24 +113,24 @@ def scan_sources() -> None:
"""
# Manual and periodic scans must contend for the current backend's lock.
with huey.lock_task("scan-webmention-sources"):
- _scan_locked()
+ _scan_and_enqueue()
@dataclass(frozen=True)
-class ScanResult:
+class TreeScanResult:
"""Count and discovered paths from a traversal of the source tree."""
scanned_count: int
seen_paths: frozenset[str]
-def scan_tree(
+def reconcile_source_tree(
root: Path,
base_url: str | None,
sources_by_path: dict[str, Source],
paths_by_url: dict[str, str],
scan_time: datetime,
-) -> ScanResult:
+) -> TreeScanResult:
"""Reconcile found pages and return the paths seen, even on parse failures.
:param root: Root of the generated static-site output tree.
@@ -157,7 +157,7 @@ def scan_tree(
index_path,
root=root,
base_url=base_url,
- ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"],
+ ignored_hostnames=current_app.config["IGNORED_TARGET_HOSTNAMES"],
)
except (OSError, SourceScanError) as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
@@ -182,10 +182,10 @@ def scan_tree(
try:
if (source := sources_by_path.get(scanned.path)) is None:
- source = create_source(scanned, scan_time)
+ source = build_source(scanned, scan_time)
db.session.add(source)
else:
- source = update_source(source, scanned, scan_time)
+ source = apply_scan_to_source(source, scanned, scan_time)
sources_by_path[source.path] = source
paths_by_url[source.url] = source.path
@@ -193,10 +193,10 @@ def scan_tree(
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
- return ScanResult(scanned_count, frozenset(seen_paths))
+ return TreeScanResult(scanned_count, frozenset(seen_paths))
-def mark_missing(
+def mark_missing_sources_deleted(
sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime
) -> int:
"""Mark absent, previously active sources deleted and return their count.
@@ -225,7 +225,7 @@ def mark_missing(
return marked
-def queue_pending(scan_time: datetime) -> int:
+def enqueue_due_sends(scan_time: datetime) -> int:
"""Enqueue due Webmentions and return the number of tasks submitted.
:param scan_time: Current scan timestamp used to evaluate retry deadlines.
@@ -249,7 +249,7 @@ def queue_pending(scan_time: datetime) -> int:
return queued
-def _scan_locked() -> None:
+def _scan_and_enqueue() -> None:
"""Scan sources while the caller holds the shared scan lock.
Reconcile the generated source tree, persist changes, and enqueue eligible outgoing
@@ -257,7 +257,7 @@ def _scan_locked() -> None:
:raises RuntimeError: If the source directory or configured base URL is invalid.
"""
- directory = current_app.config.get("SOURCE_DIRECTORY")
+ directory = current_app.config.get("SITE_OUTPUT_DIRECTORY")
if directory is None:
return
@@ -265,9 +265,9 @@ def _scan_locked() -> None:
if not root.is_dir():
raise RuntimeError(f"Source directory is not a directory: {root}")
- if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None:
+ if (base_url := current_app.config.get("SITE_BASE_URL")) is not None:
if not is_http_url(base_url):
- raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
+ raise RuntimeError("SITE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
sources_by_path = {
@@ -281,7 +281,9 @@ def _scan_locked() -> None:
# take ownership merely by appearing earlier in the traversal.
paths_by_url = {source.url: source.path for source in sources_by_path.values()}
scan_time = datetime.now(UTC)
- scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time)
+ scan = reconcile_source_tree(
+ root, base_url, sources_by_path, paths_by_url, scan_time
+ )
if scan.scanned_count == 0:
current_app.logger.warning(
@@ -290,9 +292,11 @@ def _scan_locked() -> None:
)
return
- missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time)
+ missing_count = mark_missing_sources_deleted(
+ sources_by_path, scan.seen_paths, scan_time
+ )
db.session.commit()
- pending_count = queue_pending(scan_time)
+ pending_count = enqueue_due_sends(scan_time)
current_app.logger.info(
"Webmention source scan complete: %d sources scanned, "