aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/scanner.py14
1 files changed, 12 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 5ab2521..4bb6736 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -459,7 +459,18 @@ def scan_sources() -> None:
base_url=base_url,
ignored_hostnames=ignored_hostnames,
)
+ except (OSError, SourceScanError) as exc:
+ current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
+ continue
+ except Exception:
+ # Third-party HTML parsers can raise unexpected exceptions (including
+ # RecursionError). A bad page must not prevent other sources being saved.
+ current_app.logger.exception(
+ "Unexpected error scanning source %s", index_path
+ )
+ continue
+ try:
if (source := sources_by_path.get(scanned.path)) is None:
source = create_source(scanned, scan_time)
db.session.add(source)
@@ -468,8 +479,7 @@ def scan_sources() -> None:
sources_by_path[source.path] = source
scanned_count += 1
-
- except (OSError, SourceScanError) as exc:
+ except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)
for path, source in sources_by_path.items():