diff options
Diffstat (limited to 'webmentions_ssg/tasks')
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 14 |
1 files changed, 12 insertions, 2 deletions
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 5ab2521..4bb6736 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -459,7 +459,18 @@ def scan_sources() -> None: base_url=base_url, ignored_hostnames=ignored_hostnames, ) + except (OSError, SourceScanError) as exc: + current_app.logger.warning("Could not scan source %s: %s", index_path, exc) + continue + except Exception: + # Third-party HTML parsers can raise unexpected exceptions (including + # RecursionError). A bad page must not prevent other sources being saved. + current_app.logger.exception( + "Unexpected error scanning source %s", index_path + ) + continue + try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) db.session.add(source) @@ -468,8 +479,7 @@ def scan_sources() -> None: sources_by_path[source.path] = source scanned_count += 1 - - except (OSError, SourceScanError) as exc: + except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) for path, source in sources_by_path.items(): |
