diff options
Diffstat (limited to 'webmentions_ssg/tasks')
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 25 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/discovery.py | 5 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 29 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/source_parser.py | 6 |
6 files changed, 32 insertions, 48 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 474e775..362ebe8 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -21,23 +21,14 @@ def create_consumer() -> Huey: """ app = create_app() - schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True) - queue.periodic_task(schedule)(scanner.scan_sources) - - recovery_schedule = crontab( - *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions) - - notification_schedule = crontab( - *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(notification_schedule)(notifications.send_webmention_digest) - - retention_schedule = crontab( - *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True - ) - queue.periodic_task(retention_schedule)(retention.purge_old_received_webmentions) + for config_key, task in ( + ("SCANNER_SCHEDULE", scanner.scan_sources), + ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions), + ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest), + ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), + ): + schedule = crontab(*app.config[config_key].split(), strict=True) + queue.periodic_task(schedule)(task) return queue.huey diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py index 107d30a..becac58 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/discovery.py @@ -14,6 +14,7 @@ from flask import current_app from ..net import ( ResponseTooLargeError, + content_type_of, is_http_url, is_temporary_status, read_limited_body, @@ -173,9 +174,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non if not response.is_success: raise PermanentSenderError(f"Target returned HTTP {response.status_code}") - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) + media_type = content_type_of(response.headers) if media_type not in {"text/html", "application/xhtml+xml"}: return None diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 78089af..1df363f 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -7,12 +7,12 @@ import uuid from datetime import UTC, datetime, timedelta from http import HTTPStatus -from typing import Any, cast import httpx import sqlalchemy as sa from flask import current_app +from ..db_helpers import rows_affected from ..extensions import db, huey from ..models import ReceivedStatus, ReceivedWebmention from ..net import ( @@ -20,6 +20,7 @@ from ..net import ( ResponseTooLargeError, UnsafeAddressError, build_client, + content_type_of, is_temporary_status, read_limited_body, ) @@ -103,9 +104,7 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: """ response, body = fetch_source(source_url) - media_type = ( - response.headers.get("Content-Type", "").partition(";")[0].strip().lower() - ) + media_type = content_type_of(response.headers) match media_type: case "text/html" | "application/xhtml+xml": @@ -172,7 +171,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: ) ) - if cast(sa.CursorResult[Any], claimed).rowcount != 1: + if rows_affected(claimed) != 1: db.session.rollback() return @@ -285,7 +284,7 @@ def recover_stale_webmentions() -> int: ) ) - if cast(sa.CursorResult[Any], changed).rowcount != 1: + if rows_affected(changed) != 1: db.session.rollback() continue diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 754c891..86283b7 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -5,12 +5,11 @@ """Delete old received Webmentions in terminal unsuccessful states.""" from datetime import UTC, datetime, timedelta -from typing import Any, cast import sqlalchemy as sa from flask import current_app -from sqlalchemy.engine import CursorResult +from ..db_helpers import rows_affected from ..extensions import db from ..models import ReceivedStatus, ReceivedWebmention @@ -26,22 +25,18 @@ def purge_old_received_webmentions() -> int: days=current_app.config["RECEIVED_RETENTION_DAYS"] ) - result = cast( - CursorResult[Any], - db.session.execute( - sa.delete(ReceivedWebmention).where( - ReceivedWebmention.status.in_( - (ReceivedStatus.FAILED, ReceivedStatus.DELETED) - ), - ReceivedWebmention.updated_at < cutoff, - ) - ), + result = db.session.execute( + sa.delete(ReceivedWebmention).where( + ReceivedWebmention.status.in_( + (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ), + ReceivedWebmention.updated_at < cutoff, + ) ) db.session.commit() - if result.rowcount: - current_app.logger.info( - "Purged %d old failed/deleted Webmentions", result.rowcount - ) + count = rows_affected(result) + if count: + current_app.logger.info("Purged %d old failed/deleted Webmentions", count) - return result.rowcount + return count diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 4f994d4..92d8dfa 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,13 +6,13 @@ import uuid from datetime import UTC, datetime, timedelta -from typing import Any, cast import httpx import sqlalchemy as sa from flask import current_app from huey.exceptions import TaskLockedException +from ..db_helpers import rows_affected from ..extensions import db, huey from ..models import SentWebmention, SentWebmentionStatus from ..net import ( @@ -159,7 +159,7 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None: ) db.session.commit() - if cast(sa.CursorResult[Any], claim).rowcount != 1: + if rows_affected(claim) != 1: return db.session.refresh(webmention) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index 9cbe071..294402c 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -11,14 +11,14 @@ from fnmatch import fnmatchcase from itertools import chain from pathlib import Path from typing import Any -from urllib.parse import quote, urldefrag, urljoin, urlsplit +from urllib.parse import quote, urldefrag, urljoin import mf2py import xxhash from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement -from ..net import is_http_url +from ..net import hostname_of, is_http_url REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") @@ -283,7 +283,7 @@ def iter_targets( for value in chain(hrefs, reactions): target = normalize_target(value, base_url=base_url, source_url=source_url) if target is not None: - hostname = (urlsplit(target).hostname or "").rstrip(".") + hostname = hostname_of(target) or "" if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): yield target |
