aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks')
-rw-r--r--webmentions_ssg/tasks/consumer.py25
-rw-r--r--webmentions_ssg/tasks/discovery.py5
-rw-r--r--webmentions_ssg/tasks/receiver.py11
-rw-r--r--webmentions_ssg/tasks/retention.py29
-rw-r--r--webmentions_ssg/tasks/sender.py4
-rw-r--r--webmentions_ssg/tasks/source_parser.py6
6 files changed, 32 insertions, 48 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py
index 474e775..362ebe8 100644
--- a/webmentions_ssg/tasks/consumer.py
+++ b/webmentions_ssg/tasks/consumer.py
@@ -21,23 +21,14 @@ def create_consumer() -> Huey:
"""
app = create_app()
- schedule = crontab(*app.config["SCANNER_SCHEDULE"].split(), strict=True)
- queue.periodic_task(schedule)(scanner.scan_sources)
-
- recovery_schedule = crontab(
- *app.config["VERIFICATION_REAPER_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(recovery_schedule)(receiver.recover_stale_webmentions)
-
- notification_schedule = crontab(
- *app.config["NOTIFICATION_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(notification_schedule)(notifications.send_webmention_digest)
-
- retention_schedule = crontab(
- *app.config["RECEIVED_RETENTION_SCHEDULE"].split(), strict=True
- )
- queue.periodic_task(retention_schedule)(retention.purge_old_received_webmentions)
+ for config_key, task in (
+ ("SCANNER_SCHEDULE", scanner.scan_sources),
+ ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions),
+ ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest),
+ ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions),
+ ):
+ schedule = crontab(*app.config[config_key].split(), strict=True)
+ queue.periodic_task(schedule)(task)
return queue.huey
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index 107d30a..becac58 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -14,6 +14,7 @@ from flask import current_app
from ..net import (
ResponseTooLargeError,
+ content_type_of,
is_http_url,
is_temporary_status,
read_limited_body,
@@ -173,9 +174,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
if not response.is_success:
raise PermanentSenderError(f"Target returned HTTP {response.status_code}")
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
+ media_type = content_type_of(response.headers)
if media_type not in {"text/html", "application/xhtml+xml"}:
return None
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 78089af..1df363f 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -7,12 +7,12 @@
import uuid
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
-from typing import Any, cast
import httpx
import sqlalchemy as sa
from flask import current_app
+from ..db_helpers import rows_affected
from ..extensions import db, huey
from ..models import ReceivedStatus, ReceivedWebmention
from ..net import (
@@ -20,6 +20,7 @@ from ..net import (
ResponseTooLargeError,
UnsafeAddressError,
build_client,
+ content_type_of,
is_temporary_status,
read_limited_body,
)
@@ -103,9 +104,7 @@ def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
response, body = fetch_source(source_url)
- media_type = (
- response.headers.get("Content-Type", "").partition(";")[0].strip().lower()
- )
+ media_type = content_type_of(response.headers)
match media_type:
case "text/html" | "application/xhtml+xml":
@@ -172,7 +171,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
)
- if cast(sa.CursorResult[Any], claimed).rowcount != 1:
+ if rows_affected(claimed) != 1:
db.session.rollback()
return
@@ -285,7 +284,7 @@ def recover_stale_webmentions() -> int:
)
)
- if cast(sa.CursorResult[Any], changed).rowcount != 1:
+ if rows_affected(changed) != 1:
db.session.rollback()
continue
diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py
index 754c891..86283b7 100644
--- a/webmentions_ssg/tasks/retention.py
+++ b/webmentions_ssg/tasks/retention.py
@@ -5,12 +5,11 @@
"""Delete old received Webmentions in terminal unsuccessful states."""
from datetime import UTC, datetime, timedelta
-from typing import Any, cast
import sqlalchemy as sa
from flask import current_app
-from sqlalchemy.engine import CursorResult
+from ..db_helpers import rows_affected
from ..extensions import db
from ..models import ReceivedStatus, ReceivedWebmention
@@ -26,22 +25,18 @@ def purge_old_received_webmentions() -> int:
days=current_app.config["RECEIVED_RETENTION_DAYS"]
)
- result = cast(
- CursorResult[Any],
- db.session.execute(
- sa.delete(ReceivedWebmention).where(
- ReceivedWebmention.status.in_(
- (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
- ),
- ReceivedWebmention.updated_at < cutoff,
- )
- ),
+ result = db.session.execute(
+ sa.delete(ReceivedWebmention).where(
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.FAILED, ReceivedStatus.DELETED)
+ ),
+ ReceivedWebmention.updated_at < cutoff,
+ )
)
db.session.commit()
- if result.rowcount:
- current_app.logger.info(
- "Purged %d old failed/deleted Webmentions", result.rowcount
- )
+ count = rows_affected(result)
+ if count:
+ current_app.logger.info("Purged %d old failed/deleted Webmentions", count)
- return result.rowcount
+ return count
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 4f994d4..92d8dfa 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -6,13 +6,13 @@
import uuid
from datetime import UTC, datetime, timedelta
-from typing import Any, cast
import httpx
import sqlalchemy as sa
from flask import current_app
from huey.exceptions import TaskLockedException
+from ..db_helpers import rows_affected
from ..extensions import db, huey
from ..models import SentWebmention, SentWebmentionStatus
from ..net import (
@@ -159,7 +159,7 @@ def _send_locked(webmention_uuid: uuid.UUID) -> None:
)
db.session.commit()
- if cast(sa.CursorResult[Any], claim).rowcount != 1:
+ if rows_affected(claim) != 1:
return
db.session.refresh(webmention)
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 9cbe071..294402c 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -11,14 +11,14 @@ from fnmatch import fnmatchcase
from itertools import chain
from pathlib import Path
from typing import Any
-from urllib.parse import quote, urldefrag, urljoin, urlsplit
+from urllib.parse import quote, urldefrag, urljoin
import mf2py
import xxhash
from bs4 import BeautifulSoup, Tag
from bs4.element import NavigableString, PageElement
-from ..net import is_http_url
+from ..net import hostname_of, is_http_url
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
@@ -283,7 +283,7 @@ def iter_targets(
for value in chain(hrefs, reactions):
target = normalize_target(value, base_url=base_url, source_url=source_url)
if target is not None:
- hostname = (urlsplit(target).hostname or "").rstrip(".")
+ hostname = hostname_of(target) or ""
if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
yield target