diff options
Diffstat (limited to '')
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 8 | ||||
| -rw-r--r-- | tests/cli/test_database.py | 5 | ||||
| -rw-r--r-- | tests/tasks/test_scanner.py | 51 | ||||
| -rw-r--r-- | tests/tasks/test_sender.py | 130 | ||||
| -rw-r--r-- | tests/test_views.py | 9 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py | 38 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 117 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 2 |
11 files changed, 358 insertions, 24 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index d214312..7dfdd7a 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -79,6 +79,14 @@ SOURCE_BASE_URL = "https://example.com/" # Huey cron expression: minute hour day-of-month month day-of-week. SCANNER_SCHEDULE = "0 * * * *" # every hour +# Temporary delivery failures use persisted exponential backoff: 1, 2, 4, +# 8, then 16 hours between attempts with these defaults. After six total +# attempts for a revision, the failure becomes terminal. A changed source or +# explicit manual resend starts a fresh retry budget. +SEND_MAX_ATTEMPTS = 6 +SEND_RETRY_BASE_SECONDS = 3600 +SEND_RETRY_MAX_SECONDS = 86400 + # Do not send Webmentions to your own site. Glob patterns are supported here. IGNORED_HOSTNAMES = ("example.com", "*.example.com") MAX_TARGET_BYTES = 1_000_000 diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 51b79ac..f25aade 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -80,8 +80,11 @@ def test_create_db_builds_schema_and_records_latest_migration( assert _database_revision(database_app) == _head_revision(database_app) with database_app.app_context(): - table_names = set(inspect(DATABASE.engine).get_table_names()) + inspector = inspect(DATABASE.engine) + table_names = set(inspector.get_table_names()) + columns = {column["name"] for column in inspector.get_columns("sent_webmentions")} assert {"alembic_version", "users", "received_webmentions"} <= table_names + assert {"attempt_count", "next_retry_at"} <= columns def test_create_db_refuses_existing_database_without_modifying_it( diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 3d138ac..6ace77f 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +from datetime import UTC, datetime, timedelta from pathlib import Path from types import ModuleType from typing import Any @@ -1552,6 +1553,56 @@ def test_pending_webmention_is_requeued_on_next_scan( assert queued == [identifier] + +def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + path = write_post(tmp_path, "example", '<a href="https://example.com/a">A</a>') + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + assert len(queued) == 1 + identifier = queued.pop() + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.attempt_count = 2 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=4) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [] + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [identifier] + queued.clear() + + # Changing the h-entry creates a new desired revision with a fresh budget. + path.write_text(path.read_text().replace('>A</a>', '>A updated</a>')) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.next_retry_at = datetime.now(UTC) + timedelta(days=2) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [identifier] + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None + def test_invalid_known_source_is_not_deleted( app: Flask, scanner_module: ModuleType, diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index ecfc27a..6b189ad 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -4,7 +4,7 @@ import logging import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from types import ModuleType from unittest.mock import MagicMock, Mock @@ -34,6 +34,8 @@ def create_sent_webmention( endpoint: str | None = None, response_status: int | None = None, status_url: str | None = None, + attempt_count: int = 0, + next_retry_at: datetime | None = None, ) -> uuid.UUID: now = datetime.now(UTC) @@ -58,6 +60,8 @@ def create_sent_webmention( endpoint=endpoint, response_status=response_status, status_url=status_url, + attempt_count=attempt_count, + next_retry_at=next_retry_at, ) source.sent_webmentions.append(webmention) @@ -686,6 +690,8 @@ def test_send_webmention_persists_success( assert webmention.status_url == STATUS_URL assert webmention.last_attempted_at is not None assert webmention.last_sent_at is not None + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is None def test_send_webmention_marks_unsupported_target_processed( @@ -754,7 +760,7 @@ def test_send_webmention_persists_permanent_failure( assert webmention.status_url is None -def test_send_webmention_persists_temporary_failure_and_reraises( +def test_send_webmention_persists_temporary_failure_with_backoff( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( @@ -768,11 +774,7 @@ def test_send_webmention_persists_temporary_failure_and_reraises( mock_sender_requests(sender_module, monkeypatch, result=(503, None)) - with pytest.raises( - sender_module.TemporarySenderError, - match="Webmention endpoint returned HTTP 503", - ): - sender_module.send_webmention.call_local(identifier) + sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) @@ -785,6 +787,120 @@ def test_send_webmention_persists_temporary_failure_and_reraises( assert webmention.endpoint == ENDPOINT_URL assert webmention.response_status == 503 assert webmention.status_url is None + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is not None + assert webmention.last_attempted_at is not None + # SQLite drops timezone info when loading timestamp columns. + assert ( + webmention.next_retry_at.replace(tzinfo=None) + - webmention.last_attempted_at.replace(tzinfo=None) + ).total_seconds() == 3600 + + +def test_retry_delay_is_exponential_and_capped( + app: Flask, sender_module: ModuleType +) -> None: + with app.app_context(): + assert [sender_module.retry_delay(n).total_seconds() for n in range(1, 8)] == [ + 3600, + 7200, + 14400, + 28800, + 57600, + 86400, + 86400, + ] + + +def test_temporary_failure_backoff_suppresses_duplicate_tasks_and_caps_attempts( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 3 + identifier = create_sent_webmention(app) + _, _, discover, post = mock_sender_requests( + sender_module, monkeypatch, result=(503, None) + ) + + for attempt in range(1, 4): + sender_module.send_webmention.call_local(identifier) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.attempt_count == attempt + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.processed_revision == (1 if attempt == 3 else None) + if attempt < 3: + assert webmention.next_retry_at is not None + sender_module.send_webmention.call_local(identifier) + assert webmention.attempt_count == attempt + webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1) + db.session.commit() + else: + assert webmention.next_retry_at is None + + # A terminal failure stays terminal across scans or stray queued tasks. + sender_module.send_webmention.call_local(identifier) + assert discover.call_count == 3 + assert post.call_count == 3 + + +def test_sender_does_not_retry_before_due_even_after_restart( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + attempt_count=2, + next_retry_at=datetime.now(UTC) + timedelta(days=1), + ) + httpx_client, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) + sender_module.send_webmention.call_local(identifier) + httpx_client.assert_not_called() + discover.assert_not_called() + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.attempt_count == 2 + assert webmention.processed_revision is None + + +def test_sender_finishes_abandoned_final_attempt_without_new_request( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 2 + identifier = create_sent_webmention( + app, + attempt_count=2, + next_retry_at=datetime.now(UTC) - timedelta(seconds=1), + status=SentWebmentionStatus.FAILED, + failure_reason="Network unreachable", + ) + httpx_client, _, _, _ = mock_sender_requests(sender_module, monkeypatch) + sender_module.send_webmention.call_local(identifier) + httpx_client.assert_not_called() + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.processed_revision == 1 + assert webmention.failure_reason == "Network unreachable" + assert webmention.next_retry_at is None + + +def test_sender_request_error_uses_same_retry_budget( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 1 + identifier = create_sent_webmention(app) + _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) + discover.side_effect = httpx.ConnectError("DNS lookup failed") + sender_module.send_webmention.call_local(identifier) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.failure_reason == "DNS lookup failed" + assert webmention.processed_revision == 1 + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is None def test_send_webmention_ignores_processed_revision( diff --git a/tests/test_views.py b/tests/test_views.py index 8ac57b1..fb12b8b 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -693,6 +693,13 @@ def test_resend_sent_webmention_reopens_current_revision( send_webmention = Mock() monkeypatch.setattr(views_module, "send_webmention", send_webmention) + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + webmention.attempt_count = 6 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) + db.session.commit() + response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") assert response.status_code == 302 @@ -703,6 +710,8 @@ def test_resend_sent_webmention_reopens_current_revision( assert webmention is not None assert webmention.desired_revision == 3 assert webmention.processed_revision is None + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None assert webmention.sent_revision == 2 assert webmention.status is None assert webmention.failure_reason is None diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 8cad51f..3aec00d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -49,6 +49,10 @@ class DefaultConfig: SOURCE_DIRECTORY = None SOURCE_BASE_URL = None SCANNER_SCHEDULE = "0 * * * *" + # Total send attempts per source revision (not additional retries). + SEND_MAX_ATTEMPTS = 6 + SEND_RETRY_BASE_SECONDS = 3600 + SEND_RETRY_MAX_SECONDS = 86400 NOTIFICATION_EMAIL = None NOTIFICATION_SCHEDULE = "0 8 * * *" diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py new file mode 100644 index 0000000..c4eac52 --- /dev/null +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -0,0 +1,38 @@ +"""Add Webmention retry backoff + +Revision ID: 85f2b226ec99 +Revises: 408fc2eca9ce +Create Date: 2026-10-30 13:08:17.340631 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "85f2b226ec99" +down_revision = "408fc2eca9ce" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.add_column( + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0") + ) + batch_op.add_column( + sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True) + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.drop_column("next_retry_at") + batch_op.drop_column("attempt_count") + + # ### end Alembic commands ### diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 11740a8..4f688e5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -245,6 +245,14 @@ class SentWebmention(Base): ) status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column( + Integer(), nullable=False, default=0 + ) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_attempted_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 7e163f7..03f4b8d 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -385,6 +385,8 @@ def update_source( for webmention in existing.values(): webmention.active = webmention.target in scanned.targets webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -522,6 +524,8 @@ def scan_sources() -> None: for webmention in source.sent_webmentions: webmention.active = False webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -536,7 +540,11 @@ def scan_sources() -> None: sa.or_( SentWebmention.processed_revision.is_(None), SentWebmention.processed_revision < SentWebmention.desired_revision, - ) + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= scan_time, + ), ) .order_by(SentWebmention.uuid) ): diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 8834934..4aa59ee 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,10 +6,11 @@ import re import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: ) -@huey.task(retries=2, retry_delay=50) +def retry_delay(attempt: int) -> timedelta: + """Calculate the capped exponential delay after a numbered send attempt.""" + base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) + maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) + return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) + + +@huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: """ Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. - Temporary failures are re-raised so Huey can retry them, while permanent - failures mark the current revision as processed. + Temporary failures are recorded with an exponential retry deadline. The + scanner requeues them when due; Huey does not retry them independently. + The retry budget is per desired source revision. :param webmention_uuid: Identifier of the sent Webmention to process. - :raises TemporarySenderError: If endpoint discovery or delivery fails for a - potentially temporary reason. - :raises httpx.RequestError: If an HTTP request fails. """ webmention = db.session.get(SentWebmention, webmention_uuid) @@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: revision = webmention.desired_revision source = webmention.source.url target = webmention.target + now = datetime.now(UTC) + deadline = webmention.next_retry_at + + # SQLite returns timezone-naive datetimes even for timezone-aware columns. + if deadline is not None: + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + if aware_deadline > now: + return + + max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: + # Recover records whose worker died during the final attempt, without + # touching a newer revision that may have been queued in the meantime. + db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count >= max_attempts, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + processed_revision=revision, + next_retry_at=None, + status=SentWebmentionStatus.FAILED, + failure_reason=( + webmention.failure_reason + or f"Retry limit reached ({max_attempts} attempts)" + ), + ) + .execution_options(synchronize_session=False) + ) + db.session.commit() + return - webmention.last_attempted_at = datetime.now(UTC) - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None + attempt = webmention.attempt_count + 1 + next_retry_at = now + retry_delay(attempt) + # Claim the attempt atomically: duplicate queued tasks must not send the + # same revision concurrently or bypass its retry deadline. + claim = db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count == attempt - 1, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + attempt_count=attempt, + last_attempted_at=now, + next_retry_at=next_retry_at, + endpoint=None, + response_status=None, + status_url=None, + ) + .execution_options(synchronize_session=False) + ) db.session.commit() + if claim.rowcount != 1: + return + + db.session.refresh(webmention) + endpoint: str | None = None response_status: int | None = None @@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.UNSUPPORTED webmention.failure_reason = "No Webmention endpoint discovered" webmention.endpoint = None @@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) or "Webmention request failed" + if attempt >= max_attempts: + webmention.processed_revision = revision + webmention.next_retry_at = None + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + webmention.failure_reason, + ) webmention.endpoint = endpoint webmention.response_status = response_status db.session.commit() - # Leave processed_revision unchanged. Huey may retry the - # attempt, and the periodic scanner will also rediscover - # outstanding work. - raise + # No Huey retries: only the scanner may requeue after next_retry_at. + return except PermanentSenderError as exc: if not attempt_is_current(webmention, revision): return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) webmention.endpoint = endpoint @@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.sent_revision = revision webmention.status = SentWebmentionStatus.SENT diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index 2ab02fa..1dcdd2e 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -296,6 +296,8 @@ def resend_sent_webmention( # revision. send_webmention() otherwise exits when this revision has already # been processed. webmention.processed_revision = None + webmention.attempt_count = 0 + webmention.next_retry_at = None webmention.status = None webmention.failure_reason = None webmention.endpoint = None |
