From f6d96df2c3aecb51ca97426efddb101dce6350d7 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 13:24:53 +0100 Subject: feat(sender): add persistent retry backoff Persist retry state per source revision so temporary delivery failures survive worker restarts and duplicate queued tasks cannot bypass the retry schedule. Use capped exponential backoff, stop after a configurable attempt limit, and reset the retry budget when a source changes or an admin resends manually. --- contrib/webmentions-ssg.example.cfg | 8 ++ tests/cli/test_database.py | 5 +- tests/tasks/test_scanner.py | 51 ++++++++ tests/tasks/test_sender.py | 130 +++++++++++++++++++-- tests/test_views.py | 9 ++ webmentions_ssg/config.py | 4 + .../85f2b226ec99_add_webmention_retry_backoff.py | 38 ++++++ webmentions_ssg/models.py | 8 ++ webmentions_ssg/tasks/scanner.py | 10 +- webmentions_ssg/tasks/sender.py | 117 ++++++++++++++++--- webmentions_ssg/views.py | 2 + 11 files changed, 358 insertions(+), 24 deletions(-) create mode 100644 webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index d214312..7dfdd7a 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -79,6 +79,14 @@ SOURCE_BASE_URL = "https://example.com/" # Huey cron expression: minute hour day-of-month month day-of-week. SCANNER_SCHEDULE = "0 * * * *" # every hour +# Temporary delivery failures use persisted exponential backoff: 1, 2, 4, +# 8, then 16 hours between attempts with these defaults. After six total +# attempts for a revision, the failure becomes terminal. A changed source or +# explicit manual resend starts a fresh retry budget. +SEND_MAX_ATTEMPTS = 6 +SEND_RETRY_BASE_SECONDS = 3600 +SEND_RETRY_MAX_SECONDS = 86400 + # Do not send Webmentions to your own site. Glob patterns are supported here. IGNORED_HOSTNAMES = ("example.com", "*.example.com") MAX_TARGET_BYTES = 1_000_000 diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 51b79ac..f25aade 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -80,8 +80,11 @@ def test_create_db_builds_schema_and_records_latest_migration( assert _database_revision(database_app) == _head_revision(database_app) with database_app.app_context(): - table_names = set(inspect(DATABASE.engine).get_table_names()) + inspector = inspect(DATABASE.engine) + table_names = set(inspector.get_table_names()) + columns = {column["name"] for column in inspector.get_columns("sent_webmentions")} assert {"alembic_version", "users", "received_webmentions"} <= table_names + assert {"attempt_count", "next_retry_at"} <= columns def test_create_db_refuses_existing_database_without_modifying_it( diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 3d138ac..6ace77f 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +from datetime import UTC, datetime, timedelta from pathlib import Path from types import ModuleType from typing import Any @@ -1552,6 +1553,56 @@ def test_pending_webmention_is_requeued_on_next_scan( assert queued == [identifier] + +def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + path = write_post(tmp_path, "example", 'A') + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + assert len(queued) == 1 + identifier = queued.pop() + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.attempt_count = 2 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=4) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [] + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [identifier] + queued.clear() + + # Changing the h-entry creates a new desired revision with a fresh budget. + path.write_text(path.read_text().replace('>A', '>A updated')) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + webmention.next_retry_at = datetime.now(UTC) + timedelta(days=2) + db.session.commit() + + run_scan(app, scanner_module) + assert queued == [identifier] + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None + def test_invalid_known_source_is_not_deleted( app: Flask, scanner_module: ModuleType, diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index ecfc27a..6b189ad 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -4,7 +4,7 @@ import logging import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from types import ModuleType from unittest.mock import MagicMock, Mock @@ -34,6 +34,8 @@ def create_sent_webmention( endpoint: str | None = None, response_status: int | None = None, status_url: str | None = None, + attempt_count: int = 0, + next_retry_at: datetime | None = None, ) -> uuid.UUID: now = datetime.now(UTC) @@ -58,6 +60,8 @@ def create_sent_webmention( endpoint=endpoint, response_status=response_status, status_url=status_url, + attempt_count=attempt_count, + next_retry_at=next_retry_at, ) source.sent_webmentions.append(webmention) @@ -686,6 +690,8 @@ def test_send_webmention_persists_success( assert webmention.status_url == STATUS_URL assert webmention.last_attempted_at is not None assert webmention.last_sent_at is not None + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is None def test_send_webmention_marks_unsupported_target_processed( @@ -754,7 +760,7 @@ def test_send_webmention_persists_permanent_failure( assert webmention.status_url is None -def test_send_webmention_persists_temporary_failure_and_reraises( +def test_send_webmention_persists_temporary_failure_with_backoff( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( @@ -768,11 +774,7 @@ def test_send_webmention_persists_temporary_failure_and_reraises( mock_sender_requests(sender_module, monkeypatch, result=(503, None)) - with pytest.raises( - sender_module.TemporarySenderError, - match="Webmention endpoint returned HTTP 503", - ): - sender_module.send_webmention.call_local(identifier) + sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) @@ -785,6 +787,120 @@ def test_send_webmention_persists_temporary_failure_and_reraises( assert webmention.endpoint == ENDPOINT_URL assert webmention.response_status == 503 assert webmention.status_url is None + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is not None + assert webmention.last_attempted_at is not None + # SQLite drops timezone info when loading timestamp columns. + assert ( + webmention.next_retry_at.replace(tzinfo=None) + - webmention.last_attempted_at.replace(tzinfo=None) + ).total_seconds() == 3600 + + +def test_retry_delay_is_exponential_and_capped( + app: Flask, sender_module: ModuleType +) -> None: + with app.app_context(): + assert [sender_module.retry_delay(n).total_seconds() for n in range(1, 8)] == [ + 3600, + 7200, + 14400, + 28800, + 57600, + 86400, + 86400, + ] + + +def test_temporary_failure_backoff_suppresses_duplicate_tasks_and_caps_attempts( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 3 + identifier = create_sent_webmention(app) + _, _, discover, post = mock_sender_requests( + sender_module, monkeypatch, result=(503, None) + ) + + for attempt in range(1, 4): + sender_module.send_webmention.call_local(identifier) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.attempt_count == attempt + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.processed_revision == (1 if attempt == 3 else None) + if attempt < 3: + assert webmention.next_retry_at is not None + sender_module.send_webmention.call_local(identifier) + assert webmention.attempt_count == attempt + webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1) + db.session.commit() + else: + assert webmention.next_retry_at is None + + # A terminal failure stays terminal across scans or stray queued tasks. + sender_module.send_webmention.call_local(identifier) + assert discover.call_count == 3 + assert post.call_count == 3 + + +def test_sender_does_not_retry_before_due_even_after_restart( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + attempt_count=2, + next_retry_at=datetime.now(UTC) + timedelta(days=1), + ) + httpx_client, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) + sender_module.send_webmention.call_local(identifier) + httpx_client.assert_not_called() + discover.assert_not_called() + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.attempt_count == 2 + assert webmention.processed_revision is None + + +def test_sender_finishes_abandoned_final_attempt_without_new_request( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 2 + identifier = create_sent_webmention( + app, + attempt_count=2, + next_retry_at=datetime.now(UTC) - timedelta(seconds=1), + status=SentWebmentionStatus.FAILED, + failure_reason="Network unreachable", + ) + httpx_client, _, _, _ = mock_sender_requests(sender_module, monkeypatch) + sender_module.send_webmention.call_local(identifier) + httpx_client.assert_not_called() + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.processed_revision == 1 + assert webmention.failure_reason == "Network unreachable" + assert webmention.next_retry_at is None + + +def test_sender_request_error_uses_same_retry_budget( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["SEND_MAX_ATTEMPTS"] = 1 + identifier = create_sent_webmention(app) + _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) + discover.side_effect = httpx.ConnectError("DNS lookup failed") + sender_module.send_webmention.call_local(identifier) + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + assert webmention is not None + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.failure_reason == "DNS lookup failed" + assert webmention.processed_revision == 1 + assert webmention.attempt_count == 1 + assert webmention.next_retry_at is None def test_send_webmention_ignores_processed_revision( diff --git a/tests/test_views.py b/tests/test_views.py index 8ac57b1..fb12b8b 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -693,6 +693,13 @@ def test_resend_sent_webmention_reopens_current_revision( send_webmention = Mock() monkeypatch.setattr(views_module, "send_webmention", send_webmention) + with app.app_context(): + webmention = db.session.get(SentWebmention, webmention_identifier) + assert webmention is not None + webmention.attempt_count = 6 + webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24) + db.session.commit() + response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend") assert response.status_code == 302 @@ -703,6 +710,8 @@ def test_resend_sent_webmention_reopens_current_revision( assert webmention is not None assert webmention.desired_revision == 3 assert webmention.processed_revision is None + assert webmention.attempt_count == 0 + assert webmention.next_retry_at is None assert webmention.sent_revision == 2 assert webmention.status is None assert webmention.failure_reason is None diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 8cad51f..3aec00d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -49,6 +49,10 @@ class DefaultConfig: SOURCE_DIRECTORY = None SOURCE_BASE_URL = None SCANNER_SCHEDULE = "0 * * * *" + # Total send attempts per source revision (not additional retries). + SEND_MAX_ATTEMPTS = 6 + SEND_RETRY_BASE_SECONDS = 3600 + SEND_RETRY_MAX_SECONDS = 86400 NOTIFICATION_EMAIL = None NOTIFICATION_SCHEDULE = "0 8 * * *" diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py new file mode 100644 index 0000000..c4eac52 --- /dev/null +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -0,0 +1,38 @@ +"""Add Webmention retry backoff + +Revision ID: 85f2b226ec99 +Revises: 408fc2eca9ce +Create Date: 2026-10-30 13:08:17.340631 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "85f2b226ec99" +down_revision = "408fc2eca9ce" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.add_column( + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0") + ) + batch_op.add_column( + sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True) + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.drop_column("next_retry_at") + batch_op.drop_column("attempt_count") + + # ### end Alembic commands ### diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 11740a8..4f688e5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -245,6 +245,14 @@ class SentWebmention(Base): ) status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column( + Integer(), nullable=False, default=0 + ) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_attempted_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 7e163f7..03f4b8d 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -385,6 +385,8 @@ def update_source( for webmention in existing.values(): webmention.active = webmention.target in scanned.targets webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -522,6 +524,8 @@ def scan_sources() -> None: for webmention in source.sent_webmentions: webmention.active = False webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -536,7 +540,11 @@ def scan_sources() -> None: sa.or_( SentWebmention.processed_revision.is_(None), SentWebmention.processed_revision < SentWebmention.desired_revision, - ) + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= scan_time, + ), ) .order_by(SentWebmention.uuid) ): diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 8834934..4aa59ee 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,10 +6,11 @@ import re import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: ) -@huey.task(retries=2, retry_delay=50) +def retry_delay(attempt: int) -> timedelta: + """Calculate the capped exponential delay after a numbered send attempt.""" + base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) + maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) + return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) + + +@huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: """ Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. - Temporary failures are re-raised so Huey can retry them, while permanent - failures mark the current revision as processed. + Temporary failures are recorded with an exponential retry deadline. The + scanner requeues them when due; Huey does not retry them independently. + The retry budget is per desired source revision. :param webmention_uuid: Identifier of the sent Webmention to process. - :raises TemporarySenderError: If endpoint discovery or delivery fails for a - potentially temporary reason. - :raises httpx.RequestError: If an HTTP request fails. """ webmention = db.session.get(SentWebmention, webmention_uuid) @@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: revision = webmention.desired_revision source = webmention.source.url target = webmention.target + now = datetime.now(UTC) + deadline = webmention.next_retry_at - webmention.last_attempted_at = datetime.now(UTC) - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None + # SQLite returns timezone-naive datetimes even for timezone-aware columns. + if deadline is not None: + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + if aware_deadline > now: + return + + max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: + # Recover records whose worker died during the final attempt, without + # touching a newer revision that may have been queued in the meantime. + db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count >= max_attempts, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + processed_revision=revision, + next_retry_at=None, + status=SentWebmentionStatus.FAILED, + failure_reason=( + webmention.failure_reason + or f"Retry limit reached ({max_attempts} attempts)" + ), + ) + .execution_options(synchronize_session=False) + ) + db.session.commit() + return + attempt = webmention.attempt_count + 1 + next_retry_at = now + retry_delay(attempt) + + # Claim the attempt atomically: duplicate queued tasks must not send the + # same revision concurrently or bypass its retry deadline. + claim = db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count == attempt - 1, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + attempt_count=attempt, + last_attempted_at=now, + next_retry_at=next_retry_at, + endpoint=None, + response_status=None, + status_url=None, + ) + .execution_options(synchronize_session=False) + ) db.session.commit() + if claim.rowcount != 1: + return + + db.session.refresh(webmention) + endpoint: str | None = None response_status: int | None = None @@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.UNSUPPORTED webmention.failure_reason = "No Webmention endpoint discovered" webmention.endpoint = None @@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) or "Webmention request failed" + if attempt >= max_attempts: + webmention.processed_revision = revision + webmention.next_retry_at = None + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + webmention.failure_reason, + ) webmention.endpoint = endpoint webmention.response_status = response_status db.session.commit() - # Leave processed_revision unchanged. Huey may retry the - # attempt, and the periodic scanner will also rediscover - # outstanding work. - raise + # No Huey retries: only the scanner may requeue after next_retry_at. + return except PermanentSenderError as exc: if not attempt_is_current(webmention, revision): return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) webmention.endpoint = endpoint @@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.sent_revision = revision webmention.status = SentWebmentionStatus.SENT diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index 2ab02fa..1dcdd2e 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -296,6 +296,8 @@ def resend_sent_webmention( # revision. send_webmention() otherwise exits when this revision has already # been processed. webmention.processed_revision = None + webmention.attempt_count = 0 + webmention.next_retry_at = None webmention.status = None webmention.failure_reason = None webmention.endpoint = None -- cgit v1.3.1