aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 13:24:53 +0100
committerDennis Fink2026-10-30 13:24:53 +0100
commitf6d96df2c3aecb51ca97426efddb101dce6350d7 (patch)
tree2968f65a627fbfd15051277d07eaa321db0e3cc4 /tests
parent2b59a3082000842d980f9e19931949dbe5011229 (diff)
downloadwebmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.tar.gz
webmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.zip
feat(sender): add persistent retry backoff
Persist retry state per source revision so temporary delivery failures survive worker restarts and duplicate queued tasks cannot bypass the retry schedule. Use capped exponential backoff, stop after a configurable attempt limit, and reset the retry budget when a source changes or an admin resends manually.
Diffstat (limited to '')
-rw-r--r--tests/cli/test_database.py5
-rw-r--r--tests/tasks/test_scanner.py51
-rw-r--r--tests/tasks/test_sender.py130
-rw-r--r--tests/test_views.py9
4 files changed, 187 insertions, 8 deletions
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
index 51b79ac..f25aade 100644
--- a/tests/cli/test_database.py
+++ b/tests/cli/test_database.py
@@ -80,8 +80,11 @@ def test_create_db_builds_schema_and_records_latest_migration(
assert _database_revision(database_app) == _head_revision(database_app)
with database_app.app_context():
- table_names = set(inspect(DATABASE.engine).get_table_names())
+ inspector = inspect(DATABASE.engine)
+ table_names = set(inspector.get_table_names())
+ columns = {column["name"] for column in inspector.get_columns("sent_webmentions")}
assert {"alembic_version", "users", "received_webmentions"} <= table_names
+ assert {"attempt_count", "next_retry_at"} <= columns
def test_create_db_refuses_existing_database_without_modifying_it(
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 3d138ac..6ace77f 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import ModuleType
from typing import Any
@@ -1552,6 +1553,56 @@ def test_pending_webmention_is_requeued_on_next_scan(
assert queued == [identifier]
+
+def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ configure_scanner(app, tmp_path)
+ path = write_post(tmp_path, "example", '<a href="https://example.com/a">A</a>')
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+ assert len(queued) == 1
+ identifier = queued.pop()
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.attempt_count = 2
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=4)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == []
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == [identifier]
+ queued.clear()
+
+ # Changing the h-entry creates a new desired revision with a fresh budget.
+ path.write_text(path.read_text().replace('>A</a>', '>A updated</a>'))
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(days=2)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == [identifier]
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.desired_revision == 2
+ assert webmention.attempt_count == 0
+ assert webmention.next_retry_at is None
+
def test_invalid_known_source_is_not_deleted(
app: Flask,
scanner_module: ModuleType,
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index ecfc27a..6b189ad 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -4,7 +4,7 @@
import logging
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from types import ModuleType
from unittest.mock import MagicMock, Mock
@@ -34,6 +34,8 @@ def create_sent_webmention(
endpoint: str | None = None,
response_status: int | None = None,
status_url: str | None = None,
+ attempt_count: int = 0,
+ next_retry_at: datetime | None = None,
) -> uuid.UUID:
now = datetime.now(UTC)
@@ -58,6 +60,8 @@ def create_sent_webmention(
endpoint=endpoint,
response_status=response_status,
status_url=status_url,
+ attempt_count=attempt_count,
+ next_retry_at=next_retry_at,
)
source.sent_webmentions.append(webmention)
@@ -686,6 +690,8 @@ def test_send_webmention_persists_success(
assert webmention.status_url == STATUS_URL
assert webmention.last_attempted_at is not None
assert webmention.last_sent_at is not None
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is None
def test_send_webmention_marks_unsupported_target_processed(
@@ -754,7 +760,7 @@ def test_send_webmention_persists_permanent_failure(
assert webmention.status_url is None
-def test_send_webmention_persists_temporary_failure_and_reraises(
+def test_send_webmention_persists_temporary_failure_with_backoff(
app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
identifier = create_sent_webmention(
@@ -768,11 +774,7 @@ def test_send_webmention_persists_temporary_failure_and_reraises(
mock_sender_requests(sender_module, monkeypatch, result=(503, None))
- with pytest.raises(
- sender_module.TemporarySenderError,
- match="Webmention endpoint returned HTTP 503",
- ):
- sender_module.send_webmention.call_local(identifier)
+ sender_module.send_webmention.call_local(identifier)
with app.app_context():
webmention = db.session.get(SentWebmention, identifier)
@@ -785,6 +787,120 @@ def test_send_webmention_persists_temporary_failure_and_reraises(
assert webmention.endpoint == ENDPOINT_URL
assert webmention.response_status == 503
assert webmention.status_url is None
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is not None
+ assert webmention.last_attempted_at is not None
+ # SQLite drops timezone info when loading timestamp columns.
+ assert (
+ webmention.next_retry_at.replace(tzinfo=None)
+ - webmention.last_attempted_at.replace(tzinfo=None)
+ ).total_seconds() == 3600
+
+
+def test_retry_delay_is_exponential_and_capped(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ with app.app_context():
+ assert [sender_module.retry_delay(n).total_seconds() for n in range(1, 8)] == [
+ 3600,
+ 7200,
+ 14400,
+ 28800,
+ 57600,
+ 86400,
+ 86400,
+ ]
+
+
+def test_temporary_failure_backoff_suppresses_duplicate_tasks_and_caps_attempts(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 3
+ identifier = create_sent_webmention(app)
+ _, _, discover, post = mock_sender_requests(
+ sender_module, monkeypatch, result=(503, None)
+ )
+
+ for attempt in range(1, 4):
+ sender_module.send_webmention.call_local(identifier)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.attempt_count == attempt
+ assert webmention.status == SentWebmentionStatus.FAILED
+ assert webmention.processed_revision == (1 if attempt == 3 else None)
+ if attempt < 3:
+ assert webmention.next_retry_at is not None
+ sender_module.send_webmention.call_local(identifier)
+ assert webmention.attempt_count == attempt
+ webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1)
+ db.session.commit()
+ else:
+ assert webmention.next_retry_at is None
+
+ # A terminal failure stays terminal across scans or stray queued tasks.
+ sender_module.send_webmention.call_local(identifier)
+ assert discover.call_count == 3
+ assert post.call_count == 3
+
+
+def test_sender_does_not_retry_before_due_even_after_restart(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_sent_webmention(
+ app,
+ attempt_count=2,
+ next_retry_at=datetime.now(UTC) + timedelta(days=1),
+ )
+ httpx_client, _, discover, _ = mock_sender_requests(sender_module, monkeypatch)
+ sender_module.send_webmention.call_local(identifier)
+ httpx_client.assert_not_called()
+ discover.assert_not_called()
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.attempt_count == 2
+ assert webmention.processed_revision is None
+
+
+def test_sender_finishes_abandoned_final_attempt_without_new_request(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 2
+ identifier = create_sent_webmention(
+ app,
+ attempt_count=2,
+ next_retry_at=datetime.now(UTC) - timedelta(seconds=1),
+ status=SentWebmentionStatus.FAILED,
+ failure_reason="Network unreachable",
+ )
+ httpx_client, _, _, _ = mock_sender_requests(sender_module, monkeypatch)
+ sender_module.send_webmention.call_local(identifier)
+ httpx_client.assert_not_called()
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.processed_revision == 1
+ assert webmention.failure_reason == "Network unreachable"
+ assert webmention.next_retry_at is None
+
+
+def test_sender_request_error_uses_same_retry_budget(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 1
+ identifier = create_sent_webmention(app)
+ _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch)
+ discover.side_effect = httpx.ConnectError("DNS lookup failed")
+ sender_module.send_webmention.call_local(identifier)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.status == SentWebmentionStatus.FAILED
+ assert webmention.failure_reason == "DNS lookup failed"
+ assert webmention.processed_revision == 1
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is None
def test_send_webmention_ignores_processed_revision(
diff --git a/tests/test_views.py b/tests/test_views.py
index 8ac57b1..fb12b8b 100644
--- a/tests/test_views.py
+++ b/tests/test_views.py
@@ -693,6 +693,13 @@ def test_resend_sent_webmention_reopens_current_revision(
send_webmention = Mock()
monkeypatch.setattr(views_module, "send_webmention", send_webmention)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+ assert webmention is not None
+ webmention.attempt_count = 6
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24)
+ db.session.commit()
+
response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
assert response.status_code == 302
@@ -703,6 +710,8 @@ def test_resend_sent_webmention_reopens_current_revision(
assert webmention is not None
assert webmention.desired_revision == 3
assert webmention.processed_revision is None
+ assert webmention.attempt_count == 0
+ assert webmention.next_retry_at is None
assert webmention.sent_revision == 2
assert webmention.status is None
assert webmention.failure_reason is None