aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--contrib/webmentions-ssg.example.cfg8
-rw-r--r--tests/cli/test_database.py5
-rw-r--r--tests/tasks/test_scanner.py51
-rw-r--r--tests/tasks/test_sender.py130
-rw-r--r--tests/test_views.py9
-rw-r--r--webmentions_ssg/config.py4
-rw-r--r--webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py38
-rw-r--r--webmentions_ssg/models.py8
-rw-r--r--webmentions_ssg/tasks/scanner.py10
-rw-r--r--webmentions_ssg/tasks/sender.py117
-rw-r--r--webmentions_ssg/views.py2
11 files changed, 358 insertions, 24 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg
index d214312..7dfdd7a 100644
--- a/contrib/webmentions-ssg.example.cfg
+++ b/contrib/webmentions-ssg.example.cfg
@@ -79,6 +79,14 @@ SOURCE_BASE_URL = "https://example.com/"
# Huey cron expression: minute hour day-of-month month day-of-week.
SCANNER_SCHEDULE = "0 * * * *" # every hour
+# Temporary delivery failures use persisted exponential backoff: 1, 2, 4,
+# 8, then 16 hours between attempts with these defaults. After six total
+# attempts for a revision, the failure becomes terminal. A changed source or
+# explicit manual resend starts a fresh retry budget.
+SEND_MAX_ATTEMPTS = 6
+SEND_RETRY_BASE_SECONDS = 3600
+SEND_RETRY_MAX_SECONDS = 86400
+
# Do not send Webmentions to your own site. Glob patterns are supported here.
IGNORED_HOSTNAMES = ("example.com", "*.example.com")
MAX_TARGET_BYTES = 1_000_000
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
index 51b79ac..f25aade 100644
--- a/tests/cli/test_database.py
+++ b/tests/cli/test_database.py
@@ -80,8 +80,11 @@ def test_create_db_builds_schema_and_records_latest_migration(
assert _database_revision(database_app) == _head_revision(database_app)
with database_app.app_context():
- table_names = set(inspect(DATABASE.engine).get_table_names())
+ inspector = inspect(DATABASE.engine)
+ table_names = set(inspector.get_table_names())
+ columns = {column["name"] for column in inspector.get_columns("sent_webmentions")}
assert {"alembic_version", "users", "received_webmentions"} <= table_names
+ assert {"attempt_count", "next_retry_at"} <= columns
def test_create_db_refuses_existing_database_without_modifying_it(
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 3d138ac..6ace77f 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import ModuleType
from typing import Any
@@ -1552,6 +1553,56 @@ def test_pending_webmention_is_requeued_on_next_scan(
assert queued == [identifier]
+
+def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ configure_scanner(app, tmp_path)
+ path = write_post(tmp_path, "example", '<a href="https://example.com/a">A</a>')
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+ assert len(queued) == 1
+ identifier = queued.pop()
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.attempt_count = 2
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=4)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == []
+
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == [identifier]
+ queued.clear()
+
+ # Changing the h-entry creates a new desired revision with a fresh budget.
+ path.write_text(path.read_text().replace('>A</a>', '>A updated</a>'))
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(days=2)
+ db.session.commit()
+
+ run_scan(app, scanner_module)
+ assert queued == [identifier]
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.desired_revision == 2
+ assert webmention.attempt_count == 0
+ assert webmention.next_retry_at is None
+
def test_invalid_known_source_is_not_deleted(
app: Flask,
scanner_module: ModuleType,
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index ecfc27a..6b189ad 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -4,7 +4,7 @@
import logging
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from types import ModuleType
from unittest.mock import MagicMock, Mock
@@ -34,6 +34,8 @@ def create_sent_webmention(
endpoint: str | None = None,
response_status: int | None = None,
status_url: str | None = None,
+ attempt_count: int = 0,
+ next_retry_at: datetime | None = None,
) -> uuid.UUID:
now = datetime.now(UTC)
@@ -58,6 +60,8 @@ def create_sent_webmention(
endpoint=endpoint,
response_status=response_status,
status_url=status_url,
+ attempt_count=attempt_count,
+ next_retry_at=next_retry_at,
)
source.sent_webmentions.append(webmention)
@@ -686,6 +690,8 @@ def test_send_webmention_persists_success(
assert webmention.status_url == STATUS_URL
assert webmention.last_attempted_at is not None
assert webmention.last_sent_at is not None
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is None
def test_send_webmention_marks_unsupported_target_processed(
@@ -754,7 +760,7 @@ def test_send_webmention_persists_permanent_failure(
assert webmention.status_url is None
-def test_send_webmention_persists_temporary_failure_and_reraises(
+def test_send_webmention_persists_temporary_failure_with_backoff(
app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
identifier = create_sent_webmention(
@@ -768,11 +774,7 @@ def test_send_webmention_persists_temporary_failure_and_reraises(
mock_sender_requests(sender_module, monkeypatch, result=(503, None))
- with pytest.raises(
- sender_module.TemporarySenderError,
- match="Webmention endpoint returned HTTP 503",
- ):
- sender_module.send_webmention.call_local(identifier)
+ sender_module.send_webmention.call_local(identifier)
with app.app_context():
webmention = db.session.get(SentWebmention, identifier)
@@ -785,6 +787,120 @@ def test_send_webmention_persists_temporary_failure_and_reraises(
assert webmention.endpoint == ENDPOINT_URL
assert webmention.response_status == 503
assert webmention.status_url is None
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is not None
+ assert webmention.last_attempted_at is not None
+ # SQLite drops timezone info when loading timestamp columns.
+ assert (
+ webmention.next_retry_at.replace(tzinfo=None)
+ - webmention.last_attempted_at.replace(tzinfo=None)
+ ).total_seconds() == 3600
+
+
+def test_retry_delay_is_exponential_and_capped(
+ app: Flask, sender_module: ModuleType
+) -> None:
+ with app.app_context():
+ assert [sender_module.retry_delay(n).total_seconds() for n in range(1, 8)] == [
+ 3600,
+ 7200,
+ 14400,
+ 28800,
+ 57600,
+ 86400,
+ 86400,
+ ]
+
+
+def test_temporary_failure_backoff_suppresses_duplicate_tasks_and_caps_attempts(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 3
+ identifier = create_sent_webmention(app)
+ _, _, discover, post = mock_sender_requests(
+ sender_module, monkeypatch, result=(503, None)
+ )
+
+ for attempt in range(1, 4):
+ sender_module.send_webmention.call_local(identifier)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.attempt_count == attempt
+ assert webmention.status == SentWebmentionStatus.FAILED
+ assert webmention.processed_revision == (1 if attempt == 3 else None)
+ if attempt < 3:
+ assert webmention.next_retry_at is not None
+ sender_module.send_webmention.call_local(identifier)
+ assert webmention.attempt_count == attempt
+ webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1)
+ db.session.commit()
+ else:
+ assert webmention.next_retry_at is None
+
+ # A terminal failure stays terminal across scans or stray queued tasks.
+ sender_module.send_webmention.call_local(identifier)
+ assert discover.call_count == 3
+ assert post.call_count == 3
+
+
+def test_sender_does_not_retry_before_due_even_after_restart(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_sent_webmention(
+ app,
+ attempt_count=2,
+ next_retry_at=datetime.now(UTC) + timedelta(days=1),
+ )
+ httpx_client, _, discover, _ = mock_sender_requests(sender_module, monkeypatch)
+ sender_module.send_webmention.call_local(identifier)
+ httpx_client.assert_not_called()
+ discover.assert_not_called()
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.attempt_count == 2
+ assert webmention.processed_revision is None
+
+
+def test_sender_finishes_abandoned_final_attempt_without_new_request(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 2
+ identifier = create_sent_webmention(
+ app,
+ attempt_count=2,
+ next_retry_at=datetime.now(UTC) - timedelta(seconds=1),
+ status=SentWebmentionStatus.FAILED,
+ failure_reason="Network unreachable",
+ )
+ httpx_client, _, _, _ = mock_sender_requests(sender_module, monkeypatch)
+ sender_module.send_webmention.call_local(identifier)
+ httpx_client.assert_not_called()
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.processed_revision == 1
+ assert webmention.failure_reason == "Network unreachable"
+ assert webmention.next_retry_at is None
+
+
+def test_sender_request_error_uses_same_retry_budget(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["SEND_MAX_ATTEMPTS"] = 1
+ identifier = create_sent_webmention(app)
+ _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch)
+ discover.side_effect = httpx.ConnectError("DNS lookup failed")
+ sender_module.send_webmention.call_local(identifier)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.status == SentWebmentionStatus.FAILED
+ assert webmention.failure_reason == "DNS lookup failed"
+ assert webmention.processed_revision == 1
+ assert webmention.attempt_count == 1
+ assert webmention.next_retry_at is None
def test_send_webmention_ignores_processed_revision(
diff --git a/tests/test_views.py b/tests/test_views.py
index 8ac57b1..fb12b8b 100644
--- a/tests/test_views.py
+++ b/tests/test_views.py
@@ -693,6 +693,13 @@ def test_resend_sent_webmention_reopens_current_revision(
send_webmention = Mock()
monkeypatch.setattr(views_module, "send_webmention", send_webmention)
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, webmention_identifier)
+ assert webmention is not None
+ webmention.attempt_count = 6
+ webmention.next_retry_at = datetime.now(UTC) + timedelta(hours=24)
+ db.session.commit()
+
response = client.post(f"/sent/{source_identifier}/{webmention_identifier}/resend")
assert response.status_code == 302
@@ -703,6 +710,8 @@ def test_resend_sent_webmention_reopens_current_revision(
assert webmention is not None
assert webmention.desired_revision == 3
assert webmention.processed_revision is None
+ assert webmention.attempt_count == 0
+ assert webmention.next_retry_at is None
assert webmention.sent_revision == 2
assert webmention.status is None
assert webmention.failure_reason is None
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 8cad51f..3aec00d 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -49,6 +49,10 @@ class DefaultConfig:
SOURCE_DIRECTORY = None
SOURCE_BASE_URL = None
SCANNER_SCHEDULE = "0 * * * *"
+ # Total send attempts per source revision (not additional retries).
+ SEND_MAX_ATTEMPTS = 6
+ SEND_RETRY_BASE_SECONDS = 3600
+ SEND_RETRY_MAX_SECONDS = 86400
NOTIFICATION_EMAIL = None
NOTIFICATION_SCHEDULE = "0 8 * * *"
diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
new file mode 100644
index 0000000..c4eac52
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
@@ -0,0 +1,38 @@
+"""Add Webmention retry backoff
+
+Revision ID: 85f2b226ec99
+Revises: 408fc2eca9ce
+Create Date: 2026-10-30 13:08:17.340631
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "85f2b226ec99"
+down_revision = "408fc2eca9ce"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("sent_webmentions", schema=None) as batch_op:
+ batch_op.add_column(
+ sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0")
+ )
+ batch_op.add_column(
+ sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True)
+ )
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("sent_webmentions", schema=None) as batch_op:
+ batch_op.drop_column("next_retry_at")
+ batch_op.drop_column("attempt_count")
+
+ # ### end Alembic commands ###
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 11740a8..4f688e5 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -245,6 +245,14 @@ class SentWebmention(Base):
)
status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None)
+ # Retry state belongs to the desired revision, not to the target forever.
+ attempt_count: Mapped[int] = mapped_column(
+ Integer(), nullable=False, default=0
+ )
+ next_retry_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True, default=None
+ )
+
last_attempted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, default=None
)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 7e163f7..03f4b8d 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -385,6 +385,8 @@ def update_source(
for webmention in existing.values():
webmention.active = webmention.target in scanned.targets
webmention.desired_revision = source.revision
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -522,6 +524,8 @@ def scan_sources() -> None:
for webmention in source.sent_webmentions:
webmention.active = False
webmention.desired_revision = source.revision
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -536,7 +540,11 @@ def scan_sources() -> None:
sa.or_(
SentWebmention.processed_revision.is_(None),
SentWebmention.processed_revision < SentWebmention.desired_revision,
- )
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= scan_time,
+ ),
)
.order_by(SentWebmention.uuid)
):
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 8834934..4aa59ee 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -6,10 +6,11 @@
import re
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from urllib.parse import urljoin
import httpx
+import sqlalchemy as sa
from bs4 import BeautifulSoup
from flask import current_app
@@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
)
-@huey.task(retries=2, retry_delay=50)
+def retry_delay(attempt: int) -> timedelta:
+ """Calculate the capped exponential delay after a numbered send attempt."""
+ base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"]))
+ maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"]))
+ return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum))
+
+
+@huey.task()
def send_webmention(webmention_uuid: uuid.UUID) -> None:
"""
Discover a receiver endpoint and send a Webmention.
The result of the attempt is stored on the corresponding sent Webmention.
- Temporary failures are re-raised so Huey can retry them, while permanent
- failures mark the current revision as processed.
+ Temporary failures are recorded with an exponential retry deadline. The
+ scanner requeues them when due; Huey does not retry them independently.
+ The retry budget is per desired source revision.
:param webmention_uuid: Identifier of the sent Webmention to process.
- :raises TemporarySenderError: If endpoint discovery or delivery fails for a
- potentially temporary reason.
- :raises httpx.RequestError: If an HTTP request fails.
"""
webmention = db.session.get(SentWebmention, webmention_uuid)
@@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
revision = webmention.desired_revision
source = webmention.source.url
target = webmention.target
+ now = datetime.now(UTC)
+ deadline = webmention.next_retry_at
+
+ # SQLite returns timezone-naive datetimes even for timezone-aware columns.
+ if deadline is not None:
+ aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
+ if aware_deadline > now:
+ return
+
+ max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
+ if webmention.attempt_count >= max_attempts:
+ # Recover records whose worker died during the final attempt, without
+ # touching a newer revision that may have been queued in the meantime.
+ db.session.execute(
+ sa.update(SentWebmention)
+ .where(
+ SentWebmention.uuid == webmention_uuid,
+ SentWebmention.desired_revision == revision,
+ SentWebmention.attempt_count >= max_attempts,
+ sa.or_(
+ SentWebmention.processed_revision.is_(None),
+ SentWebmention.processed_revision < revision,
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= now,
+ ),
+ )
+ .values(
+ processed_revision=revision,
+ next_retry_at=None,
+ status=SentWebmentionStatus.FAILED,
+ failure_reason=(
+ webmention.failure_reason
+ or f"Retry limit reached ({max_attempts} attempts)"
+ ),
+ )
+ .execution_options(synchronize_session=False)
+ )
+ db.session.commit()
+ return
- webmention.last_attempted_at = datetime.now(UTC)
- webmention.endpoint = None
- webmention.response_status = None
- webmention.status_url = None
+ attempt = webmention.attempt_count + 1
+ next_retry_at = now + retry_delay(attempt)
+ # Claim the attempt atomically: duplicate queued tasks must not send the
+ # same revision concurrently or bypass its retry deadline.
+ claim = db.session.execute(
+ sa.update(SentWebmention)
+ .where(
+ SentWebmention.uuid == webmention_uuid,
+ SentWebmention.desired_revision == revision,
+ SentWebmention.attempt_count == attempt - 1,
+ sa.or_(
+ SentWebmention.processed_revision.is_(None),
+ SentWebmention.processed_revision < revision,
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= now,
+ ),
+ )
+ .values(
+ attempt_count=attempt,
+ last_attempted_at=now,
+ next_retry_at=next_retry_at,
+ endpoint=None,
+ response_status=None,
+ status_url=None,
+ )
+ .execution_options(synchronize_session=False)
+ )
db.session.commit()
+ if claim.rowcount != 1:
+ return
+
+ db.session.refresh(webmention)
+
endpoint: str | None = None
response_status: int | None = None
@@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.status = SentWebmentionStatus.UNSUPPORTED
webmention.failure_reason = "No Webmention endpoint discovered"
webmention.endpoint = None
@@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
webmention.status = SentWebmentionStatus.FAILED
webmention.failure_reason = str(exc) or "Webmention request failed"
+ if attempt >= max_attempts:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ webmention.failure_reason,
+ )
webmention.endpoint = endpoint
webmention.response_status = response_status
db.session.commit()
- # Leave processed_revision unchanged. Huey may retry the
- # attempt, and the periodic scanner will also rediscover
- # outstanding work.
- raise
+ # No Huey retries: only the scanner may requeue after next_retry_at.
+ return
except PermanentSenderError as exc:
if not attempt_is_current(webmention, revision):
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.status = SentWebmentionStatus.FAILED
webmention.failure_reason = str(exc)
webmention.endpoint = endpoint
@@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.sent_revision = revision
webmention.status = SentWebmentionStatus.SENT
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
index 2ab02fa..1dcdd2e 100644
--- a/webmentions_ssg/views.py
+++ b/webmentions_ssg/views.py
@@ -296,6 +296,8 @@ def resend_sent_webmention(
# revision. send_webmention() otherwise exits when this revision has already
# been processed.
webmention.processed_revision = None
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
webmention.status = None
webmention.failure_reason = None
webmention.endpoint = None