aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
authorDennis Fink2026-10-30 13:24:53 +0100
committerDennis Fink2026-10-30 13:24:53 +0100
commitf6d96df2c3aecb51ca97426efddb101dce6350d7 (patch)
tree2968f65a627fbfd15051277d07eaa321db0e3cc4 /webmentions_ssg
parent2b59a3082000842d980f9e19931949dbe5011229 (diff)
downloadwebmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.tar.gz
webmentions-ssg-f6d96df2c3aecb51ca97426efddb101dce6350d7.zip
feat(sender): add persistent retry backoff
Persist retry state per source revision so temporary delivery failures survive worker restarts and duplicate queued tasks cannot bypass the retry schedule. Use capped exponential backoff, stop after a configurable attempt limit, and reset the retry budget when a source changes or an admin resends manually.
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/config.py4
-rw-r--r--webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py38
-rw-r--r--webmentions_ssg/models.py8
-rw-r--r--webmentions_ssg/tasks/scanner.py10
-rw-r--r--webmentions_ssg/tasks/sender.py117
-rw-r--r--webmentions_ssg/views.py2
6 files changed, 163 insertions, 16 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 8cad51f..3aec00d 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -49,6 +49,10 @@ class DefaultConfig:
SOURCE_DIRECTORY = None
SOURCE_BASE_URL = None
SCANNER_SCHEDULE = "0 * * * *"
+ # Total send attempts per source revision (not additional retries).
+ SEND_MAX_ATTEMPTS = 6
+ SEND_RETRY_BASE_SECONDS = 3600
+ SEND_RETRY_MAX_SECONDS = 86400
NOTIFICATION_EMAIL = None
NOTIFICATION_SCHEDULE = "0 8 * * *"
diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
new file mode 100644
index 0000000..c4eac52
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py
@@ -0,0 +1,38 @@
+"""Add Webmention retry backoff
+
+Revision ID: 85f2b226ec99
+Revises: 408fc2eca9ce
+Create Date: 2026-10-30 13:08:17.340631
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "85f2b226ec99"
+down_revision = "408fc2eca9ce"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("sent_webmentions", schema=None) as batch_op:
+ batch_op.add_column(
+ sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0")
+ )
+ batch_op.add_column(
+ sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True)
+ )
+
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("sent_webmentions", schema=None) as batch_op:
+ batch_op.drop_column("next_retry_at")
+ batch_op.drop_column("attempt_count")
+
+ # ### end Alembic commands ###
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 11740a8..4f688e5 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -245,6 +245,14 @@ class SentWebmention(Base):
)
status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None)
+ # Retry state belongs to the desired revision, not to the target forever.
+ attempt_count: Mapped[int] = mapped_column(
+ Integer(), nullable=False, default=0
+ )
+ next_retry_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True, default=None
+ )
+
last_attempted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, default=None
)
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 7e163f7..03f4b8d 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -385,6 +385,8 @@ def update_source(
for webmention in existing.values():
webmention.active = webmention.target in scanned.targets
webmention.desired_revision = source.revision
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
if not webmention.active and webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -522,6 +524,8 @@ def scan_sources() -> None:
for webmention in source.sent_webmentions:
webmention.active = False
webmention.desired_revision = source.revision
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
if webmention.sent_revision is None:
webmention.processed_revision = source.revision
@@ -536,7 +540,11 @@ def scan_sources() -> None:
sa.or_(
SentWebmention.processed_revision.is_(None),
SentWebmention.processed_revision < SentWebmention.desired_revision,
- )
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= scan_time,
+ ),
)
.order_by(SentWebmention.uuid)
):
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 8834934..4aa59ee 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -6,10 +6,11 @@
import re
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from urllib.parse import urljoin
import httpx
+import sqlalchemy as sa
from bs4 import BeautifulSoup
from flask import current_app
@@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool:
)
-@huey.task(retries=2, retry_delay=50)
+def retry_delay(attempt: int) -> timedelta:
+ """Calculate the capped exponential delay after a numbered send attempt."""
+ base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"]))
+ maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"]))
+ return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum))
+
+
+@huey.task()
def send_webmention(webmention_uuid: uuid.UUID) -> None:
"""
Discover a receiver endpoint and send a Webmention.
The result of the attempt is stored on the corresponding sent Webmention.
- Temporary failures are re-raised so Huey can retry them, while permanent
- failures mark the current revision as processed.
+ Temporary failures are recorded with an exponential retry deadline. The
+ scanner requeues them when due; Huey does not retry them independently.
+ The retry budget is per desired source revision.
:param webmention_uuid: Identifier of the sent Webmention to process.
- :raises TemporarySenderError: If endpoint discovery or delivery fails for a
- potentially temporary reason.
- :raises httpx.RequestError: If an HTTP request fails.
"""
webmention = db.session.get(SentWebmention, webmention_uuid)
@@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
revision = webmention.desired_revision
source = webmention.source.url
target = webmention.target
+ now = datetime.now(UTC)
+ deadline = webmention.next_retry_at
+
+ # SQLite returns timezone-naive datetimes even for timezone-aware columns.
+ if deadline is not None:
+ aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC)
+ if aware_deadline > now:
+ return
+
+ max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"]))
+ if webmention.attempt_count >= max_attempts:
+ # Recover records whose worker died during the final attempt, without
+ # touching a newer revision that may have been queued in the meantime.
+ db.session.execute(
+ sa.update(SentWebmention)
+ .where(
+ SentWebmention.uuid == webmention_uuid,
+ SentWebmention.desired_revision == revision,
+ SentWebmention.attempt_count >= max_attempts,
+ sa.or_(
+ SentWebmention.processed_revision.is_(None),
+ SentWebmention.processed_revision < revision,
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= now,
+ ),
+ )
+ .values(
+ processed_revision=revision,
+ next_retry_at=None,
+ status=SentWebmentionStatus.FAILED,
+ failure_reason=(
+ webmention.failure_reason
+ or f"Retry limit reached ({max_attempts} attempts)"
+ ),
+ )
+ .execution_options(synchronize_session=False)
+ )
+ db.session.commit()
+ return
- webmention.last_attempted_at = datetime.now(UTC)
- webmention.endpoint = None
- webmention.response_status = None
- webmention.status_url = None
+ attempt = webmention.attempt_count + 1
+ next_retry_at = now + retry_delay(attempt)
+ # Claim the attempt atomically: duplicate queued tasks must not send the
+ # same revision concurrently or bypass its retry deadline.
+ claim = db.session.execute(
+ sa.update(SentWebmention)
+ .where(
+ SentWebmention.uuid == webmention_uuid,
+ SentWebmention.desired_revision == revision,
+ SentWebmention.attempt_count == attempt - 1,
+ sa.or_(
+ SentWebmention.processed_revision.is_(None),
+ SentWebmention.processed_revision < revision,
+ ),
+ sa.or_(
+ SentWebmention.next_retry_at.is_(None),
+ SentWebmention.next_retry_at <= now,
+ ),
+ )
+ .values(
+ attempt_count=attempt,
+ last_attempted_at=now,
+ next_retry_at=next_retry_at,
+ endpoint=None,
+ response_status=None,
+ status_url=None,
+ )
+ .execution_options(synchronize_session=False)
+ )
db.session.commit()
+ if claim.rowcount != 1:
+ return
+
+ db.session.refresh(webmention)
+
endpoint: str | None = None
response_status: int | None = None
@@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.status = SentWebmentionStatus.UNSUPPORTED
webmention.failure_reason = "No Webmention endpoint discovered"
webmention.endpoint = None
@@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
webmention.status = SentWebmentionStatus.FAILED
webmention.failure_reason = str(exc) or "Webmention request failed"
+ if attempt >= max_attempts:
+ webmention.processed_revision = revision
+ webmention.next_retry_at = None
+ current_app.logger.warning(
+ "Webmention %s exhausted %d send attempts: %s",
+ webmention_uuid,
+ attempt,
+ webmention.failure_reason,
+ )
webmention.endpoint = endpoint
webmention.response_status = response_status
db.session.commit()
- # Leave processed_revision unchanged. Huey may retry the
- # attempt, and the periodic scanner will also rediscover
- # outstanding work.
- raise
+ # No Huey retries: only the scanner may requeue after next_retry_at.
+ return
except PermanentSenderError as exc:
if not attempt_is_current(webmention, revision):
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.status = SentWebmentionStatus.FAILED
webmention.failure_reason = str(exc)
webmention.endpoint = endpoint
@@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None:
return
webmention.processed_revision = revision
+ webmention.next_retry_at = None
webmention.sent_revision = revision
webmention.status = SentWebmentionStatus.SENT
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
index 2ab02fa..1dcdd2e 100644
--- a/webmentions_ssg/views.py
+++ b/webmentions_ssg/views.py
@@ -296,6 +296,8 @@ def resend_sent_webmention(
# revision. send_webmention() otherwise exits when this revision has already
# been processed.
webmention.processed_revision = None
+ webmention.attempt_count = 0
+ webmention.next_retry_at = None
webmention.status = None
webmention.failure_reason = None
webmention.endpoint = None