diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/config.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py | 38 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 8 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 117 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 2 |
6 files changed, 163 insertions, 16 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 8cad51f..3aec00d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -49,6 +49,10 @@ class DefaultConfig: SOURCE_DIRECTORY = None SOURCE_BASE_URL = None SCANNER_SCHEDULE = "0 * * * *" + # Total send attempts per source revision (not additional retries). + SEND_MAX_ATTEMPTS = 6 + SEND_RETRY_BASE_SECONDS = 3600 + SEND_RETRY_MAX_SECONDS = 86400 NOTIFICATION_EMAIL = None NOTIFICATION_SCHEDULE = "0 8 * * *" diff --git a/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py new file mode 100644 index 0000000..c4eac52 --- /dev/null +++ b/webmentions_ssg/migrations/versions/85f2b226ec99_add_webmention_retry_backoff.py @@ -0,0 +1,38 @@ +"""Add Webmention retry backoff + +Revision ID: 85f2b226ec99 +Revises: 408fc2eca9ce +Create Date: 2026-10-30 13:08:17.340631 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "85f2b226ec99" +down_revision = "408fc2eca9ce" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.add_column( + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0") + ) + batch_op.add_column( + sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True) + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sent_webmentions", schema=None) as batch_op: + batch_op.drop_column("next_retry_at") + batch_op.drop_column("attempt_count") + + # ### end Alembic commands ### diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 11740a8..4f688e5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -245,6 +245,14 @@ class SentWebmention(Base): ) status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + # Retry state belongs to the desired revision, not to the target forever. + attempt_count: Mapped[int] = mapped_column( + Integer(), nullable=False, default=0 + ) + next_retry_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_attempted_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 7e163f7..03f4b8d 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -385,6 +385,8 @@ def update_source( for webmention in existing.values(): webmention.active = webmention.target in scanned.targets webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if not webmention.active and webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -522,6 +524,8 @@ def scan_sources() -> None: for webmention in source.sent_webmentions: webmention.active = False webmention.desired_revision = source.revision + webmention.attempt_count = 0 + webmention.next_retry_at = None if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -536,7 +540,11 @@ def scan_sources() -> None: sa.or_( SentWebmention.processed_revision.is_(None), SentWebmention.processed_revision < SentWebmention.desired_revision, - ) + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= scan_time, + ), ) .order_by(SentWebmention.uuid) ): diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 8834934..4aa59ee 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -6,10 +6,11 @@ import re import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from urllib.parse import urljoin import httpx +import sqlalchemy as sa from bs4 import BeautifulSoup from flask import current_app @@ -288,19 +289,24 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: ) -@huey.task(retries=2, retry_delay=50) +def retry_delay(attempt: int) -> timedelta: + """Calculate the capped exponential delay after a numbered send attempt.""" + base = max(1, int(current_app.config["SEND_RETRY_BASE_SECONDS"])) + maximum = max(base, int(current_app.config["SEND_RETRY_MAX_SECONDS"])) + return timedelta(seconds=min(base * 2 ** (attempt - 1), maximum)) + + +@huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: """ Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. - Temporary failures are re-raised so Huey can retry them, while permanent - failures mark the current revision as processed. + Temporary failures are recorded with an exponential retry deadline. The + scanner requeues them when due; Huey does not retry them independently. + The retry budget is per desired source revision. :param webmention_uuid: Identifier of the sent Webmention to process. - :raises TemporarySenderError: If endpoint discovery or delivery fails for a - potentially temporary reason. - :raises httpx.RequestError: If an HTTP request fails. """ webmention = db.session.get(SentWebmention, webmention_uuid) @@ -319,14 +325,85 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: revision = webmention.desired_revision source = webmention.source.url target = webmention.target + now = datetime.now(UTC) + deadline = webmention.next_retry_at + + # SQLite returns timezone-naive datetimes even for timezone-aware columns. + if deadline is not None: + aware_deadline = deadline if deadline.tzinfo else deadline.replace(tzinfo=UTC) + if aware_deadline > now: + return + + max_attempts = max(1, int(current_app.config["SEND_MAX_ATTEMPTS"])) + if webmention.attempt_count >= max_attempts: + # Recover records whose worker died during the final attempt, without + # touching a newer revision that may have been queued in the meantime. + db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count >= max_attempts, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + processed_revision=revision, + next_retry_at=None, + status=SentWebmentionStatus.FAILED, + failure_reason=( + webmention.failure_reason + or f"Retry limit reached ({max_attempts} attempts)" + ), + ) + .execution_options(synchronize_session=False) + ) + db.session.commit() + return - webmention.last_attempted_at = datetime.now(UTC) - webmention.endpoint = None - webmention.response_status = None - webmention.status_url = None + attempt = webmention.attempt_count + 1 + next_retry_at = now + retry_delay(attempt) + # Claim the attempt atomically: duplicate queued tasks must not send the + # same revision concurrently or bypass its retry deadline. + claim = db.session.execute( + sa.update(SentWebmention) + .where( + SentWebmention.uuid == webmention_uuid, + SentWebmention.desired_revision == revision, + SentWebmention.attempt_count == attempt - 1, + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < revision, + ), + sa.or_( + SentWebmention.next_retry_at.is_(None), + SentWebmention.next_retry_at <= now, + ), + ) + .values( + attempt_count=attempt, + last_attempted_at=now, + next_retry_at=next_retry_at, + endpoint=None, + response_status=None, + status_url=None, + ) + .execution_options(synchronize_session=False) + ) db.session.commit() + if claim.rowcount != 1: + return + + db.session.refresh(webmention) + endpoint: str | None = None response_status: int | None = None @@ -346,6 +423,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.UNSUPPORTED webmention.failure_reason = "No Webmention endpoint discovered" webmention.endpoint = None @@ -372,21 +450,29 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) or "Webmention request failed" + if attempt >= max_attempts: + webmention.processed_revision = revision + webmention.next_retry_at = None + current_app.logger.warning( + "Webmention %s exhausted %d send attempts: %s", + webmention_uuid, + attempt, + webmention.failure_reason, + ) webmention.endpoint = endpoint webmention.response_status = response_status db.session.commit() - # Leave processed_revision unchanged. Huey may retry the - # attempt, and the periodic scanner will also rediscover - # outstanding work. - raise + # No Huey retries: only the scanner may requeue after next_retry_at. + return except PermanentSenderError as exc: if not attempt_is_current(webmention, revision): return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.status = SentWebmentionStatus.FAILED webmention.failure_reason = str(exc) webmention.endpoint = endpoint @@ -399,6 +485,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: return webmention.processed_revision = revision + webmention.next_retry_at = None webmention.sent_revision = revision webmention.status = SentWebmentionStatus.SENT diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index 2ab02fa..1dcdd2e 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -296,6 +296,8 @@ def resend_sent_webmention( # revision. send_webmention() otherwise exits when this revision has already # been processed. webmention.processed_revision = None + webmention.attempt_count = 0 + webmention.next_retry_at = None webmention.status = None webmention.failure_reason = None webmention.endpoint = None |
