diff options
| author | Dennis Fink | 2026-10-31 19:00:17 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-31 19:00:17 +0100 |
| commit | df3b79e2c89978ec21dc2da7f90801dce33aeab2 (patch) | |
| tree | 146f34d59b2b1957e53856a58777a3453960a2a5 /webmentions_ssg | |
| parent | f7c0691e0da84644ea8b2b83ad5574fe179da59b (diff) | |
| download | webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.tar.gz webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.zip | |
fix(receiver): prevent endless verification recovery
Cap stale verification requeues so repeatedly lost or crashing tasks
eventually become terminal failures instead of cycling indefinitely.
Reject URLs that httpx cannot safely fetch before enqueueing, and
persist unexpected verification errors as terminal failures.
Diffstat (limited to 'webmentions_ssg')
| -rw-r--r-- | webmentions_ssg/config.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 30 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py | 31 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 5 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 43 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 8 |
7 files changed, 125 insertions, 4 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 97dd682..e91c651 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -56,6 +56,8 @@ class DefaultConfig: REQUEST_TOTAL_TIMEOUT = 30.0 # Requeue abandoned verification attempts after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 + # Maximum automatic recoveries of a stale received/verifying row. + VERIFICATION_MAX_RECOVERIES = 3 VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" SOURCE_DIRECTORY = None diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index 5006365..589cbd9 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,13 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes +from .validators import ( + AllowedHostname, + HTTPXURL, + NotEqualTo, + PublicIPLiteral, + URLMaxBytes, +) class LoginForm(FlaskForm): @@ -36,6 +42,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", @@ -51,6 +58,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index c1f1c97..384c7a1 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -5,10 +5,13 @@ """Provide validators for URL safety and form field constraints.""" import ipaddress +import unicodedata from urllib.parse import urlsplit +import httpx from flask import current_app from wtforms import Field, ValidationError +from wtforms.validators import StopValidation from wtforms.form import BaseForm from ..url_security import AddressResolutionError, is_public_address, is_public_url @@ -40,6 +43,33 @@ class URLMaxBytes: ) +class HTTPXURL: + """Reject URLs that cannot be fetched by httpx or contain whitespace/controls. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or unicodedata.category(character) == "Cc" + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + class NotEqualTo: """ Validate that a field does not equal another field. diff --git a/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py new file mode 100644 index 0000000..c516636 --- /dev/null +++ b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cap stale received Webmention verification recovery per row. + +Revision ID: e62b1a7c4d90 +Revises: d845aa7fbc19 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "e62b1a7c4d90" +down_revision = "d845aa7fbc19" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.add_column( + sa.Column( + "recovery_attempts", sa.Integer(), nullable=False, server_default="0" + ) + ) + + +def downgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.drop_column("recovery_attempts") diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 0dfbb78..31516d5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -121,6 +121,11 @@ class ReceivedWebmention(Base): return value status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) failure_reason: Mapped[str | None] = mapped_column( Text(), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index f999d1b..acd66a1 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport from ..url_security import AddressResolutionError, UnsafeAddressError +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: # should not start another concurrent verification. if webmention.status not in {"received", "failed"}: return + # A reaper-exhausted row must not be revived by a late Huey retry. + if ( + webmention.status == "failed" + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return previous_status = webmention.status previous_updated_at = webmention.updated_at @@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: except (TemporaryFetchError, httpx.RequestError) as exc: finish("failed", str(exc) or "Source could not be fetched") raise + except Exception as exc: + # InvalidURL is not a RequestError; other unexpected source errors must + # also leave the row in a terminal state, without an endless retry loop. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", webmention_uuid + ) + finish("failed", str(exc) or type(exc).__name__) else: if mentions_target: finish("verified", None) @@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int: updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. :return: Number of webmentions requeued. """ @@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_(("verifying", "received")), ReceivedWebmention.updated_at <= cutoff, @@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int: ).all() recovered = 0 - for identifier, status, updated_at in stale: + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + status="failed" if at_limit else "received", + failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, + updated_at=now, + recovery_attempts=( + recovery_attempts if at_limit else recovery_attempts + 1 + ), ) - .values(status="received", failure_reason=None, updated_at=now) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: + exhausted += 1 + continue verify_webmention(identifier) recovered += 1 + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index fcb5f2d..0836869 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -185,6 +185,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: webmention.status = "received" webmention.failure_reason = None + webmention.recovery_attempts = 0 db.session.commit() @@ -395,7 +396,12 @@ def endpoint() -> ResponseReturnValue: ReceivedWebmention.uuid == webmention.uuid, ReceivedWebmention.updated_at <= cutoff, ) - .values(status="received", failure_reason=None, updated_at=now) + .values( + status="received", + failure_reason=None, + recovery_attempts=0, + updated_at=now, + ) .execution_options(synchronize_session=False) ), ) |
