diff options
| author | Dennis Fink | 2026-10-31 19:00:17 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-31 19:00:17 +0100 |
| commit | df3b79e2c89978ec21dc2da7f90801dce33aeab2 (patch) | |
| tree | 146f34d59b2b1957e53856a58777a3453960a2a5 | |
| parent | f7c0691e0da84644ea8b2b83ad5574fe179da59b (diff) | |
| download | webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.tar.gz webmentions-ssg-df3b79e2c89978ec21dc2da7f90801dce33aeab2.zip | |
fix(receiver): prevent endless verification recovery
Cap stale verification requeues so repeatedly lost or crashing tasks
eventually become terminal failures instead of cycling indefinitely.
Reject URLs that httpx cannot safely fetch before enqueueing, and
persist unexpected verification errors as terminal failures.
Diffstat (limited to '')
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 3 | ||||
| -rw-r--r-- | tests/cli/test_database.py | 8 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 37 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 102 | ||||
| -rw-r--r-- | tests/test_views.py | 32 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 30 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py | 31 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 5 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 43 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 8 |
12 files changed, 306 insertions, 5 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 45ae297..577eeef 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -73,6 +73,9 @@ MAX_REDIRECTS = 20 # also used for outgoing requests MAX_SOURCE_BYTES = 1_000_000 # Requeue stuck verifications and lost queued tasks after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 +# Max stale-row requeues before marking the Webmention failed. Manual +# reverification or a new submission after the cooldown resets this counter. +VERIFICATION_MAX_RECOVERIES = 3 # Huey cron expression: every five minutes (in UTC). VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 7095d51..7d325aa 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -85,11 +85,15 @@ def test_create_db_builds_schema_and_records_latest_migration( columns = { column["name"] for column in inspector.get_columns("sent_webmentions") } + received_columns = { + column["name"] for column in inspector.get_columns("received_webmentions") + } received_indexes = { index["name"] for index in inspector.get_indexes("received_webmentions") } assert {"alembic_version", "users", "received_webmentions"} <= table_names assert {"attempt_count", "next_retry_at"} <= columns + assert "recovery_attempts" in received_columns assert "ix_received_webmentions_status_updated_at" in received_indexes @@ -247,6 +251,10 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows( ) ).one() assert result.status == "verified" + recovery_attempts = connection.execute( + sa.text("SELECT recovery_attempts FROM received_webmentions") + ).scalar_one() + assert recovery_attempts == 0 assert result.url_digest == received_url_digest( result.source, result.target ) diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index 2a68509..42e6a2c 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -144,7 +144,7 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: meta={"csrf": False}, ) assert not form.validate() - assert "Invalid input." in form.errors["target"] + assert "Invalid URL." in form.errors["target"] def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( @@ -168,3 +168,38 @@ def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( assert not form.validate() assert "target" in form.errors assert any("bytes" in message for message in form.errors["target"]) + + +@pytest.mark.parametrize("field_name", ["source", "target"]) +@pytest.mark.parametrize( + "invalid_url", + [ + pytest.param("http://ex☃mple.test/", id="invalid-idn"), + pytest.param("https://example.com/a\nb", id="newline"), + pytest.param("https://example.com/a\rb", id="carriage-return"), + pytest.param("https://example.com/\x00b", id="nul"), + pytest.param("https://example.com/a b", id="ascii-space"), + pytest.param("https://example.com/a\u00a0b", id="unicode-whitespace"), + pytest.param("https://example.com/\x7fb", id="del"), + ], +) +def test_endpoint_rejects_invalid_httpx_urls_and_whitespace( + app: Flask, field_name: str, invalid_url: str +) -> None: + data = {"source": VALID_SOURCE, "target": VALID_TARGET} + data[field_name] = invalid_url + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm(formdata=MultiDict(data), meta={"csrf": False}) + assert not form.validate() + assert "Invalid URL." in form.errors[field_name] + + +def test_endpoint_accepts_encoded_path_space(app: Flask) -> None: + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm( + formdata=MultiDict( + {"source": "https://source.example/a%20b", "target": VALID_TARGET} + ), + meta={"csrf": False}, + ) + assert form.validate() diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index c795115..44d1cc2 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -35,6 +35,7 @@ def create_webmention( target: str = TARGET_URL, status: str = "received", failure_reason: str | None = None, + recovery_attempts: int = 0, ) -> uuid.UUID: identifier = uuid.uuid7() @@ -45,6 +46,7 @@ def create_webmention( target=target, status=status, failure_reason=failure_reason, + recovery_attempts=recovery_attempts, ) db.session.add(webmention) @@ -669,6 +671,106 @@ def test_reaper_requeues_and_verifies_integration( check.assert_called_once_with(SOURCE_URL, TARGET_URL) +@patch("webmentions_ssg.tasks.receiver.source_mentions_target") +@pytest.mark.parametrize( + "unexpected_error", + [ + pytest.param(httpx.InvalidURL("Invalid IDNA hostname"), id="invalid-url"), + pytest.param( + RuntimeError("Unexpected HTML parser error"), id="unexpected-error" + ), + ], +) +def test_verify_webmention_records_unexpected_error_as_terminal_failure( + source_mentions_target: Mock, + app: Flask, + receiver: ModuleType, + unexpected_error: Exception, +) -> None: + identifier = create_webmention(app, source="http://ex☃mple.test/") + source_mentions_target.side_effect = unexpected_error + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == ("failed", str(unexpected_error)) + source_mentions_target.assert_called_once() + # Failed rows are not eligible for automatic reaper recovery. + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + + +def test_reaper_caps_stale_verification_recoveries( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["VERIFICATION_MAX_RECOVERIES"] = 3 + identifier = create_webmention(app, status="verifying") + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + # Simulate workers that repeatedly crash before completing the task. + for attempt in range(3): + set_webmention_updated_at( + app, identifier, datetime.now(UTC) - timedelta(hours=1) + ) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.status == "received" + assert row.recovery_attempts == attempt + 1 + + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.status == "failed" + assert row.failure_reason == receiver.RECOVERY_EXHAUSTED_REASON + assert row.recovery_attempts == 3 + assert receiver.recover_stale_webmentions() == 0 + assert enqueue.call_count == 3 + + +def test_reaper_caps_received_rows_lost_before_first_task( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["VERIFICATION_MAX_RECOVERIES"] = 1 + identifier = create_webmention(app, status="received", recovery_attempts=1) + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + assert get_webmention_state(app, identifier) == ( + "failed", + receiver.RECOVERY_EXHAUSTED_REASON, + ) + enqueue.assert_not_called() + + +def test_exhausted_row_cannot_be_revived_by_late_huey_retry( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention( + app, + status="failed", + failure_reason=receiver.RECOVERY_EXHAUSTED_REASON, + recovery_attempts=3, + ) + source_mentions_target = Mock(return_value=True) + monkeypatch.setattr(receiver, "source_mentions_target", source_mentions_target) + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == ( + "failed", + receiver.RECOVERY_EXHAUSTED_REASON, + ) + source_mentions_target.assert_not_called() + + def test_verify_webmention_ignores_unknown_identifier( receiver: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_views.py b/tests/test_views.py index f5e301f..2108b38 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -576,6 +576,11 @@ def test_reverify_received_webmention( identifier = create_received_webmention( app, status="failed", failure_reason="Previous failure" ) + with app.app_context(): + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + row.recovery_attempts = 3 + db.session.commit() mock_admin_action_form(views_module, monkeypatch, valid=True) verify_webmention = Mock() monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) @@ -590,6 +595,7 @@ def test_reverify_received_webmention( assert webmention is not None assert webmention.status == "received" assert webmention.failure_reason is None + assert webmention.recovery_attempts == 0 verify_webmention.assert_called_once_with(identifier) @@ -911,6 +917,7 @@ def test_endpoint_reverifies_after_cooldown( target=TARGET_URL, status="failed", failure_reason="Previous failure", + recovery_attempts=3, updated_at=datetime.now(UTC) - timedelta(minutes=11), ) db.session.add(webmention) @@ -926,6 +933,7 @@ def test_endpoint_reverifies_after_cooldown( assert stored is not None assert stored.status == "received" assert stored.failure_reason is None + assert stored.recovery_attempts == 0 verify.assert_called_once_with(identifier) @@ -1194,3 +1202,27 @@ def test_endpoint_rejects_request_bodies_over_configured_maximum( "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} ) assert response.status_code == 413 + + +def test_endpoint_rejects_bad_url_before_inserting_row( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg import views + + verify_webmention = Mock() + monkeypatch.setattr(views, "verify_webmention", verify_webmention) + + response = client.post( + "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL} + ) + + assert response.status_code == 400 + assert "source" in response.get_json() + verify_webmention.assert_not_called() + with app.app_context(): + assert ( + db.session.scalar( + sa.select(sa.func.count()).select_from(ReceivedWebmention) + ) + == 0 + ) diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 97dd682..e91c651 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -56,6 +56,8 @@ class DefaultConfig: REQUEST_TOTAL_TIMEOUT = 30.0 # Requeue abandoned verification attempts after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 + # Maximum automatic recoveries of a stale received/verifying row. + VERIFICATION_MAX_RECOVERIES = 3 VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" SOURCE_DIRECTORY = None diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index 5006365..589cbd9 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,13 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes +from .validators import ( + AllowedHostname, + HTTPXURL, + NotEqualTo, + PublicIPLiteral, + URLMaxBytes, +) class LoginForm(FlaskForm): @@ -36,6 +42,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", @@ -51,6 +58,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index c1f1c97..384c7a1 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -5,10 +5,13 @@ """Provide validators for URL safety and form field constraints.""" import ipaddress +import unicodedata from urllib.parse import urlsplit +import httpx from flask import current_app from wtforms import Field, ValidationError +from wtforms.validators import StopValidation from wtforms.form import BaseForm from ..url_security import AddressResolutionError, is_public_address, is_public_url @@ -40,6 +43,33 @@ class URLMaxBytes: ) +class HTTPXURL: + """Reject URLs that cannot be fetched by httpx or contain whitespace/controls. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or unicodedata.category(character) == "Cc" + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + class NotEqualTo: """ Validate that a field does not equal another field. diff --git a/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py new file mode 100644 index 0000000..c516636 --- /dev/null +++ b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cap stale received Webmention verification recovery per row. + +Revision ID: e62b1a7c4d90 +Revises: d845aa7fbc19 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "e62b1a7c4d90" +down_revision = "d845aa7fbc19" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.add_column( + sa.Column( + "recovery_attempts", sa.Integer(), nullable=False, server_default="0" + ) + ) + + +def downgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.drop_column("recovery_attempts") diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 0dfbb78..31516d5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -121,6 +121,11 @@ class ReceivedWebmention(Base): return value status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) failure_reason: Mapped[str | None] = mapped_column( Text(), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index f999d1b..acd66a1 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport from ..url_security import AddressResolutionError, UnsafeAddressError +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: # should not start another concurrent verification. if webmention.status not in {"received", "failed"}: return + # A reaper-exhausted row must not be revived by a late Huey retry. + if ( + webmention.status == "failed" + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return previous_status = webmention.status previous_updated_at = webmention.updated_at @@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: except (TemporaryFetchError, httpx.RequestError) as exc: finish("failed", str(exc) or "Source could not be fetched") raise + except Exception as exc: + # InvalidURL is not a RequestError; other unexpected source errors must + # also leave the row in a terminal state, without an endless retry loop. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", webmention_uuid + ) + finish("failed", str(exc) or type(exc).__name__) else: if mentions_target: finish("verified", None) @@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int: updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. :return: Number of webmentions requeued. """ @@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_(("verifying", "received")), ReceivedWebmention.updated_at <= cutoff, @@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int: ).all() recovered = 0 - for identifier, status, updated_at in stale: + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + status="failed" if at_limit else "received", + failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, + updated_at=now, + recovery_attempts=( + recovery_attempts if at_limit else recovery_attempts + 1 + ), ) - .values(status="received", failure_reason=None, updated_at=now) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: + exhausted += 1 + continue verify_webmention(identifier) recovered += 1 + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index fcb5f2d..0836869 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -185,6 +185,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: webmention.status = "received" webmention.failure_reason = None + webmention.recovery_attempts = 0 db.session.commit() @@ -395,7 +396,12 @@ def endpoint() -> ResponseReturnValue: ReceivedWebmention.uuid == webmention.uuid, ReceivedWebmention.updated_at <= cutoff, ) - .values(status="received", failure_reason=None, updated_at=now) + .values( + status="received", + failure_reason=None, + recovery_attempts=0, + updated_at=now, + ) .execution_options(synchronize_session=False) ), ) |
