diff options
Diffstat (limited to '')
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 3 | ||||
| -rw-r--r-- | tests/cli/test_database.py | 8 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 37 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 102 | ||||
| -rw-r--r-- | tests/test_views.py | 32 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 10 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 30 | ||||
| -rw-r--r-- | webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py | 31 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 5 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 43 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 8 |
12 files changed, 306 insertions, 5 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 45ae297..577eeef 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -73,6 +73,9 @@ MAX_REDIRECTS = 20 # also used for outgoing requests MAX_SOURCE_BYTES = 1_000_000 # Requeue stuck verifications and lost queued tasks after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 +# Max stale-row requeues before marking the Webmention failed. Manual +# reverification or a new submission after the cooldown resets this counter. +VERIFICATION_MAX_RECOVERIES = 3 # Huey cron expression: every five minutes (in UTC). VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py index 7095d51..7d325aa 100644 --- a/tests/cli/test_database.py +++ b/tests/cli/test_database.py @@ -85,11 +85,15 @@ def test_create_db_builds_schema_and_records_latest_migration( columns = { column["name"] for column in inspector.get_columns("sent_webmentions") } + received_columns = { + column["name"] for column in inspector.get_columns("received_webmentions") + } received_indexes = { index["name"] for index in inspector.get_indexes("received_webmentions") } assert {"alembic_version", "users", "received_webmentions"} <= table_names assert {"attempt_count", "next_retry_at"} <= columns + assert "recovery_attempts" in received_columns assert "ix_received_webmentions_status_updated_at" in received_indexes @@ -247,6 +251,10 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows( ) ).one() assert result.status == "verified" + recovery_attempts = connection.execute( + sa.text("SELECT recovery_attempts FROM received_webmentions") + ).scalar_one() + assert recovery_attempts == 0 assert result.url_digest == received_url_digest( result.source, result.target ) diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index 2a68509..42e6a2c 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -144,7 +144,7 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: meta={"csrf": False}, ) assert not form.validate() - assert "Invalid input." in form.errors["target"] + assert "Invalid URL." in form.errors["target"] def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( @@ -168,3 +168,38 @@ def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields( assert not form.validate() assert "target" in form.errors assert any("bytes" in message for message in form.errors["target"]) + + +@pytest.mark.parametrize("field_name", ["source", "target"]) +@pytest.mark.parametrize( + "invalid_url", + [ + pytest.param("http://ex☃mple.test/", id="invalid-idn"), + pytest.param("https://example.com/a\nb", id="newline"), + pytest.param("https://example.com/a\rb", id="carriage-return"), + pytest.param("https://example.com/\x00b", id="nul"), + pytest.param("https://example.com/a b", id="ascii-space"), + pytest.param("https://example.com/a\u00a0b", id="unicode-whitespace"), + pytest.param("https://example.com/\x7fb", id="del"), + ], +) +def test_endpoint_rejects_invalid_httpx_urls_and_whitespace( + app: Flask, field_name: str, invalid_url: str +) -> None: + data = {"source": VALID_SOURCE, "target": VALID_TARGET} + data[field_name] = invalid_url + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm(formdata=MultiDict(data), meta={"csrf": False}) + assert not form.validate() + assert "Invalid URL." in form.errors[field_name] + + +def test_endpoint_accepts_encoded_path_space(app: Flask) -> None: + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm( + formdata=MultiDict( + {"source": "https://source.example/a%20b", "target": VALID_TARGET} + ), + meta={"csrf": False}, + ) + assert form.validate() diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index c795115..44d1cc2 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -35,6 +35,7 @@ def create_webmention( target: str = TARGET_URL, status: str = "received", failure_reason: str | None = None, + recovery_attempts: int = 0, ) -> uuid.UUID: identifier = uuid.uuid7() @@ -45,6 +46,7 @@ def create_webmention( target=target, status=status, failure_reason=failure_reason, + recovery_attempts=recovery_attempts, ) db.session.add(webmention) @@ -669,6 +671,106 @@ def test_reaper_requeues_and_verifies_integration( check.assert_called_once_with(SOURCE_URL, TARGET_URL) +@patch("webmentions_ssg.tasks.receiver.source_mentions_target") +@pytest.mark.parametrize( + "unexpected_error", + [ + pytest.param(httpx.InvalidURL("Invalid IDNA hostname"), id="invalid-url"), + pytest.param( + RuntimeError("Unexpected HTML parser error"), id="unexpected-error" + ), + ], +) +def test_verify_webmention_records_unexpected_error_as_terminal_failure( + source_mentions_target: Mock, + app: Flask, + receiver: ModuleType, + unexpected_error: Exception, +) -> None: + identifier = create_webmention(app, source="http://ex☃mple.test/") + source_mentions_target.side_effect = unexpected_error + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == ("failed", str(unexpected_error)) + source_mentions_target.assert_called_once() + # Failed rows are not eligible for automatic reaper recovery. + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + + +def test_reaper_caps_stale_verification_recoveries( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["VERIFICATION_MAX_RECOVERIES"] = 3 + identifier = create_webmention(app, status="verifying") + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + # Simulate workers that repeatedly crash before completing the task. + for attempt in range(3): + set_webmention_updated_at( + app, identifier, datetime.now(UTC) - timedelta(hours=1) + ) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 1 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.status == "received" + assert row.recovery_attempts == attempt + 1 + + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + assert row.status == "failed" + assert row.failure_reason == receiver.RECOVERY_EXHAUSTED_REASON + assert row.recovery_attempts == 3 + assert receiver.recover_stale_webmentions() == 0 + assert enqueue.call_count == 3 + + +def test_reaper_caps_received_rows_lost_before_first_task( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + app.config["VERIFICATION_MAX_RECOVERIES"] = 1 + identifier = create_webmention(app, status="received", recovery_attempts=1) + set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1)) + enqueue = Mock() + monkeypatch.setattr(receiver, "verify_webmention", enqueue) + + with app.app_context(): + assert receiver.recover_stale_webmentions() == 0 + assert get_webmention_state(app, identifier) == ( + "failed", + receiver.RECOVERY_EXHAUSTED_REASON, + ) + enqueue.assert_not_called() + + +def test_exhausted_row_cannot_be_revived_by_late_huey_retry( + app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_webmention( + app, + status="failed", + failure_reason=receiver.RECOVERY_EXHAUSTED_REASON, + recovery_attempts=3, + ) + source_mentions_target = Mock(return_value=True) + monkeypatch.setattr(receiver, "source_mentions_target", source_mentions_target) + + receiver.verify_webmention.call_local(identifier) + + assert get_webmention_state(app, identifier) == ( + "failed", + receiver.RECOVERY_EXHAUSTED_REASON, + ) + source_mentions_target.assert_not_called() + + def test_verify_webmention_ignores_unknown_identifier( receiver: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_views.py b/tests/test_views.py index f5e301f..2108b38 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -576,6 +576,11 @@ def test_reverify_received_webmention( identifier = create_received_webmention( app, status="failed", failure_reason="Previous failure" ) + with app.app_context(): + row = db.session.get(ReceivedWebmention, identifier) + assert row is not None + row.recovery_attempts = 3 + db.session.commit() mock_admin_action_form(views_module, monkeypatch, valid=True) verify_webmention = Mock() monkeypatch.setattr(views_module, "verify_webmention", verify_webmention) @@ -590,6 +595,7 @@ def test_reverify_received_webmention( assert webmention is not None assert webmention.status == "received" assert webmention.failure_reason is None + assert webmention.recovery_attempts == 0 verify_webmention.assert_called_once_with(identifier) @@ -911,6 +917,7 @@ def test_endpoint_reverifies_after_cooldown( target=TARGET_URL, status="failed", failure_reason="Previous failure", + recovery_attempts=3, updated_at=datetime.now(UTC) - timedelta(minutes=11), ) db.session.add(webmention) @@ -926,6 +933,7 @@ def test_endpoint_reverifies_after_cooldown( assert stored is not None assert stored.status == "received" assert stored.failure_reason is None + assert stored.recovery_attempts == 0 verify.assert_called_once_with(identifier) @@ -1194,3 +1202,27 @@ def test_endpoint_rejects_request_bodies_over_configured_maximum( "/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL} ) assert response.status_code == 413 + + +def test_endpoint_rejects_bad_url_before_inserting_row( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + from webmentions_ssg import views + + verify_webmention = Mock() + monkeypatch.setattr(views, "verify_webmention", verify_webmention) + + response = client.post( + "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL} + ) + + assert response.status_code == 400 + assert "source" in response.get_json() + verify_webmention.assert_not_called() + with app.app_context(): + assert ( + db.session.scalar( + sa.select(sa.func.count()).select_from(ReceivedWebmention) + ) + == 0 + ) diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 97dd682..e91c651 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -56,6 +56,8 @@ class DefaultConfig: REQUEST_TOTAL_TIMEOUT = 30.0 # Requeue abandoned verification attempts after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 + # Maximum automatic recoveries of a stale received/verifying row. + VERIFICATION_MAX_RECOVERIES = 3 VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" SOURCE_DIRECTORY = None diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index 5006365..589cbd9 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,13 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes +from .validators import ( + AllowedHostname, + HTTPXURL, + NotEqualTo, + PublicIPLiteral, + URLMaxBytes, +) class LoginForm(FlaskForm): @@ -36,6 +42,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", @@ -51,6 +58,7 @@ class EndpointForm(FlaskForm): validators=[ InputRequired(), URLMaxBytes(), + HTTPXURL(), URL(), Regexp( "^https?://.*", diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index c1f1c97..384c7a1 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -5,10 +5,13 @@ """Provide validators for URL safety and form field constraints.""" import ipaddress +import unicodedata from urllib.parse import urlsplit +import httpx from flask import current_app from wtforms import Field, ValidationError +from wtforms.validators import StopValidation from wtforms.form import BaseForm from ..url_security import AddressResolutionError, is_public_address, is_public_url @@ -40,6 +43,33 @@ class URLMaxBytes: ) +class HTTPXURL: + """Reject URLs that cannot be fetched by httpx or contain whitespace/controls. + + httpx automatically percent-encodes some whitespace. Reject it instead of + silently changing the submitted URL (which may be used in exact URL matches). + """ + + def __call__(self, form: BaseForm, field: Field) -> None: + value = field.data + if not isinstance(value, str): + raise StopValidation(field.gettext("Invalid URL.")) + + if any( + character.isspace() or unicodedata.category(character) == "Cc" + for character in value + ): + raise StopValidation(field.gettext("Invalid URL.")) + + try: + parsed = httpx.URL(value) + except (httpx.InvalidURL, ValueError, UnicodeError) as exc: + raise StopValidation(field.gettext("Invalid URL.")) from exc + + if not parsed.host: + raise StopValidation(field.gettext("Invalid URL.")) + + class NotEqualTo: """ Validate that a field does not equal another field. diff --git a/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py new file mode 100644 index 0000000..c516636 --- /dev/null +++ b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cap stale received Webmention verification recovery per row. + +Revision ID: e62b1a7c4d90 +Revises: d845aa7fbc19 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "e62b1a7c4d90" +down_revision = "d845aa7fbc19" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.add_column( + sa.Column( + "recovery_attempts", sa.Integer(), nullable=False, server_default="0" + ) + ) + + +def downgrade(): + with op.batch_alter_table("received_webmentions") as batch_op: + batch_op.drop_column("recovery_attempts") diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 0dfbb78..31516d5 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -121,6 +121,11 @@ class ReceivedWebmention(Base): return value status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh + # explicit submission resets this counter. + recovery_attempts: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) failure_reason: Mapped[str | None] = mapped_column( Text(), nullable=True, default=None ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index f999d1b..acd66a1 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport from ..url_security import AddressResolutionError, UnsafeAddressError +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + class VerificationError(Exception): """The source permanently failed ReceivedWebmention verification.""" @@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: # should not start another concurrent verification. if webmention.status not in {"received", "failed"}: return + # A reaper-exhausted row must not be revived by a late Huey retry. + if ( + webmention.status == "failed" + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return previous_status = webmention.status previous_updated_at = webmention.updated_at @@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: except (TemporaryFetchError, httpx.RequestError) as exc: finish("failed", str(exc) or "Source could not be fetched") raise + except Exception as exc: + # InvalidURL is not a RequestError; other unexpected source errors must + # also leave the row in a terminal state, without an endless retry loop. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", webmention_uuid + ) + finish("failed", str(exc) or type(exc).__name__) else: if mentions_target: finish("verified", None) @@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int: updates prevent overlapping reapers from enqueueing the same row. Each reclaimed row is committed *before* enqueueing, so the worker can claim it. A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. :return: Number of webmentions requeued. """ @@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int: ReceivedWebmention.uuid, ReceivedWebmention.status, ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_(("verifying", "received")), ReceivedWebmention.updated_at <= cutoff, @@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int: ).all() recovered = 0 - for identifier, status, updated_at in stale: + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == identifier, ReceivedWebmention.status == status, ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + status="failed" if at_limit else "received", + failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None, + updated_at=now, + recovery_attempts=( + recovery_attempts if at_limit else recovery_attempts + 1 + ), ) - .values(status="received", failure_reason=None, updated_at=now) ) if changed.rowcount != 1: db.session.rollback() continue db.session.commit() + if at_limit: + exhausted += 1 + continue verify_webmention(identifier) recovered += 1 + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + if recovered: current_app.logger.warning( "Requeued %d stale received webmention verifications", recovered diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index fcb5f2d..0836869 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -185,6 +185,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: webmention.status = "received" webmention.failure_reason = None + webmention.recovery_attempts = 0 db.session.commit() @@ -395,7 +396,12 @@ def endpoint() -> ResponseReturnValue: ReceivedWebmention.uuid == webmention.uuid, ReceivedWebmention.updated_at <= cutoff, ) - .values(status="received", failure_reason=None, updated_at=now) + .values( + status="received", + failure_reason=None, + recovery_attempts=0, + updated_at=now, + ) .execution_options(synchronize_session=False) ), ) |
