From 53370723df05193938ce3c2d55a8e12f2a87b17b Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 20:41:25 +0100 Subject: fix(core): replace Huey pickle serialization with JSON Prevent broker-controlled task and result payloads from deserializing arbitrary Python objects by using a restricted JSON serializer with an explicit type allowlist. Fail closed on serializer overrides and document broker isolation and migration requirements for existing pickle-backed deployments. --- tests/forms/test_init.py | 7 +-- tests/tasks/test_json_serializer.py | 99 +++++++++++++++++++++++++++++++++++++ tests/tasks/test_receiver.py | 10 ++-- tests/test_public_transport.py | 13 +++-- 4 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 tests/tasks/test_json_serializer.py (limited to 'tests') diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index ffcbd8e..0f5cb23 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -113,8 +113,7 @@ def test_endpoint_does_not_do_dns_in_web_request( raise AssertionError("No DNS lookup belongs in form validation") monkeypatch.setattr( - "webmentions_ssg.url_security.dns.resolver.resolve_name", - unexpected_dns_lookup, + "webmentions_ssg.url_security.dns.resolver.resolve_name", unexpected_dns_lookup ) with app.test_request_context("/endpoint", method="POST"): for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"): @@ -125,7 +124,9 @@ def test_endpoint_does_not_do_dns_in_web_request( assert form.validate() is (source == VALID_SOURCE) -@pytest.mark.parametrize("source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"]) +@pytest.mark.parametrize( + "source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"] +) def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None: with app.test_request_context("/endpoint", method="POST"): form = EndpointForm( diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py new file mode 100644 index 0000000..108e8a6 --- /dev/null +++ b/tests/tasks/test_json_serializer.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Ensure broker-controlled data is never deserialized using pickle.""" + +import pickle +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from flask import Flask +from huey.utils import Error, SKIPPED + +from webmentions_ssg.tasks.extension import Huey +from webmentions_ssg.tasks.json_serializer import JSONSerializer + + +def test_json_is_default_even_for_production_style_broker(app: Flask) -> None: + app.config["HUEY"] = {"immediate": False} + huey = Huey(app) + + assert isinstance(huey.huey.serializer, JSONSerializer) + assert huey.huey.serializer.serialize("hello") == b'"hello"' + + +def test_uuid_and_retry_message_survive_enqueue_dequeue(app: Flask) -> None: + app.config["HUEY"] = {"immediate": False} + huey = Huey(app) + identifier = uuid.uuid4() + + @huey.task(retries=2, retry_delay=50) + def check(webmention_uuid: uuid.UUID) -> str: + return str(webmention_uuid) + + task = check.s(identifier) + task.eta = datetime.now(UTC) + timedelta(minutes=5) + raw = huey.huey.serialize_task(task) + assert raw.startswith(b'{"type":"message"') + assert b"pickle" not in raw + + restored = huey.huey.deserialize_task(raw) + assert restored.args == (identifier,) + assert restored.eta == task.eta + assert restored.retries == 2 + assert restored.retry_delay == 50 + + huey.huey.enqueue(task) + from_queue = huey.huey.dequeue() + assert from_queue.args == (identifier,) + + +def test_result_and_error_data_roundtrip() -> None: + serializer = JSONSerializer() + identifier = uuid.uuid4() + payload = { + "ids": [identifier], + "nested": ("str", None, 3), + "error": Error({"error": "failed", "retries": 0}), + "skipped": SKIPPED, + "tag_collision": {"type": "uuid", "value": "not a UUID"}, + } + assert serializer.deserialize(serializer.serialize(payload)) == payload + + +def test_rejects_pickle_payload() -> None: + serializer = JSONSerializer() + with pytest.raises((ValueError, UnicodeDecodeError)): + serializer.deserialize(pickle.dumps({"untrusted": True})) + + +def test_rejects_unknown_and_malformed_types() -> None: + serializer = JSONSerializer() + with pytest.raises(ValueError, match="Unsupported Huey JSON type"): + serializer.deserialize(b'{"type":"arbitrary.class","value":{}}') + with pytest.raises(ValueError, match="Invalid Huey JSON data"): + serializer.deserialize(b'{"type":"uuid","value":"x","extra":1}') + with pytest.raises(ValueError, match="Invalid Huey JSON message field count"): + serializer.deserialize(b'{"type":"message","value":[]}') + with pytest.raises(ValueError, match="Invalid non-finite JSON value"): + serializer.deserialize(b"NaN") + with pytest.raises(TypeError, match="Unsupported Huey JSON value"): + serializer.serialize(object()) + + +def test_refuses_unsafe_serializer_configuration(app: Flask) -> None: + app.config["HUEY"] = {"serializer": None} + with pytest.raises(RuntimeError, match="JSON is required"): + Huey(app) + + +def test_compression_with_json(app: Flask) -> None: + app.config["HUEY"] = {"compression": True, "immediate": False} + huey = Huey(app) + payload = {"id": uuid.uuid4()} + + raw = huey.huey.serializer.serialize(payload) + assert raw.startswith(b"\x1f\x8b") + assert huey.huey.serializer.deserialize(raw) == payload diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 102410d..c795115 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -688,8 +688,9 @@ def test_fetch_source_rejects_unsafe_destination( client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = UnsafeAddressError("Local address") - with app.app_context(), pytest.raises( - receiver.VerificationError, match="non-public address" + with ( + app.app_context(), + pytest.raises(receiver.VerificationError, match="non-public address"), ): receiver.fetch_source(SOURCE_URL) @@ -700,8 +701,9 @@ def test_fetch_source_treats_dns_failure_as_temporary( ) -> None: client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = AddressResolutionError("DNS timeout") - with app.app_context(), pytest.raises( - receiver.TemporaryFetchError, match="could not be resolved" + with ( + app.app_context(), + pytest.raises(receiver.TemporaryFetchError, match="could not be resolved"), ): receiver.fetch_source(SOURCE_URL) diff --git a/tests/test_public_transport.py b/tests/test_public_transport.py index e7d77fa..df7a1c3 100644 --- a/tests/test_public_transport.py +++ b/tests/test_public_transport.py @@ -70,7 +70,10 @@ class FakeStream(httpcore.NetworkStream): self.writes.extend(buffer) def start_tls( - self, ssl_context, server_hostname: str | None = None, timeout: float | None = None + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, ) -> httpcore.NetworkStream: self.sni = server_hostname return self @@ -107,7 +110,9 @@ def test_deadline_clips_each_read_and_stops_trickle( monkeypatch: pytest.MonkeyPatch, ) -> None: clock = [100.0] - monkeypatch.setattr("webmentions_ssg.public_transport.time.monotonic", lambda: clock[0]) + monkeypatch.setattr( + "webmentions_ssg.public_transport.time.monotonic", lambda: clock[0] + ) fake_stream = Mock() fake_stream.read.return_value = b"x" stream = DeadlineStream(fake_stream, NetworkDeadline(5.0)) @@ -141,7 +146,9 @@ def test_redirects_go_through_same_destination_guard( monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", connect) with ( - httpx.Client(transport=PublicHTTPTransport(5.0), follow_redirects=True) as client, + httpx.Client( + transport=PublicHTTPTransport(5.0), follow_redirects=True + ) as client, pytest.raises(UnsafeAddressError, match="Non-public"), ): client.get("http://source.example/post") -- cgit v1.3.1