diff options
| author | Dennis Fink | 2026-10-30 20:41:25 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-10-30 20:41:25 +0100 |
| commit | 53370723df05193938ce3c2d55a8e12f2a87b17b (patch) | |
| tree | e67ec12d3635d141bdc670fbfc5fa6bfb87cd69c | |
| parent | 13a9d9ddce09230017699bc34ac3bf08ed2f84ed (diff) | |
| download | webmentions-ssg-53370723df05193938ce3c2d55a8e12f2a87b17b.tar.gz webmentions-ssg-53370723df05193938ce3c2d55a8e12f2a87b17b.zip | |
fix(core): replace Huey pickle serialization with JSON
Prevent broker-controlled task and result payloads from deserializing
arbitrary Python objects by using a restricted JSON serializer with an
explicit type allowlist.
Fail closed on serializer overrides and document broker isolation and
migration requirements for existing pickle-backed deployments.
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 18 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 7 | ||||
| -rw-r--r-- | tests/tasks/test_json_serializer.py | 99 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 10 | ||||
| -rw-r--r-- | tests/test_public_transport.py | 13 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 13 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/json_serializer.py | 104 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/url_security.py | 4 |
10 files changed, 258 insertions, 14 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index 02b0744..33afdfa 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -105,7 +105,23 @@ MAX_TARGET_BYTES = 1_000_000 # Background tasks (Huey) # --------------------------------------------------------------------------- # Use a shared, persistent backend for production app and worker processes. -# Redis must be running and reachable by both. +# Huey messages and results use a restricted JSON serializer; do not configure +# a pickle serializer in either process. Keep both on the same app version. +# +# SECURITY: Only the web app and trusted worker should be able to write to the +# broker. Redis should bind to loopback or a private network, with protected +# mode, firewall rules, authentication/dedicated ACLs and TLS when remote. +# Do not expose Redis to public networks or untrusted tenants: even without +# pickle, an attacker who can write jobs can invoke registered tasks. +# For file/SQLite backends, restrict directory and database permissions to +# the app/worker account. For Postgres, use a dedicated least-privilege role +# and private/ACL-restricted network access. +# +# MIGRATION FROM PICKLE: Drain outstanding jobs on the old trusted deployment +# first, stop all producers and workers, and clear or migrate old queue, +# scheduled-task and result data before restarting *both* with JSON. +# Old pickle payloads cannot be read by the new serializer. Do not deserialize +# legacy data from any backend that untrusted parties could have modified. HUEY_URL = "redis://127.0.0.1:6379/0" # --------------------------------------------------------------------------- diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index ffcbd8e..0f5cb23 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -113,8 +113,7 @@ def test_endpoint_does_not_do_dns_in_web_request( raise AssertionError("No DNS lookup belongs in form validation") monkeypatch.setattr( - "webmentions_ssg.url_security.dns.resolver.resolve_name", - unexpected_dns_lookup, + "webmentions_ssg.url_security.dns.resolver.resolve_name", unexpected_dns_lookup ) with app.test_request_context("/endpoint", method="POST"): for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"): @@ -125,7 +124,9 @@ def test_endpoint_does_not_do_dns_in_web_request( assert form.validate() is (source == VALID_SOURCE) -@pytest.mark.parametrize("source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"]) +@pytest.mark.parametrize( + "source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"] +) def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None: with app.test_request_context("/endpoint", method="POST"): form = EndpointForm( diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py new file mode 100644 index 0000000..108e8a6 --- /dev/null +++ b/tests/tasks/test_json_serializer.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Ensure broker-controlled data is never deserialized using pickle.""" + +import pickle +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from flask import Flask +from huey.utils import Error, SKIPPED + +from webmentions_ssg.tasks.extension import Huey +from webmentions_ssg.tasks.json_serializer import JSONSerializer + + +def test_json_is_default_even_for_production_style_broker(app: Flask) -> None: + app.config["HUEY"] = {"immediate": False} + huey = Huey(app) + + assert isinstance(huey.huey.serializer, JSONSerializer) + assert huey.huey.serializer.serialize("hello") == b'"hello"' + + +def test_uuid_and_retry_message_survive_enqueue_dequeue(app: Flask) -> None: + app.config["HUEY"] = {"immediate": False} + huey = Huey(app) + identifier = uuid.uuid4() + + @huey.task(retries=2, retry_delay=50) + def check(webmention_uuid: uuid.UUID) -> str: + return str(webmention_uuid) + + task = check.s(identifier) + task.eta = datetime.now(UTC) + timedelta(minutes=5) + raw = huey.huey.serialize_task(task) + assert raw.startswith(b'{"type":"message"') + assert b"pickle" not in raw + + restored = huey.huey.deserialize_task(raw) + assert restored.args == (identifier,) + assert restored.eta == task.eta + assert restored.retries == 2 + assert restored.retry_delay == 50 + + huey.huey.enqueue(task) + from_queue = huey.huey.dequeue() + assert from_queue.args == (identifier,) + + +def test_result_and_error_data_roundtrip() -> None: + serializer = JSONSerializer() + identifier = uuid.uuid4() + payload = { + "ids": [identifier], + "nested": ("str", None, 3), + "error": Error({"error": "failed", "retries": 0}), + "skipped": SKIPPED, + "tag_collision": {"type": "uuid", "value": "not a UUID"}, + } + assert serializer.deserialize(serializer.serialize(payload)) == payload + + +def test_rejects_pickle_payload() -> None: + serializer = JSONSerializer() + with pytest.raises((ValueError, UnicodeDecodeError)): + serializer.deserialize(pickle.dumps({"untrusted": True})) + + +def test_rejects_unknown_and_malformed_types() -> None: + serializer = JSONSerializer() + with pytest.raises(ValueError, match="Unsupported Huey JSON type"): + serializer.deserialize(b'{"type":"arbitrary.class","value":{}}') + with pytest.raises(ValueError, match="Invalid Huey JSON data"): + serializer.deserialize(b'{"type":"uuid","value":"x","extra":1}') + with pytest.raises(ValueError, match="Invalid Huey JSON message field count"): + serializer.deserialize(b'{"type":"message","value":[]}') + with pytest.raises(ValueError, match="Invalid non-finite JSON value"): + serializer.deserialize(b"NaN") + with pytest.raises(TypeError, match="Unsupported Huey JSON value"): + serializer.serialize(object()) + + +def test_refuses_unsafe_serializer_configuration(app: Flask) -> None: + app.config["HUEY"] = {"serializer": None} + with pytest.raises(RuntimeError, match="JSON is required"): + Huey(app) + + +def test_compression_with_json(app: Flask) -> None: + app.config["HUEY"] = {"compression": True, "immediate": False} + huey = Huey(app) + payload = {"id": uuid.uuid4()} + + raw = huey.huey.serializer.serialize(payload) + assert raw.startswith(b"\x1f\x8b") + assert huey.huey.serializer.deserialize(raw) == payload diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 102410d..c795115 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -688,8 +688,9 @@ def test_fetch_source_rejects_unsafe_destination( client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = UnsafeAddressError("Local address") - with app.app_context(), pytest.raises( - receiver.VerificationError, match="non-public address" + with ( + app.app_context(), + pytest.raises(receiver.VerificationError, match="non-public address"), ): receiver.fetch_source(SOURCE_URL) @@ -700,8 +701,9 @@ def test_fetch_source_treats_dns_failure_as_temporary( ) -> None: client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = AddressResolutionError("DNS timeout") - with app.app_context(), pytest.raises( - receiver.TemporaryFetchError, match="could not be resolved" + with ( + app.app_context(), + pytest.raises(receiver.TemporaryFetchError, match="could not be resolved"), ): receiver.fetch_source(SOURCE_URL) diff --git a/tests/test_public_transport.py b/tests/test_public_transport.py index e7d77fa..df7a1c3 100644 --- a/tests/test_public_transport.py +++ b/tests/test_public_transport.py @@ -70,7 +70,10 @@ class FakeStream(httpcore.NetworkStream): self.writes.extend(buffer) def start_tls( - self, ssl_context, server_hostname: str | None = None, timeout: float | None = None + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, ) -> httpcore.NetworkStream: self.sni = server_hostname return self @@ -107,7 +110,9 @@ def test_deadline_clips_each_read_and_stops_trickle( monkeypatch: pytest.MonkeyPatch, ) -> None: clock = [100.0] - monkeypatch.setattr("webmentions_ssg.public_transport.time.monotonic", lambda: clock[0]) + monkeypatch.setattr( + "webmentions_ssg.public_transport.time.monotonic", lambda: clock[0] + ) fake_stream = Mock() fake_stream.read.return_value = b"x" stream = DeadlineStream(fake_stream, NetworkDeadline(5.0)) @@ -141,7 +146,9 @@ def test_redirects_go_through_same_destination_guard( monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", connect) with ( - httpx.Client(transport=PublicHTTPTransport(5.0), follow_redirects=True) as client, + httpx.Client( + transport=PublicHTTPTransport(5.0), follow_redirects=True + ) as client, pytest.raises(UnsafeAddressError, match="Non-public"), ): client.get("http://source.example/post") diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index e8f33eb..2ce2b5b 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -88,7 +88,7 @@ class AllowedHostname: """ try: hostname = urlsplit(field.data).hostname - except (ValueError, TypeError): + except ValueError, TypeError: hostname = None if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]: return diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index 79d7df9..df6cb21 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -13,6 +13,8 @@ from flask import Flask from huey import Huey as BaseHuey from huey.api import TaskWrapper +from .json_serializer import JSONSerializer + P = ParamSpec("P") R = TypeVar("R") @@ -57,6 +59,17 @@ class Huey: **app.config.get("HUEY", {}), } + # An arbitrary serializer supplied via HUEY would silently restore + # pickle deserialization of broker-controlled bytes. Fail closed. + if "serializer" in config: + raise RuntimeError( + "HUEY.serializer cannot be overridden; JSON is required." + ) + config["serializer"] = JSONSerializer( + compression=config.pop("compression", False), + use_zlib=config.pop("use_zlib", False), + ) + url = app.config.get("HUEY_URL", config.pop("url", "memory://")) huey_class, storage_kwargs = self.backend_from_url(url) diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py new file mode 100644 index 0000000..473a544 --- /dev/null +++ b/webmentions_ssg/tasks/json_serializer.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Serialize Huey messages as JSON without reconstructing arbitrary Python objects.""" + +import json +import uuid +from datetime import datetime +from typing import Any + +from huey.registry import Message +from huey.serializer import Serializer +from huey.utils import Error, SKIPPED + + +class JSONSerializer(Serializer): + """A deliberately restricted, non-pickle serializer for our Huey tasks. + + JSON has no native UUID, datetime, tuple or Huey message representation. + Each non-primitive value is encoded with a fixed tag and decoded using an + allowlist; no module imports, user-supplied classes or object constructors + are evaluated. Unsupported objects fail closed rather than being pickled. + """ + + def _serialize(self, data: Any) -> bytes: + return json.dumps( + self._encode(data), + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ).encode("utf-8") + + def _deserialize(self, data: bytes) -> Any: + def reject_nonfinite(value: str) -> None: + raise ValueError(f"Invalid non-finite JSON value: {value}") + + return self._decode(json.loads(data, parse_constant=reject_nonfinite)) + + @classmethod + def _encode(cls, value: Any) -> Any: + if value is None or type(value) in {str, int, float, bool}: + return value + if isinstance(value, uuid.UUID): + return {"type": "uuid", "value": str(value)} + if isinstance(value, datetime): + return {"type": "datetime", "value": value.isoformat()} + if isinstance(value, Message): + return {"type": "message", "value": [cls._encode(v) for v in value]} + if isinstance(value, Error): + return {"type": "error", "value": cls._encode(value.metadata)} + if value is SKIPPED: + return {"type": "skipped", "value": None} + if isinstance(value, tuple): + return {"type": "tuple", "value": [cls._encode(v) for v in value]} + if isinstance(value, list): + return {"type": "list", "value": [cls._encode(v) for v in value]} + if isinstance(value, dict) and all(type(k) is str for k in value): + return { + "type": "dict", + "value": [[k, cls._encode(v)] for k, v in value.items()], + } + raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}") + + @classmethod + def _decode(cls, value: Any) -> Any: + if value is None or type(value) in {str, int, float, bool}: + return value + if not isinstance(value, dict) or set(value) != {"type", "value"}: + raise ValueError("Invalid Huey JSON data") + + kind, payload = value["type"], value["value"] + if kind == "uuid" and isinstance(payload, str): + return uuid.UUID(payload) + if kind == "datetime" and isinstance(payload, str): + return datetime.fromisoformat(payload) + if kind == "message" and isinstance(payload, list): + if len(payload) != len(Message._fields): + raise ValueError("Invalid Huey JSON message field count") + return Message(*(cls._decode(v) for v in payload)) + if kind == "error": + metadata = cls._decode(payload) + if not isinstance(metadata, dict): + raise ValueError("Invalid Huey JSON error metadata") + return Error(metadata) + if kind == "skipped" and payload is None: + return SKIPPED + if kind in {"tuple", "list"} and isinstance(payload, list): + items = [cls._decode(v) for v in payload] + return tuple(items) if kind == "tuple" else items + if kind == "dict" and isinstance(payload, list): + result: dict[str, Any] = {} + for pair in payload: + if ( + not isinstance(pair, list) + or len(pair) != 2 + or type(pair[0]) is not str + ): + raise ValueError("Invalid Huey JSON dictionary entry") + if pair[0] in result: + raise ValueError("Duplicate Huey JSON dictionary key") + result[pair[0]] = cls._decode(pair[1]) + return result + raise ValueError("Unsupported Huey JSON type") diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 7121f6a..4da1a5c 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -210,7 +210,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non """ try: head_response = client.head(target, headers=DISCOVERY_HEADERS) - except (httpx.RequestError, TemporarySenderError): + except httpx.RequestError, TemporarySenderError: # HEAD is only an optimization. Servers and intermediaries may reject # or drop it even when fetching the document with GET would succeed. pass diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py index f943c7e..4e7940b 100644 --- a/webmentions_ssg/url_security.py +++ b/webmentions_ssg/url_security.py @@ -19,7 +19,9 @@ class UnsafeAddressError(ValueError): """A hostname or IP literal identifies a non-public network address.""" -def public_addresses(hostname: str, *, lifetime: float | None = None) -> tuple[str, ...]: +def public_addresses( + hostname: str, *, lifetime: float | None = None +) -> tuple[str, ...]: """Resolve *once* and return only checked, globally routable IP addresses. A DNS name with a mixture of public and private answers is rejected in its |
