aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--contrib/webmentions-ssg.example.cfg18
-rw-r--r--tests/forms/test_init.py7
-rw-r--r--tests/tasks/test_json_serializer.py99
-rw-r--r--tests/tasks/test_receiver.py10
-rw-r--r--tests/test_public_transport.py13
-rw-r--r--webmentions_ssg/forms/validators.py2
-rw-r--r--webmentions_ssg/tasks/extension.py13
-rw-r--r--webmentions_ssg/tasks/json_serializer.py104
-rw-r--r--webmentions_ssg/tasks/sender.py2
-rw-r--r--webmentions_ssg/url_security.py4
10 files changed, 258 insertions, 14 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg
index 02b0744..33afdfa 100644
--- a/contrib/webmentions-ssg.example.cfg
+++ b/contrib/webmentions-ssg.example.cfg
@@ -105,7 +105,23 @@ MAX_TARGET_BYTES = 1_000_000
# Background tasks (Huey)
# ---------------------------------------------------------------------------
# Use a shared, persistent backend for production app and worker processes.
-# Redis must be running and reachable by both.
+# Huey messages and results use a restricted JSON serializer; do not configure
+# a pickle serializer in either process. Keep both on the same app version.
+#
+# SECURITY: Only the web app and trusted worker should be able to write to the
+# broker. Redis should bind to loopback or a private network, with protected
+# mode, firewall rules, authentication/dedicated ACLs and TLS when remote.
+# Do not expose Redis to public networks or untrusted tenants: even without
+# pickle, an attacker who can write jobs can invoke registered tasks.
+# For file/SQLite backends, restrict directory and database permissions to
+# the app/worker account. For Postgres, use a dedicated least-privilege role
+# and private/ACL-restricted network access.
+#
+# MIGRATION FROM PICKLE: Drain outstanding jobs on the old trusted deployment
+# first, stop all producers and workers, and clear or migrate old queue,
+# scheduled-task and result data before restarting *both* with JSON.
+# Old pickle payloads cannot be read by the new serializer. Do not deserialize
+# legacy data from any backend that untrusted parties could have modified.
HUEY_URL = "redis://127.0.0.1:6379/0"
# ---------------------------------------------------------------------------
diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py
index ffcbd8e..0f5cb23 100644
--- a/tests/forms/test_init.py
+++ b/tests/forms/test_init.py
@@ -113,8 +113,7 @@ def test_endpoint_does_not_do_dns_in_web_request(
raise AssertionError("No DNS lookup belongs in form validation")
monkeypatch.setattr(
- "webmentions_ssg.url_security.dns.resolver.resolve_name",
- unexpected_dns_lookup,
+ "webmentions_ssg.url_security.dns.resolver.resolve_name", unexpected_dns_lookup
)
with app.test_request_context("/endpoint", method="POST"):
for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"):
@@ -125,7 +124,9 @@ def test_endpoint_does_not_do_dns_in_web_request(
assert form.validate() is (source == VALID_SOURCE)
-@pytest.mark.parametrize("source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"])
+@pytest.mark.parametrize(
+ "source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"]
+)
def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None:
with app.test_request_context("/endpoint", method="POST"):
form = EndpointForm(
diff --git a/tests/tasks/test_json_serializer.py b/tests/tasks/test_json_serializer.py
new file mode 100644
index 0000000..108e8a6
--- /dev/null
+++ b/tests/tasks/test_json_serializer.py
@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Ensure broker-controlled data is never deserialized using pickle."""
+
+import pickle
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from flask import Flask
+from huey.utils import Error, SKIPPED
+
+from webmentions_ssg.tasks.extension import Huey
+from webmentions_ssg.tasks.json_serializer import JSONSerializer
+
+
+def test_json_is_default_even_for_production_style_broker(app: Flask) -> None:
+ app.config["HUEY"] = {"immediate": False}
+ huey = Huey(app)
+
+ assert isinstance(huey.huey.serializer, JSONSerializer)
+ assert huey.huey.serializer.serialize("hello") == b'"hello"'
+
+
+def test_uuid_and_retry_message_survive_enqueue_dequeue(app: Flask) -> None:
+ app.config["HUEY"] = {"immediate": False}
+ huey = Huey(app)
+ identifier = uuid.uuid4()
+
+ @huey.task(retries=2, retry_delay=50)
+ def check(webmention_uuid: uuid.UUID) -> str:
+ return str(webmention_uuid)
+
+ task = check.s(identifier)
+ task.eta = datetime.now(UTC) + timedelta(minutes=5)
+ raw = huey.huey.serialize_task(task)
+ assert raw.startswith(b'{"type":"message"')
+ assert b"pickle" not in raw
+
+ restored = huey.huey.deserialize_task(raw)
+ assert restored.args == (identifier,)
+ assert restored.eta == task.eta
+ assert restored.retries == 2
+ assert restored.retry_delay == 50
+
+ huey.huey.enqueue(task)
+ from_queue = huey.huey.dequeue()
+ assert from_queue.args == (identifier,)
+
+
+def test_result_and_error_data_roundtrip() -> None:
+ serializer = JSONSerializer()
+ identifier = uuid.uuid4()
+ payload = {
+ "ids": [identifier],
+ "nested": ("str", None, 3),
+ "error": Error({"error": "failed", "retries": 0}),
+ "skipped": SKIPPED,
+ "tag_collision": {"type": "uuid", "value": "not a UUID"},
+ }
+ assert serializer.deserialize(serializer.serialize(payload)) == payload
+
+
+def test_rejects_pickle_payload() -> None:
+ serializer = JSONSerializer()
+ with pytest.raises((ValueError, UnicodeDecodeError)):
+ serializer.deserialize(pickle.dumps({"untrusted": True}))
+
+
+def test_rejects_unknown_and_malformed_types() -> None:
+ serializer = JSONSerializer()
+ with pytest.raises(ValueError, match="Unsupported Huey JSON type"):
+ serializer.deserialize(b'{"type":"arbitrary.class","value":{}}')
+ with pytest.raises(ValueError, match="Invalid Huey JSON data"):
+ serializer.deserialize(b'{"type":"uuid","value":"x","extra":1}')
+ with pytest.raises(ValueError, match="Invalid Huey JSON message field count"):
+ serializer.deserialize(b'{"type":"message","value":[]}')
+ with pytest.raises(ValueError, match="Invalid non-finite JSON value"):
+ serializer.deserialize(b"NaN")
+ with pytest.raises(TypeError, match="Unsupported Huey JSON value"):
+ serializer.serialize(object())
+
+
+def test_refuses_unsafe_serializer_configuration(app: Flask) -> None:
+ app.config["HUEY"] = {"serializer": None}
+ with pytest.raises(RuntimeError, match="JSON is required"):
+ Huey(app)
+
+
+def test_compression_with_json(app: Flask) -> None:
+ app.config["HUEY"] = {"compression": True, "immediate": False}
+ huey = Huey(app)
+ payload = {"id": uuid.uuid4()}
+
+ raw = huey.huey.serializer.serialize(payload)
+ assert raw.startswith(b"\x1f\x8b")
+ assert huey.huey.serializer.deserialize(raw) == payload
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 102410d..c795115 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -688,8 +688,9 @@ def test_fetch_source_rejects_unsafe_destination(
client = httpx_client.return_value.__enter__.return_value
client.stream.side_effect = UnsafeAddressError("Local address")
- with app.app_context(), pytest.raises(
- receiver.VerificationError, match="non-public address"
+ with (
+ app.app_context(),
+ pytest.raises(receiver.VerificationError, match="non-public address"),
):
receiver.fetch_source(SOURCE_URL)
@@ -700,8 +701,9 @@ def test_fetch_source_treats_dns_failure_as_temporary(
) -> None:
client = httpx_client.return_value.__enter__.return_value
client.stream.side_effect = AddressResolutionError("DNS timeout")
- with app.app_context(), pytest.raises(
- receiver.TemporaryFetchError, match="could not be resolved"
+ with (
+ app.app_context(),
+ pytest.raises(receiver.TemporaryFetchError, match="could not be resolved"),
):
receiver.fetch_source(SOURCE_URL)
diff --git a/tests/test_public_transport.py b/tests/test_public_transport.py
index e7d77fa..df7a1c3 100644
--- a/tests/test_public_transport.py
+++ b/tests/test_public_transport.py
@@ -70,7 +70,10 @@ class FakeStream(httpcore.NetworkStream):
self.writes.extend(buffer)
def start_tls(
- self, ssl_context, server_hostname: str | None = None, timeout: float | None = None
+ self,
+ ssl_context,
+ server_hostname: str | None = None,
+ timeout: float | None = None,
) -> httpcore.NetworkStream:
self.sni = server_hostname
return self
@@ -107,7 +110,9 @@ def test_deadline_clips_each_read_and_stops_trickle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = [100.0]
- monkeypatch.setattr("webmentions_ssg.public_transport.time.monotonic", lambda: clock[0])
+ monkeypatch.setattr(
+ "webmentions_ssg.public_transport.time.monotonic", lambda: clock[0]
+ )
fake_stream = Mock()
fake_stream.read.return_value = b"x"
stream = DeadlineStream(fake_stream, NetworkDeadline(5.0))
@@ -141,7 +146,9 @@ def test_redirects_go_through_same_destination_guard(
monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", connect)
with (
- httpx.Client(transport=PublicHTTPTransport(5.0), follow_redirects=True) as client,
+ httpx.Client(
+ transport=PublicHTTPTransport(5.0), follow_redirects=True
+ ) as client,
pytest.raises(UnsafeAddressError, match="Non-public"),
):
client.get("http://source.example/post")
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index e8f33eb..2ce2b5b 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -88,7 +88,7 @@ class AllowedHostname:
"""
try:
hostname = urlsplit(field.data).hostname
- except (ValueError, TypeError):
+ except ValueError, TypeError:
hostname = None
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
return
diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py
index 79d7df9..df6cb21 100644
--- a/webmentions_ssg/tasks/extension.py
+++ b/webmentions_ssg/tasks/extension.py
@@ -13,6 +13,8 @@ from flask import Flask
from huey import Huey as BaseHuey
from huey.api import TaskWrapper
+from .json_serializer import JSONSerializer
+
P = ParamSpec("P")
R = TypeVar("R")
@@ -57,6 +59,17 @@ class Huey:
**app.config.get("HUEY", {}),
}
+ # An arbitrary serializer supplied via HUEY would silently restore
+ # pickle deserialization of broker-controlled bytes. Fail closed.
+ if "serializer" in config:
+ raise RuntimeError(
+ "HUEY.serializer cannot be overridden; JSON is required."
+ )
+ config["serializer"] = JSONSerializer(
+ compression=config.pop("compression", False),
+ use_zlib=config.pop("use_zlib", False),
+ )
+
url = app.config.get("HUEY_URL", config.pop("url", "memory://"))
huey_class, storage_kwargs = self.backend_from_url(url)
diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py
new file mode 100644
index 0000000..473a544
--- /dev/null
+++ b/webmentions_ssg/tasks/json_serializer.py
@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Serialize Huey messages as JSON without reconstructing arbitrary Python objects."""
+
+import json
+import uuid
+from datetime import datetime
+from typing import Any
+
+from huey.registry import Message
+from huey.serializer import Serializer
+from huey.utils import Error, SKIPPED
+
+
+class JSONSerializer(Serializer):
+ """A deliberately restricted, non-pickle serializer for our Huey tasks.
+
+ JSON has no native UUID, datetime, tuple or Huey message representation.
+ Each non-primitive value is encoded with a fixed tag and decoded using an
+ allowlist; no module imports, user-supplied classes or object constructors
+ are evaluated. Unsupported objects fail closed rather than being pickled.
+ """
+
+ def _serialize(self, data: Any) -> bytes:
+ return json.dumps(
+ self._encode(data),
+ ensure_ascii=False,
+ allow_nan=False,
+ separators=(",", ":"),
+ ).encode("utf-8")
+
+ def _deserialize(self, data: bytes) -> Any:
+ def reject_nonfinite(value: str) -> None:
+ raise ValueError(f"Invalid non-finite JSON value: {value}")
+
+ return self._decode(json.loads(data, parse_constant=reject_nonfinite))
+
+ @classmethod
+ def _encode(cls, value: Any) -> Any:
+ if value is None or type(value) in {str, int, float, bool}:
+ return value
+ if isinstance(value, uuid.UUID):
+ return {"type": "uuid", "value": str(value)}
+ if isinstance(value, datetime):
+ return {"type": "datetime", "value": value.isoformat()}
+ if isinstance(value, Message):
+ return {"type": "message", "value": [cls._encode(v) for v in value]}
+ if isinstance(value, Error):
+ return {"type": "error", "value": cls._encode(value.metadata)}
+ if value is SKIPPED:
+ return {"type": "skipped", "value": None}
+ if isinstance(value, tuple):
+ return {"type": "tuple", "value": [cls._encode(v) for v in value]}
+ if isinstance(value, list):
+ return {"type": "list", "value": [cls._encode(v) for v in value]}
+ if isinstance(value, dict) and all(type(k) is str for k in value):
+ return {
+ "type": "dict",
+ "value": [[k, cls._encode(v)] for k, v in value.items()],
+ }
+ raise TypeError(f"Unsupported Huey JSON value: {type(value).__name__}")
+
+ @classmethod
+ def _decode(cls, value: Any) -> Any:
+ if value is None or type(value) in {str, int, float, bool}:
+ return value
+ if not isinstance(value, dict) or set(value) != {"type", "value"}:
+ raise ValueError("Invalid Huey JSON data")
+
+ kind, payload = value["type"], value["value"]
+ if kind == "uuid" and isinstance(payload, str):
+ return uuid.UUID(payload)
+ if kind == "datetime" and isinstance(payload, str):
+ return datetime.fromisoformat(payload)
+ if kind == "message" and isinstance(payload, list):
+ if len(payload) != len(Message._fields):
+ raise ValueError("Invalid Huey JSON message field count")
+ return Message(*(cls._decode(v) for v in payload))
+ if kind == "error":
+ metadata = cls._decode(payload)
+ if not isinstance(metadata, dict):
+ raise ValueError("Invalid Huey JSON error metadata")
+ return Error(metadata)
+ if kind == "skipped" and payload is None:
+ return SKIPPED
+ if kind in {"tuple", "list"} and isinstance(payload, list):
+ items = [cls._decode(v) for v in payload]
+ return tuple(items) if kind == "tuple" else items
+ if kind == "dict" and isinstance(payload, list):
+ result: dict[str, Any] = {}
+ for pair in payload:
+ if (
+ not isinstance(pair, list)
+ or len(pair) != 2
+ or type(pair[0]) is not str
+ ):
+ raise ValueError("Invalid Huey JSON dictionary entry")
+ if pair[0] in result:
+ raise ValueError("Duplicate Huey JSON dictionary key")
+ result[pair[0]] = cls._decode(pair[1])
+ return result
+ raise ValueError("Unsupported Huey JSON type")
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 7121f6a..4da1a5c 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -210,7 +210,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
"""
try:
head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except (httpx.RequestError, TemporarySenderError):
+ except httpx.RequestError, TemporarySenderError:
# HEAD is only an optimization. Servers and intermediaries may reject
# or drop it even when fetching the document with GET would succeed.
pass
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
index f943c7e..4e7940b 100644
--- a/webmentions_ssg/url_security.py
+++ b/webmentions_ssg/url_security.py
@@ -19,7 +19,9 @@ class UnsafeAddressError(ValueError):
"""A hostname or IP literal identifies a non-public network address."""
-def public_addresses(hostname: str, *, lifetime: float | None = None) -> tuple[str, ...]:
+def public_addresses(
+ hostname: str, *, lifetime: float | None = None
+) -> tuple[str, ...]:
"""Resolve *once* and return only checked, globally routable IP addresses.
A DNS name with a mixture of public and private answers is rejected in its