diff options
| -rw-r--r-- | contrib/webmentions-ssg.example.cfg | 8 | ||||
| -rw-r--r-- | pyproject.toml | 3 | ||||
| -rw-r--r-- | tests/forms/test_init.py | 49 | ||||
| -rw-r--r-- | tests/forms/test_validators.py | 6 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 58 | ||||
| -rw-r--r-- | tests/tasks/test_sender.py | 37 | ||||
| -rw-r--r-- | tests/test_config.py | 1 | ||||
| -rw-r--r-- | tests/test_public_transport.py | 149 | ||||
| -rw-r--r-- | tests/test_url_security.py | 29 | ||||
| -rw-r--r-- | uv.lock | 4 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/forms/__init__.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 27 | ||||
| -rw-r--r-- | webmentions_ssg/public_transport.py | 144 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 34 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 28 | ||||
| -rw-r--r-- | webmentions_ssg/url_security.py | 63 |
17 files changed, 495 insertions, 151 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg index bf363b8..02b0744 100644 --- a/contrib/webmentions-ssg.example.cfg +++ b/contrib/webmentions-ssg.example.cfg @@ -44,8 +44,14 @@ LOGFILE_BACKUP_COUNT = 10 # Webmention. Include each hostname you serve; wildcards are not supported. ALLOWED_HOSTNAMES = {"example.com", "www.example.com"} +# Defense in depth: restrict the Huey worker service account with outbound +# firewall/network-policy rules that block loopback, RFC1918, link-local, ULA, +# and cloud metadata ranges, including IPv6. The application also pins +# validated DNS IPs, but deployment-level filtering is still necessary. + # Limits for fetching/verifying incoming sources. -REQUEST_TIMEOUT = 5.0 # seconds; also used for outgoing requests +REQUEST_TIMEOUT = 5.0 # seconds per socket operation (both directions) +REQUEST_TOTAL_TIMEOUT = 30.0 # seconds overall, including DNS, redirects and body MAX_REDIRECTS = 20 # also used for outgoing requests MAX_SOURCE_BYTES = 1_000_000 # Requeue stuck verifications and lost queued tasks after 15 minutes. diff --git a/pyproject.toml b/pyproject.toml index f42752a..9b410ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "flask-migrate>=4.1.0", "flask-sqlalchemy>=3.1.1", "flask-wtf>=1.3.0", - "httpx>=0.28.1", + "httpx>=0.28.1,<0.29", + "httpcore>=1.0.9,<1.1", "huey>=3.1.0", "mf2py>=2.0.1", "rfc3987>=1.3.8", diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py index 6fe1849..ffcbd8e 100644 --- a/tests/forms/test_init.py +++ b/tests/forms/test_init.py @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from unittest.mock import Mock - import pytest from flask import Flask from werkzeug.datastructures import MultiDict @@ -14,13 +12,6 @@ VALID_SOURCE = "https://source.example/post" VALID_TARGET = "https://dennisfink.me/blog/example/" -@pytest.fixture(autouse=True) -def public_urls(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "webmentions_ssg.forms.validators.is_public_url", Mock(return_value=True) - ) - - @pytest.mark.parametrize( ("form_data", "invalid_field", "expected_error"), [ @@ -113,3 +104,43 @@ def test_admin_action_form_accepts_submission(app: Flask) -> None: with app.test_request_context(method="POST"): form = AdminActionForm(meta={"csrf": False}) assert form.validate() + + +def test_endpoint_does_not_do_dns_in_web_request( + app: Flask, monkeypatch: pytest.MonkeyPatch +) -> None: + def unexpected_dns_lookup(*args: object, **kwargs: object) -> None: + raise AssertionError("No DNS lookup belongs in form validation") + + monkeypatch.setattr( + "webmentions_ssg.url_security.dns.resolver.resolve_name", + unexpected_dns_lookup, + ) + with app.test_request_context("/endpoint", method="POST"): + for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"): + form = EndpointForm( + formdata=MultiDict({"source": source, "target": VALID_TARGET}), + meta={"csrf": False}, + ) + assert form.validate() is (source == VALID_SOURCE) + + +@pytest.mark.parametrize("source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"]) +def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None: + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm( + formdata=MultiDict({"source": source, "target": VALID_TARGET}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "URL must use a public address." in form.errors["source"] + + +def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None: + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm( + formdata=MultiDict({"source": VALID_SOURCE, "target": "http://[::1"}), + meta={"csrf": False}, + ) + assert not form.validate() + assert "Invalid input." in form.errors["target"] diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py index 6e26039..2858b4c 100644 --- a/tests/forms/test_validators.py +++ b/tests/forms/test_validators.py @@ -112,3 +112,9 @@ def test_public_url_uses_custom_message(monkeypatch: pytest.MonkeyPatch) -> None with pytest.raises(ValidationError, match="Public URL required"): PublicURL("Public URL required")(form, form.url) + + +def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None: + form = URLForm(MultiDict({"url": "http://[::1"})) + with app.app_context(), pytest.raises(ValidationError, match="Invalid input"): + AllowedHostname()(form, form.url) diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 4daca58..102410d 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -236,7 +236,7 @@ def test_fetch_source_returns_response_and_body( assert options["follow_redirects"] is True assert options["max_redirects"] == app.config.get("MAX_REDIRECTS", 20) assert options["trust_env"] is False - assert options["event_hooks"] == {"request": [receiver.ensure_public_request]} + assert options["transport"].__class__.__name__ == "PublicHTTPTransport" @patch("webmentions_ssg.tasks.receiver.httpx.Client") @@ -680,52 +680,30 @@ def test_verify_webmention_ignores_unknown_identifier( assert f"Cannot verify unknown ReceivedWebmention {identifier}" in caplog.text -@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=True) -def test_ensure_public_request_accepts_public_url( - is_public_url: Mock, receiver: ModuleType +@patch("webmentions_ssg.tasks.receiver.httpx.Client") +def test_fetch_source_rejects_unsafe_destination( + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - is_public_url.assert_called_once_with(SOURCE_URL) - + from webmentions_ssg.url_security import UnsafeAddressError -@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=False) -def test_ensure_public_request_rejects_non_public_address( - is_public_url: Mock, receiver: ModuleType -) -> None: - with pytest.raises( - receiver.VerificationError, match="Source resolves to a non-public address" + client = httpx_client.return_value.__enter__.return_value + client.stream.side_effect = UnsafeAddressError("Local address") + with app.app_context(), pytest.raises( + receiver.VerificationError, match="non-public address" ): - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - is_public_url.assert_called_once_with(SOURCE_URL) + receiver.fetch_source(SOURCE_URL) -@patch( - "webmentions_ssg.tasks.receiver.is_public_url", - side_effect=AddressResolutionError("Could not resolve hostname"), -) -def test_ensure_public_request_maps_dns_failure_to_temporary_error( - is_public_url: Mock, receiver: ModuleType +@patch("webmentions_ssg.tasks.receiver.httpx.Client") +def test_fetch_source_treats_dns_failure_as_temporary( + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: - with pytest.raises( - receiver.TemporaryFetchError, match="Source hostname could not be resolved" + client = httpx_client.return_value.__enter__.return_value + client.stream.side_effect = AddressResolutionError("DNS timeout") + with app.app_context(), pytest.raises( + receiver.TemporaryFetchError, match="could not be resolved" ): - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - is_public_url.assert_called_once_with(SOURCE_URL) - - -@patch( - "webmentions_ssg.tasks.receiver.is_public_url", - side_effect=ValueError("No hostname was specified"), -) -def test_ensure_public_request_rejects_url_without_hostname( - is_public_url: Mock, receiver: ModuleType -) -> None: - with pytest.raises(receiver.VerificationError, match="Source URL has no hostname"): - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - is_public_url.assert_called_once_with(SOURCE_URL) + receiver.fetch_source(SOURCE_URL) @patch("webmentions_ssg.tasks.receiver.BeautifulSoup") diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py index 0db0d8c..38bae11 100644 --- a/tests/tasks/test_sender.py +++ b/tests/tasks/test_sender.py @@ -1214,43 +1214,6 @@ def test_send_webmention_ignores_unknown_identifier( assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text -def test_ensure_public_request_accepts_public_url( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - is_public_url = Mock(return_value=True) - monkeypatch.setattr(sender_module, "is_public_url", is_public_url) - sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) - is_public_url.assert_called_once_with(TARGET_URL) - - -def test_ensure_public_request_rejects_non_public_address( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - is_public_url = Mock(return_value=False) - monkeypatch.setattr(sender_module, "is_public_url", is_public_url) - with pytest.raises( - sender_module.PermanentSenderError, - match="Request resolves to a non-public address", - ): - sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) - is_public_url.assert_called_once_with(TARGET_URL) - - -def test_ensure_public_request_maps_dns_failure_to_temporary_error( - sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch -) -> None: - is_public_url = Mock( - side_effect=AddressResolutionError("Could not resolve hostname") - ) - monkeypatch.setattr(sender_module, "is_public_url", is_public_url) - with pytest.raises( - sender_module.TemporarySenderError, - match="Request hostname could not be resolved", - ): - sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL)) - is_public_url.assert_called_once_with(TARGET_URL) - - def test_resolve_endpoint_rejects_invalid_url( sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_config.py b/tests/test_config.py index 57ac423..1b45946 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -72,6 +72,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None: assert config.MAX_REDIRECTS == 20 assert config.MAX_SOURCE_BYTES == 1_000_000 assert config.REQUEST_TIMEOUT == 5.0 + assert config.REQUEST_TOTAL_TIMEOUT == 30.0 assert DefaultConfig.ALLOWED_HOSTNAMES == set() diff --git a/tests/test_public_transport.py b/tests/test_public_transport.py new file mode 100644 index 0000000..e7d77fa --- /dev/null +++ b/tests/test_public_transport.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Security regression tests for DNS-pinned network connections.""" + +import socket +from unittest.mock import Mock + +import httpcore +import httpx +import pytest + +from webmentions_ssg.public_transport import ( + DeadlineStream, + NetworkDeadline, + PublicHTTPTransport, + PublicNetworkBackend, +) +from webmentions_ssg.url_security import UnsafeAddressError + + +@pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"]) +def test_backend_rejects_private_ip_without_socket( + monkeypatch: pytest.MonkeyPatch, host: str +) -> None: + create_socket = Mock(side_effect=AssertionError("must not open socket")) + monkeypatch.setattr(socket, "socket", create_socket) + with pytest.raises(UnsafeAddressError): + PublicNetworkBackend(NetworkDeadline(5)).connect_tcp(host, 80) + create_socket.assert_not_called() + + +def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None: + """The address is resolved exactly once and the socket sees only that IP.""" + dns = Mock(return_value=("93.184.216.34",)) + monkeypatch.setattr("webmentions_ssg.public_transport.public_addresses", dns) + socket_instance = Mock() + create_socket = Mock(return_value=socket_instance) + monkeypatch.setattr(socket, "socket", create_socket) + monkeypatch.setattr( + socket, "getaddrinfo", Mock(side_effect=AssertionError("second resolution")) + ) + + stream = PublicNetworkBackend(NetworkDeadline(5)).connect_tcp( + "rebinding.example", 443, timeout=2.0 + ) + assert isinstance(stream, DeadlineStream) + assert stream.get_extra_info("socket") is socket_instance + dns.assert_called_once() + assert dns.call_args.args == ("rebinding.example",) + assert 0 < dns.call_args.kwargs["lifetime"] <= 2.0 + socket_instance.connect.assert_called_once_with(("93.184.216.34", 443)) + create_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) + + +class FakeStream(httpcore.NetworkStream): + """In-memory TCP/TLS stream so HTTP framing and SNI can be tested.""" + + def __init__(self, response: bytes) -> None: + self.response = response + self.writes = bytearray() + self.sni: str | None = None + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + part, self.response = self.response[:max_bytes], self.response[max_bytes:] + return part + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self.writes.extend(buffer) + + def start_tls( + self, ssl_context, server_hostname: str | None = None, timeout: float | None = None + ) -> httpcore.NetworkStream: + self.sni = server_hostname + return self + + def get_extra_info(self, info: str): + return None + + def close(self) -> None: + pass + + +@pytest.mark.parametrize("scheme", ["http", "https"]) +def test_transport_keeps_original_http_host_and_tls_sni( + monkeypatch: pytest.MonkeyPatch, scheme: str +) -> None: + stream = FakeStream(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK") + backend = Mock(return_value=stream) + monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", backend) + + with httpx.Client(transport=PublicHTTPTransport(5.0), trust_env=False) as client: + response = client.get(f"{scheme}://source.example/path") + + assert response.status_code == 200 + assert response.text == "OK" + assert b"Host: source.example\r\n" in stream.writes + assert b"GET /path HTTP/1.1\r\n" in stream.writes + assert stream.sni == ("source.example" if scheme == "https" else None) + backend.assert_called_once() + assert backend.call_args.kwargs["host"] == "source.example" + assert backend.call_args.kwargs["port"] == (443 if scheme == "https" else 80) + + +def test_deadline_clips_each_read_and_stops_trickle( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = [100.0] + monkeypatch.setattr("webmentions_ssg.public_transport.time.monotonic", lambda: clock[0]) + fake_stream = Mock() + fake_stream.read.return_value = b"x" + stream = DeadlineStream(fake_stream, NetworkDeadline(5.0)) + + clock[0] = 103.0 + assert stream.read(1, timeout=5.0) == b"x" + assert fake_stream.read.call_args.kwargs["timeout"] == pytest.approx(2.0) + + clock[0] = 105.1 + with pytest.raises(httpcore.ReadTimeout, match="Overall HTTP request deadline"): + stream.read(1, timeout=5.0) + assert fake_stream.read.call_count == 1 + + +def test_redirects_go_through_same_destination_guard( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redirect = FakeStream( + b"HTTP/1.1 302 Found\r\nLocation: http://internal.example/\r\n" + b"Content-Length: 0\r\n\r\n" + ) + checked_hosts: list[str] = [] + + def connect( + self: PublicNetworkBackend, host: str, port: int, **kwargs: object + ) -> httpcore.NetworkStream: + checked_hosts.append(host) + if host == "internal.example": + raise UnsafeAddressError("Non-public network address") + return redirect + + monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", connect) + with ( + httpx.Client(transport=PublicHTTPTransport(5.0), follow_redirects=True) as client, + pytest.raises(UnsafeAddressError, match="Non-public"), + ): + client.get("http://source.example/post") + + assert checked_hosts == ["source.example", "internal.example"] diff --git a/tests/test_url_security.py b/tests/test_url_security.py index 80b45e9..d25589f 100644 --- a/tests/test_url_security.py +++ b/tests/test_url_security.py @@ -9,6 +9,8 @@ import pytest from webmentions_ssg.url_security import ( AddressResolutionError, + UnsafeAddressError, + public_addresses, is_http_url, is_public_url, is_safe_next_url, @@ -114,7 +116,7 @@ def test_is_public_url_returns_true_for_public_addresses( def test_is_public_url_raises_for_resolution_failure(resolve_name: Mock) -> None: with pytest.raises(AddressResolutionError, match="Could not resolve hostname"): is_public_url("https://nonexistent.example/") - resolve_name.assert_called_once_with("nonexistent.example") + resolve_name.assert_called_once_with("nonexistent.example", lifetime=None) @patch("webmentions_ssg.url_security.dns.resolver.resolve_name") @@ -122,3 +124,28 @@ def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> No with pytest.raises(ValueError, match="No hostname was specified"): is_public_url("/relative/url") resolve_name.assert_not_called() + + +def test_ip_literal_does_not_use_dns() -> None: + with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as dns_resolve: + with pytest.raises(UnsafeAddressError): + public_addresses("169.254.169.254") + with pytest.raises(UnsafeAddressError): + public_addresses("::1") + assert public_addresses("8.8.8.8") == ("8.8.8.8",) + dns_resolve.assert_not_called() + + +def test_public_addresses_rejects_mixed_dns_results() -> None: + with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: + resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"] + with pytest.raises(UnsafeAddressError, match="Non-public"): + public_addresses("rebinding.example", lifetime=1.5) + resolve.assert_called_once_with("rebinding.example", lifetime=1.5) + + +def test_public_addresses_rejects_empty_dns_result() -> None: + with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve: + resolve.return_value.addresses.return_value = [] + with pytest.raises(AddressResolutionError): + public_addresses("empty.example") @@ -902,6 +902,7 @@ dependencies = [ { name = "flask-migrate" }, { name = "flask-sqlalchemy" }, { name = "flask-wtf" }, + { name = "httpcore" }, { name = "httpx" }, { name = "huey" }, { name = "mf2py" }, @@ -934,7 +935,8 @@ requires-dist = [ { name = "flask-migrate", specifier = ">=4.1.0" }, { name = "flask-sqlalchemy", specifier = ">=3.1.1" }, { name = "flask-wtf", specifier = ">=1.3.0" }, - { name = "httpx", specifier = ">=0.28.1" }, + { name = "httpcore", specifier = ">=1.0.9,<1.1" }, + { name = "httpx", specifier = ">=0.28.1,<0.29" }, { name = "huey", specifier = ">=3.1.0" }, { name = "mf2py", specifier = ">=2.0.1" }, { name = "rfc3987", specifier = ">=1.3.8" }, diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index e8288db..1c7d44e 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -45,6 +45,8 @@ class DefaultConfig: MAX_REDIRECTS = 20 MAX_SOURCE_BYTES = 1_000_000 REQUEST_TIMEOUT = 5.0 + # Absolute wall-clock budget for each verification or delivery attempt. + REQUEST_TOTAL_TIMEOUT = 30.0 # Requeue abandoned verification attempts after 15 minutes. VERIFICATION_TIMEOUT_SECONDS = 900 VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *" diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py index 2e3c72c..f698026 100644 --- a/webmentions_ssg/forms/__init__.py +++ b/webmentions_ssg/forms/__init__.py @@ -10,7 +10,7 @@ from flask_wtf import FlaskForm from wtforms import PasswordField, StringField, SubmitField from wtforms.validators import URL, InputRequired, Regexp -from .validators import AllowedHostname, NotEqualTo, PublicURL +from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral class LoginForm(FlaskForm): @@ -42,7 +42,7 @@ class EndpointForm(FlaskForm): message="source must begin with http or https", ), NotEqualTo("target"), - PublicURL(), + PublicIPLiteral(), ], ) target = StringField( diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index dd0ee9b..e8f33eb 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -4,6 +4,7 @@ """Provide validators for URL safety and form field constraints.""" +import ipaddress from urllib.parse import urlsplit from flask import current_app @@ -85,7 +86,10 @@ class AllowedHostname: :param field: Field containing the URL to validate. :raises ValidationError: If the URL hostname is not configured as allowed. """ - hostname = urlsplit(field.data).hostname + try: + hostname = urlsplit(field.data).hostname + except (ValueError, TypeError): + hostname = None if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]: return raise ValidationError(self.message or field.gettext("Invalid input.")) @@ -125,3 +129,24 @@ class PublicURL: raise ValidationError(message) except (AddressResolutionError, ValueError) as exc: raise ValidationError(message) from exc + + +class PublicIPLiteral: + """Reject non-public IP literals without performing DNS in the web worker.""" + + def __call__(self, form: BaseForm, field: Field) -> None: + try: + hostname = urlsplit(field.data).hostname + except (ValueError, TypeError) as exc: + raise ValidationError(field.gettext("Invalid input.")) from exc + + if hostname is None: + raise ValidationError(field.gettext("Invalid input.")) + + try: + address = ipaddress.ip_address(hostname) + except ValueError: + return # Named hosts are resolved only during the background fetch. + + if not address.is_global: + raise ValidationError(field.gettext("URL must use a public address.")) diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/public_transport.py new file mode 100644 index 0000000..131cf55 --- /dev/null +++ b/webmentions_ssg/public_transport.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""HTTP transport that connects only to validated DNS answers. + +httpcore receives the original host for Host and TLS SNI/certificate checks, +but the socket is opened to an IP from a single, bounded DNS lookup. This +protects the receiver and sender, including redirects and advertised URLs. +""" + +import ipaddress +import socket +import time +from collections.abc import Iterable + +import httpcore +import httpx +from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation + +from .url_security import public_addresses + + +class NetworkDeadline: + """One monotonic deadline shared by DNS, TCP, TLS, and response streaming.""" + + def __init__(self, seconds: float) -> None: + if seconds <= 0: + raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero") + self.expires = time.monotonic() + seconds + + def timeout(self, original: float | None, error: type[Exception]) -> float: + remaining = self.expires - time.monotonic() + if remaining <= 0: + raise error("Overall HTTP request deadline exceeded") + return min(remaining, original) if original is not None else remaining + + +class DeadlineStream(httpcore.NetworkStream): + """Clip every read/write/TLS operation to the remaining overall deadline.""" + + def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline): + self._stream = stream + self._deadline = deadline + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + return self._stream.read( + max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout) + ) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self._stream.write( + buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout) + ) + + def start_tls( + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> httpcore.NetworkStream: + return DeadlineStream( + self._stream.start_tls( + ssl_context, + server_hostname=server_hostname, + timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout), + ), + self._deadline, + ) + + def get_extra_info(self, info: str): + return self._stream.get_extra_info(info) + + def close(self) -> None: + self._stream.close() + + +class PublicNetworkBackend(httpcore.SyncBackend): + """Resolve and validate in the TCP connection operation, never in a hook.""" + + def __init__(self, deadline: NetworkDeadline): + self._deadline = deadline + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[tuple] | None = None, + ) -> httpcore.NetworkStream: + # No second lookup of `host` is permitted. Even a DNS response that + # changes immediately after this check cannot change the socket peer. + dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout) + addresses = public_addresses(host, lifetime=dns_timeout) + last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None + + for address in addresses: + family = ( + socket.AF_INET6 + if ipaddress.ip_address(address).version == 6 + else socket.AF_INET + ) + sock = socket.socket(family, socket.SOCK_STREAM) + try: + sock.settimeout( + self._deadline.timeout(timeout, httpcore.ConnectTimeout) + ) + if local_address is not None: + sock.bind((local_address, 0)) + for option in socket_options or (): + sock.setsockopt(*option) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # This numeric address is the *same* address just validated; + # no getaddrinfo or DNS resolver runs between check and connect. + sock.connect((address, port)) + return DeadlineStream(SyncStream(sock), self._deadline) + except (TimeoutError, OSError) as exc: + sock.close() + if isinstance(exc, TimeoutError): + last_error = httpcore.ConnectTimeout(str(exc)) + else: + last_error = httpcore.ConnectError(str(exc)) + except BaseException: + sock.close() + raise + + assert last_error is not None + raise last_error + + +class PublicHTTPTransport(httpx.HTTPTransport): + """A direct (non-proxy) httpx transport with an IP-pinning network backend. + + httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we + initialize its compatible pool directly. Pin the httpx/httpcore major APIs + and keep this integration under test when upgrading either dependency. + """ + + def __init__(self, total_timeout: float): + self._pool = httpcore.ConnectionPool( + ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)), + ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 4e8df74..f999d1b 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -18,7 +18,8 @@ from .. import APP_NAME, VERSION from .. import DATABASE as db from .. import HUEY as huey from ..models import ReceivedWebmention -from ..url_security import AddressResolutionError, is_public_url +from ..public_transport import PublicHTTPTransport +from ..url_security import AddressResolutionError, UnsafeAddressError class VerificationError(Exception): @@ -124,24 +125,6 @@ def text_mentions_target(body: str, target_url: str) -> bool: return False -def ensure_public_request(request: httpx.Request) -> None: - """ - Ensure that an HTTP request targets a public network address. - - :param request: HTTP request to validate. - :raises VerificationError: If the URL has no hostname or resolves to a - non-public address. - :raises TemporaryFetchError: If the hostname cannot be resolved. - """ - try: - if not is_public_url(str(request.url)): - raise VerificationError("Source resolves to a non-public address") - except ValueError as exc: - raise VerificationError("Source URL has no hostname") from exc - except AddressResolutionError as exc: - raise TemporaryFetchError("Source hostname could not be resolved") from exc - - def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: """ Fetch a webmention source document. @@ -157,6 +140,15 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: status or exceeds the configured maximum response size. """ + try: + return _fetch_source(source_url) + except UnsafeAddressError as exc: + raise VerificationError("Source resolves to a non-public address") from exc + except AddressResolutionError as exc: + raise TemporaryFetchError("Source hostname could not be resolved") from exc + + +def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: with ( httpx.Client( headers={ @@ -167,7 +159,9 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: follow_redirects=True, max_redirects=current_app.config.get("MAX_REDIRECTS", 20), trust_env=False, - event_hooks={"request": [ensure_public_request]}, + transport=PublicHTTPTransport( + current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) + ), ) as client, client.stream("GET", source_url) as response, ): diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index 725b5c8..7121f6a 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -19,7 +19,8 @@ from .. import APP_NAME, VERSION from .. import DATABASE as db from .. import HUEY as huey from ..models import SentWebmention, SentWebmentionStatus -from ..url_security import AddressResolutionError, is_http_url, is_public_url +from ..public_transport import PublicHTTPTransport +from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url LINK_SPLIT = re.compile(r",\s*(?=<)") @@ -48,21 +49,6 @@ def temporary_http_status(status: int) -> bool: return status in {408, 425, 429} or 500 <= status <= 599 -def ensure_public_request(request: httpx.Request) -> None: - """ - Ensure that an HTTP request targets a public network address. - - :param request: HTTP request to validate. - :raises PermanentSenderError: If the request resolves to a non-public address. - :raises TemporarySenderError: If the request hostname cannot be resolved. - """ - try: - if not is_public_url(str(request.url)): - raise PermanentSenderError("Request resolves to a non-public address") - except AddressResolutionError as exc: - raise TemporarySenderError("Request hostname could not be resolved") from exc - - def resolve_endpoint(response: httpx.Response, href: str) -> str: """ Resolve and validate a discovered Webmention endpoint. @@ -224,7 +210,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non """ try: head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError, TemporarySenderError: + except (httpx.RequestError, TemporarySenderError): # HEAD is only an optimization. Servers and intermediaries may reject # or drop it even when fetching the document with GET would succeed. pass @@ -442,7 +428,9 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: follow_redirects=True, max_redirects=current_app.config.get("MAX_REDIRECTS", 20), trust_env=False, - event_hooks={"request": [ensure_public_request]}, + transport=PublicHTTPTransport( + current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0) + ), ) as client: endpoint = discover_webmention_endpoint(client, target) @@ -472,7 +460,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: raise PermanentSenderError(message) - except (TemporarySenderError, httpx.RequestError) as exc: + except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: if not attempt_is_current(webmention, revision): return @@ -495,7 +483,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None: # No Huey retries: only the scanner may requeue after next_retry_at. return - except PermanentSenderError as exc: + except (PermanentSenderError, UnsafeAddressError) as exc: if not attempt_is_current(webmention, revision): return diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py index ade64d6..f943c7e 100644 --- a/webmentions_ssg/url_security.py +++ b/webmentions_ssg/url_security.py @@ -12,17 +12,51 @@ import dns.resolver class AddressResolutionError(Exception): - pass + """A hostname could not be resolved (potentially transient).""" -def is_public_url(url: str) -> bool: +class UnsafeAddressError(ValueError): + """A hostname or IP literal identifies a non-public network address.""" + + +def public_addresses(hostname: str, *, lifetime: float | None = None) -> tuple[str, ...]: + """Resolve *once* and return only checked, globally routable IP addresses. + + A DNS name with a mixture of public and private answers is rejected in its + entirety. IP literals are handled without DNS, including IPv6 literals. + The returned strings must be used for the actual TCP connection; checking + them and then connecting to ``hostname`` reintroduces DNS rebinding. """ - Check whether a URL resolves exclusively to public IP addresses. + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + + if literal is not None: + addresses = (literal,) + else: + try: + answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) + addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) + except (dns.exception.DNSException, ValueError) as exc: + raise AddressResolutionError( + f"Could not resolve hostname {hostname!r}" + ) from exc + + if not addresses: + raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") - :param url: URL whose hostname should be resolved. - :return: Whether all resolved addresses are globally routable. - :raises ValueError: If the URL does not contain a hostname. - :raises AddressResolutionError: If the hostname cannot be resolved. + if any(not address.is_global for address in addresses): + raise UnsafeAddressError(f"Non-public network address for {hostname!r}") + + return tuple(str(address) for address in addresses) + + +def is_public_url(url: str) -> bool: + """Return whether the URL resolves entirely to public IP addresses. + + This is useful for diagnostics, but **not** sufficient to authorize a + request: network connections must use the exact validated IP address. """ hostname = urlsplit(url).hostname @@ -30,17 +64,10 @@ def is_public_url(url: str) -> bool: raise ValueError("No hostname was specified") try: - answers = dns.resolver.resolve_name(hostname) - except dns.exception.DNSException as exc: - raise AddressResolutionError( - f"Could not resolve hostname {hostname!r}" - ) from exc - else: - for address in answers.addresses(): - if not ipaddress.ip_address(address).is_global: - return False - - return True + public_addresses(hostname) + except UnsafeAddressError: + return False + return True def is_http_url(url: str) -> bool: |
