1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
|
# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
#
# SPDX-License-Identifier: BSD-3-Clause
"""Public-only HTTP transport, URL checks, and bounded response reads."""
import ipaddress
import socket
import time
from collections.abc import Iterable
from urllib.parse import urlsplit
import dns.exception
import dns.resolver
import httpcore
import httpx
from flask import current_app
from httpcore._backends.sync import SyncStream
NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
class AddressResolutionError(Exception):
"""A hostname could not be resolved (potentially transient)."""
class UnsafeAddressError(ValueError):
"""A hostname or IP literal identifies a non-public network address."""
class NetworkDeadline:
"""One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""
def __init__(self, seconds: float) -> None:
if seconds <= 0:
raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero")
self.expires = time.monotonic() + seconds
def timeout(self, original: float | None, error: type[Exception]) -> float:
remaining = self.expires - time.monotonic()
if remaining <= 0:
raise error("Overall HTTP request deadline exceeded")
return min(remaining, original) if original is not None else remaining
class DeadlineStream(httpcore.NetworkStream):
"""Clip every read/write/TLS operation to the remaining overall deadline."""
def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline):
self._stream = stream
self._deadline = deadline
def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
return self._stream.read(
max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout)
)
def write(self, buffer: bytes, timeout: float | None = None) -> None:
self._stream.write(
buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout)
)
def start_tls(
self,
ssl_context,
server_hostname: str | None = None,
timeout: float | None = None,
) -> httpcore.NetworkStream:
return DeadlineStream(
self._stream.start_tls(
ssl_context,
server_hostname=server_hostname,
timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout),
),
self._deadline,
)
def get_extra_info(self, info: str):
return self._stream.get_extra_info(info)
def close(self) -> None:
self._stream.close()
class PublicNetworkBackend(httpcore.SyncBackend):
"""Resolve and validate in the TCP connection operation, never in a hook."""
def __init__(self, deadline: NetworkDeadline):
self._deadline = deadline
def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: Iterable[tuple] | None = None,
) -> httpcore.NetworkStream:
# No second lookup of `host` is permitted. Even a DNS response that
# changes immediately after this check cannot change the socket peer.
dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout)
addresses = public_addresses(host, lifetime=dns_timeout)
last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
for address in addresses:
family = (
socket.AF_INET6
if ipaddress.ip_address(address).version == 6
else socket.AF_INET
)
sock = socket.socket(family, socket.SOCK_STREAM)
try:
sock.settimeout(
self._deadline.timeout(timeout, httpcore.ConnectTimeout)
)
if local_address is not None:
sock.bind((local_address, 0))
for option in socket_options or ():
sock.setsockopt(*option)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# This numeric address is the *same* address just validated;
# no getaddrinfo or DNS resolver runs between check and connect.
sock.connect((address, port))
return DeadlineStream(SyncStream(sock), self._deadline)
except (TimeoutError, OSError) as exc:
sock.close()
if isinstance(exc, TimeoutError):
last_error = httpcore.ConnectTimeout(str(exc))
else:
last_error = httpcore.ConnectError(str(exc))
except BaseException:
sock.close()
raise
assert last_error is not None
raise last_error
class PublicHTTPTransport(httpx.HTTPTransport):
"""A direct (non-proxy) httpx transport with an IP-pinning network backend.
httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we
initialize its compatible pool directly. Pin the httpx/httpcore major APIs
and keep this integration under test when upgrading either dependency.
"""
def __init__(self, total_timeout: float):
self._pool = httpcore.ConnectionPool(
ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
)
def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
"""Return the hardened HTTP client shared by receiver and sender."""
headers = {"User-Agent": user_agent}
if accept is not None:
headers["Accept"] = accept
return httpx.Client(
headers=headers,
timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]),
follow_redirects=True,
max_redirects=current_app.config["MAX_REDIRECTS"],
trust_env=False,
transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]),
)
def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
"""Reject non-public addresses, including IPv4 hidden in IPv6 forms.
``is_global`` alone does not reliably exclude IPv6 transition addresses
across Python versions or networks with NAT64/6to4 routing.
"""
if isinstance(address, ipaddress.IPv6Address):
# The well-known NAT64 prefix can translate an otherwise global IPv6
# address into a private IPv4 destination. IPv4-compatible addresses
# similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
return False
# 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
# destination through ipaddress; validate that destination as well.
for embedded in (address.sixtofour, address.ipv4_mapped):
if embedded is not None and not embedded.is_global:
return False
return address.is_global
def resolve_hostname(
hostname: str, *, lifetime: float | None = None
) -> tuple[ipaddress.IPv4Address | ipaddress.IPv6Address, ...]:
try:
answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
return tuple(ipaddress.ip_address(a) for a in answers.addresses())
except (dns.exception.DNSException, ValueError) as exc:
raise AddressResolutionError(
f"Could not resolve hostname {hostname!r}"
) from exc
def public_addresses(
hostname: str, *, lifetime: float | None = None
) -> tuple[str, ...]:
"""Resolve *once* and return only checked, globally routable IP addresses.
A DNS name with a mixture of public and private answers is rejected in its
entirety. IP literals are handled without DNS, including IPv6 literals.
The returned strings must be used for the actual TCP connection; checking
them and then connecting to ``hostname`` reintroduces DNS rebinding.
"""
try:
addresses = (ipaddress.ip_address(hostname),)
except ValueError:
addresses = resolve_hostname(hostname, lifetime=lifetime)
if not addresses:
raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
if any(not is_public_address(address) for address in addresses):
raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
return tuple(str(address) for address in addresses)
def is_http_url(url: str) -> bool:
"""Check whether a URL is an absolute HTTP or HTTPS URL.
:param url: URL to validate.
:return: Whether the URL uses HTTP or HTTPS and contains a hostname.
"""
try:
parsed = urlsplit(url)
except ValueError:
return False
return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
"""Read a streaming response, rejecting oversized declared or actual bodies."""
if (content_length := response.headers.get("Content-Length")) is not None:
try:
declared_length = int(content_length)
except ValueError:
pass
else:
if declared_length > max_bytes:
raise ValueError(f"{label} document is too large")
body = bytearray()
for chunk in response.iter_bytes(chunk_size=64 * 1024):
body.extend(chunk)
if len(body) > max_bytes:
raise ValueError(f"{label} document is too large")
return bytes(body)
def is_temporary_status(status: int) -> bool:
"""Return whether an HTTP status is normally retryable."""
return status in {408, 425, 429} or 500 <= status <= 599
|