1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
|
# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
#
# SPDX-License-Identifier: BSD-3-Clause
"""Public-only HTTP transport, URL checks, and bounded response reads."""
import ipaddress
import socket
import time
from collections.abc import Iterable, Mapping
from contextlib import ExitStack
from http import HTTPStatus
from urllib.parse import urlsplit
import dns.exception
import dns.resolver
import httpcore
import httpx
from flask import current_app
from httpcore._backends.sync import SyncStream
NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
NETWORK_CONNECT_TIMEOUT_ERROR = 599
class AddressResolutionError(Exception):
"""A hostname could not be resolved (potentially transient)."""
class UnsafeAddressError(ValueError):
"""A hostname or IP literal identifies a non-public network address."""
class ResponseTooLargeError(ValueError):
"""A response body exceeds its configured size limit."""
class RequestDeadline:
"""One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""
def __init__(self, seconds: float) -> None:
if seconds <= 0:
raise ValueError("REQUEST_TOTAL_TIMEOUT_SECONDS must be greater than zero")
self.expires_at = time.monotonic() + seconds
def remaining_timeout(
self, original: float | None, error: type[Exception]
) -> float:
remaining = self.expires_at - time.monotonic()
if remaining <= 0:
raise error("Overall HTTP request deadline exceeded")
return min(remaining, original) if original is not None else remaining
class DeadlineStream(httpcore.NetworkStream):
"""Clip every read/write/TLS operation to the remaining overall deadline."""
def __init__(self, stream: httpcore.NetworkStream, deadline: RequestDeadline):
self._stream = stream
self._deadline = deadline
def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
return self._stream.read(
max_bytes,
timeout=self._deadline.remaining_timeout(timeout, httpcore.ReadTimeout),
)
def write(self, buffer: bytes, timeout: float | None = None) -> None:
self._stream.write(
buffer,
timeout=self._deadline.remaining_timeout(timeout, httpcore.WriteTimeout),
)
def start_tls(
self,
ssl_context,
server_hostname: str | None = None,
timeout: float | None = None,
) -> httpcore.NetworkStream:
return DeadlineStream(
self._stream.start_tls(
ssl_context,
server_hostname=server_hostname,
timeout=self._deadline.remaining_timeout(
timeout, httpcore.ConnectTimeout
),
),
self._deadline,
)
def get_extra_info(self, info: str):
return self._stream.get_extra_info(info)
def close(self) -> None:
self._stream.close()
class PublicNetworkBackend(httpcore.SyncBackend):
"""Resolve and validate in the TCP connection operation, never in a hook."""
def __init__(self, deadline: RequestDeadline):
self._deadline = deadline
def _connect_to_address(
self,
address: str,
port: int,
timeout: float | None,
local_address: str | None,
socket_options: Iterable[tuple] | None,
) -> httpcore.NetworkStream:
family = (
socket.AF_INET6
if ipaddress.ip_address(address).version == 6
else socket.AF_INET
)
# Close the socket on any error, transferring ownership to SyncStream
# only after the connection and stream have been created successfully.
with ExitStack() as cleanup:
sock = socket.socket(family, socket.SOCK_STREAM)
cleanup.callback(sock.close)
sock.settimeout(
self._deadline.remaining_timeout(timeout, httpcore.ConnectTimeout)
)
if local_address is not None:
sock.bind((local_address, 0))
for option in socket_options or ():
sock.setsockopt(*option)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# This numeric address is the *same* address just validated;
# no getaddrinfo or DNS resolver runs between check and connect.
sock.connect((address, port))
stream = DeadlineStream(SyncStream(sock), self._deadline)
cleanup.pop_all()
return stream
def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: Iterable[tuple] | None = None,
) -> httpcore.NetworkStream:
# No second lookup of `host` is permitted. Even a DNS response that
# changes immediately after this check cannot change the socket peer.
dns_timeout = self._deadline.remaining_timeout(timeout, httpcore.ConnectTimeout)
addresses = resolve_public_addresses(host, lifetime=dns_timeout)
last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
for address in addresses:
try:
return self._connect_to_address(
address, port, timeout, local_address, socket_options
)
except OSError as exc:
if isinstance(exc, TimeoutError):
last_error = httpcore.ConnectTimeout(str(exc))
else:
last_error = httpcore.ConnectError(str(exc))
if last_error is None:
raise httpcore.ConnectError(f"No public addresses available for {host!r}")
raise last_error
class PublicHTTPTransport(httpx.HTTPTransport):
"""A direct (non-proxy) httpx transport with an IP-pinning network backend.
httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we
initialize its compatible pool directly. Pin the httpx/httpcore major APIs
and keep this integration under test when upgrading either dependency.
"""
def __init__(self, total_timeout: float):
self._pool = httpcore.ConnectionPool(
ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
network_backend=PublicNetworkBackend(RequestDeadline(total_timeout)),
)
def build_public_http_client(
user_agent: str, *, accept: str | None = None
) -> httpx.Client:
"""Return the hardened HTTP client shared by receiver and sender.
:param user_agent: User-Agent header sent with each request.
:param accept: Optional Accept header for the client.
:returns: HTTP client configured with public-only transport, deadlines, and bounded
redirects.
"""
headers = {"User-Agent": user_agent}
if accept is not None:
headers["Accept"] = accept
return httpx.Client(
headers=headers,
timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT_SECONDS"]),
follow_redirects=True,
max_redirects=current_app.config["MAX_REDIRECTS"],
trust_env=False,
transport=PublicHTTPTransport(
current_app.config["REQUEST_TOTAL_TIMEOUT_SECONDS"]
),
)
def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
"""Reject non-public addresses, including IPv4 hidden in IPv6 forms.
``is_global`` alone does not reliably exclude IPv6 transition addresses
across Python versions or networks with NAT64/6to4 routing.
:param address: IPv4 or IPv6 address to classify.
:returns: ``True`` if the address is globally routable and safe to connect to.
"""
if isinstance(address, ipaddress.IPv6Address):
# The well-known NAT64 prefix can translate an otherwise global IPv6
# address into a private IPv4 destination. IPv4-compatible addresses
# similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
return False
# 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
# destination through ipaddress; validate that destination as well.
for embedded in (address.sixtofour, address.ipv4_mapped):
if embedded is not None and not embedded.is_global:
return False
return address.is_global
def resolve_hostname(
hostname: str, *, lifetime: float | None = None
) -> tuple[ipaddress.IPv4Address | ipaddress.IPv6Address, ...]:
try:
answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
return tuple(ipaddress.ip_address(a) for a in answers.addresses())
except (dns.exception.DNSException, ValueError) as exc:
raise AddressResolutionError(
f"Could not resolve hostname {hostname!r}"
) from exc
def resolve_public_addresses(
hostname: str, *, lifetime: float | None = None
) -> tuple[str, ...]:
"""Resolve *once* and return only checked, globally routable IP addresses.
A DNS name with a mixture of public and private answers is rejected in its
entirety. IP literals are handled without DNS, including IPv6 literals.
The returned strings must be used for the actual TCP connection; checking
them and then connecting to ``hostname`` reintroduces DNS rebinding.
:param hostname: Hostname or IP literal to validate.
:param lifetime: Optional maximum time allotted for DNS resolution, in seconds.
:returns: Checked numeric IP addresses to use for the TCP connection.
:raises AddressResolutionError: If DNS resolution fails or returns no addresses.
:raises UnsafeAddressError: If any resolved address is not publicly routable.
"""
try:
addresses = (ipaddress.ip_address(hostname),)
except ValueError:
addresses = resolve_hostname(hostname, lifetime=lifetime)
if not addresses:
raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
if any(not is_public_address(address) for address in addresses):
raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
return tuple(str(address) for address in addresses)
def normalized_hostname(url: str) -> str | None:
"""Return a normalized URL hostname, or None for malformed URLs.
:param url: URL from which to extract a hostname.
:returns: Normalized hostname, or ``None`` when the URL is invalid or has no host.
"""
try:
hostname = urlsplit(url).hostname
except TypeError, ValueError:
return None
return hostname.rstrip(".") if hostname is not None else None
def media_type_of(headers: Mapping[str, str]) -> str:
"""Return a normalized media type without Content-Type parameters.
:param headers: HTTP response headers to inspect.
:returns: Lowercase media type without parameters, or an empty string if absent.
"""
return headers.get("Content-Type", "").partition(";")[0].strip().lower()
def is_http_url(url: str) -> bool:
"""Check whether a URL is an absolute HTTP or HTTPS URL.
:param url: URL to validate.
:returns: Whether the URL uses HTTP or HTTPS and contains a hostname.
"""
try:
parsed = urlsplit(url)
except ValueError:
return False
return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
"""Read a streaming response, rejecting oversized declared or actual bodies.
:param response: Streaming HTTP response to consume.
:param max_bytes: Maximum permitted response body size in bytes.
:param label: Description of the response for size-limit errors.
:returns: Response body if it fits within the configured byte limit.
:raises ResponseTooLargeError: If declared or streamed content exceeds the byte
limit.
"""
if (content_length := response.headers.get("Content-Length")) is not None:
try:
declared_length = int(content_length)
except ValueError:
pass
else:
if declared_length > max_bytes:
raise ResponseTooLargeError(f"{label} document is too large")
body = bytearray()
for chunk in response.iter_bytes(chunk_size=64 * 1024):
body.extend(chunk)
if len(body) > max_bytes:
raise ResponseTooLargeError(f"{label} document is too large")
return bytes(body)
def is_retryable_status(status: int) -> bool:
"""Return whether an HTTP status is normally retryable.
:param status: HTTP response status code to classify.
:returns: ``True`` if the HTTP status indicates a retryable failure.
"""
return (
status
in {
HTTPStatus.REQUEST_TIMEOUT,
HTTPStatus.TOO_EARLY,
HTTPStatus.TOO_MANY_REQUESTS,
}
or HTTPStatus.INTERNAL_SERVER_ERROR <= status <= NETWORK_CONNECT_TIMEOUT_ERROR
)
|