# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause """Public-only HTTP transport, URL checks, and bounded response reads.""" import ipaddress import socket import time from collections.abc import Iterable from contextlib import ExitStack from urllib.parse import urlsplit import dns.exception import dns.resolver import httpcore import httpx from flask import current_app from httpcore._backends.sync import SyncStream NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") class AddressResolutionError(Exception): """A hostname could not be resolved (potentially transient).""" class UnsafeAddressError(ValueError): """A hostname or IP literal identifies a non-public network address.""" class ResponseTooLargeError(ValueError): """A response body exceeds its configured size limit.""" class NetworkDeadline: """One monotonic deadline shared by DNS, TCP, TLS, and response streaming.""" def __init__(self, seconds: float) -> None: if seconds <= 0: raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero") self.expires = time.monotonic() + seconds def timeout(self, original: float | None, error: type[Exception]) -> float: remaining = self.expires - time.monotonic() if remaining <= 0: raise error("Overall HTTP request deadline exceeded") return min(remaining, original) if original is not None else remaining class DeadlineStream(httpcore.NetworkStream): """Clip every read/write/TLS operation to the remaining overall deadline.""" def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline): self._stream = stream self._deadline = deadline def read(self, max_bytes: int, timeout: float | None = None) -> bytes: return self._stream.read( max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout) ) def write(self, buffer: bytes, timeout: float | None = None) -> None: self._stream.write( buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout) ) def start_tls( self, ssl_context, server_hostname: str | None = None, timeout: float | None = None, ) -> httpcore.NetworkStream: return DeadlineStream( self._stream.start_tls( ssl_context, server_hostname=server_hostname, timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout), ), self._deadline, ) def get_extra_info(self, info: str): return self._stream.get_extra_info(info) def close(self) -> None: self._stream.close() class PublicNetworkBackend(httpcore.SyncBackend): """Resolve and validate in the TCP connection operation, never in a hook.""" def __init__(self, deadline: NetworkDeadline): self._deadline = deadline def _connect_one( self, address: str, port: int, timeout: float | None, local_address: str | None, socket_options: Iterable[tuple] | None, ) -> httpcore.NetworkStream: family = ( socket.AF_INET6 if ipaddress.ip_address(address).version == 6 else socket.AF_INET ) # Close the socket on any error, transferring ownership to SyncStream # only after the connection and stream have been created successfully. with ExitStack() as cleanup: sock = socket.socket(family, socket.SOCK_STREAM) cleanup.callback(sock.close) sock.settimeout(self._deadline.timeout(timeout, httpcore.ConnectTimeout)) if local_address is not None: sock.bind((local_address, 0)) for option in socket_options or (): sock.setsockopt(*option) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # This numeric address is the *same* address just validated; # no getaddrinfo or DNS resolver runs between check and connect. sock.connect((address, port)) stream = DeadlineStream(SyncStream(sock), self._deadline) cleanup.pop_all() return stream def connect_tcp( self, host: str, port: int, timeout: float | None = None, local_address: str | None = None, socket_options: Iterable[tuple] | None = None, ) -> httpcore.NetworkStream: # No second lookup of `host` is permitted. Even a DNS response that # changes immediately after this check cannot change the socket peer. dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout) addresses = public_addresses(host, lifetime=dns_timeout) last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None for address in addresses: try: return self._connect_one( address, port, timeout, local_address, socket_options ) except OSError as exc: if isinstance(exc, TimeoutError): last_error = httpcore.ConnectTimeout(str(exc)) else: last_error = httpcore.ConnectError(str(exc)) if last_error is None: raise httpcore.ConnectError(f"No public addresses available for {host!r}") raise last_error class PublicHTTPTransport(httpx.HTTPTransport): """A direct (non-proxy) httpx transport with an IP-pinning network backend. httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we initialize its compatible pool directly. Pin the httpx/httpcore major APIs and keep this integration under test when upgrading either dependency. """ def __init__(self, total_timeout: float): self._pool = httpcore.ConnectionPool( ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)), ) def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client: """Return the hardened HTTP client shared by receiver and sender.""" headers = {"User-Agent": user_agent} if accept is not None: headers["Accept"] = accept return httpx.Client( headers=headers, timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]), follow_redirects=True, max_redirects=current_app.config["MAX_REDIRECTS"], trust_env=False, transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]), ) def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: """Reject non-public addresses, including IPv4 hidden in IPv6 forms. ``is_global`` alone does not reliably exclude IPv6 transition addresses across Python versions or networks with NAT64/6to4 routing. """ if isinstance(address, ipaddress.IPv6Address): # The well-known NAT64 prefix can translate an otherwise global IPv6 # address into a private IPv4 destination. IPv4-compatible addresses # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX: return False # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 # destination through ipaddress; validate that destination as well. for embedded in (address.sixtofour, address.ipv4_mapped): if embedded is not None and not embedded.is_global: return False return address.is_global def resolve_hostname( hostname: str, *, lifetime: float | None = None ) -> tuple[ipaddress.IPv4Address | ipaddress.IPv6Address, ...]: try: answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) return tuple(ipaddress.ip_address(a) for a in answers.addresses()) except (dns.exception.DNSException, ValueError) as exc: raise AddressResolutionError( f"Could not resolve hostname {hostname!r}" ) from exc def public_addresses( hostname: str, *, lifetime: float | None = None ) -> tuple[str, ...]: """Resolve *once* and return only checked, globally routable IP addresses. A DNS name with a mixture of public and private answers is rejected in its entirety. IP literals are handled without DNS, including IPv6 literals. The returned strings must be used for the actual TCP connection; checking them and then connecting to ``hostname`` reintroduces DNS rebinding. """ try: addresses = (ipaddress.ip_address(hostname),) except ValueError: addresses = resolve_hostname(hostname, lifetime=lifetime) if not addresses: raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") if any(not is_public_address(address) for address in addresses): raise UnsafeAddressError(f"Non-public network address for {hostname!r}") return tuple(str(address) for address in addresses) def is_http_url(url: str) -> bool: """Check whether a URL is an absolute HTTP or HTTPS URL. :param url: URL to validate. :return: Whether the URL uses HTTP or HTTPS and contains a hostname. """ try: parsed = urlsplit(url) except ValueError: return False return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes: """Read a streaming response, rejecting oversized declared or actual bodies.""" if (content_length := response.headers.get("Content-Length")) is not None: try: declared_length = int(content_length) except ValueError: pass else: if declared_length > max_bytes: raise ResponseTooLargeError(f"{label} document is too large") body = bytearray() for chunk in response.iter_bytes(chunk_size=64 * 1024): body.extend(chunk) if len(body) > max_bytes: raise ResponseTooLargeError(f"{label} document is too large") return bytes(body) def is_temporary_status(status: int) -> bool: """Return whether an HTTP status is normally retryable.""" return status in {408, 425, 429} or 500 <= status <= 599