diff options
Diffstat (limited to 'webmentions_ssg/net.py')
| -rw-r--r-- | webmentions_ssg/net.py | 263 |
1 files changed, 263 insertions, 0 deletions
diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py new file mode 100644 index 0000000..01a9a80 --- /dev/null +++ b/webmentions_ssg/net.py @@ -0,0 +1,263 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Public-only HTTP transport, URL checks, and bounded response reads.""" + +import ipaddress +import socket +import time +from collections.abc import Iterable +from urllib.parse import urlsplit + +import dns.exception +import dns.resolver +import httpcore +import httpx +from flask import current_app +from httpcore._backends.sync import SyncStream + +NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96") +IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96") + + +class AddressResolutionError(Exception): + """A hostname could not be resolved (potentially transient).""" + + +class UnsafeAddressError(ValueError): + """A hostname or IP literal identifies a non-public network address.""" + + +class NetworkDeadline: + """One monotonic deadline shared by DNS, TCP, TLS, and response streaming.""" + + def __init__(self, seconds: float) -> None: + if seconds <= 0: + raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero") + self.expires = time.monotonic() + seconds + + def timeout(self, original: float | None, error: type[Exception]) -> float: + remaining = self.expires - time.monotonic() + if remaining <= 0: + raise error("Overall HTTP request deadline exceeded") + return min(remaining, original) if original is not None else remaining + + +class DeadlineStream(httpcore.NetworkStream): + """Clip every read/write/TLS operation to the remaining overall deadline.""" + + def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline): + self._stream = stream + self._deadline = deadline + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + return self._stream.read( + max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout) + ) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self._stream.write( + buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout) + ) + + def start_tls( + self, + ssl_context, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> httpcore.NetworkStream: + return DeadlineStream( + self._stream.start_tls( + ssl_context, + server_hostname=server_hostname, + timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout), + ), + self._deadline, + ) + + def get_extra_info(self, info: str): + return self._stream.get_extra_info(info) + + def close(self) -> None: + self._stream.close() + + +class PublicNetworkBackend(httpcore.SyncBackend): + """Resolve and validate in the TCP connection operation, never in a hook.""" + + def __init__(self, deadline: NetworkDeadline): + self._deadline = deadline + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[tuple] | None = None, + ) -> httpcore.NetworkStream: + # No second lookup of `host` is permitted. Even a DNS response that + # changes immediately after this check cannot change the socket peer. + dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout) + addresses = public_addresses(host, lifetime=dns_timeout) + last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None + + for address in addresses: + family = ( + socket.AF_INET6 + if ipaddress.ip_address(address).version == 6 + else socket.AF_INET + ) + sock = socket.socket(family, socket.SOCK_STREAM) + try: + sock.settimeout( + self._deadline.timeout(timeout, httpcore.ConnectTimeout) + ) + if local_address is not None: + sock.bind((local_address, 0)) + for option in socket_options or (): + sock.setsockopt(*option) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # This numeric address is the *same* address just validated; + # no getaddrinfo or DNS resolver runs between check and connect. + sock.connect((address, port)) + return DeadlineStream(SyncStream(sock), self._deadline) + except (TimeoutError, OSError) as exc: + sock.close() + if isinstance(exc, TimeoutError): + last_error = httpcore.ConnectTimeout(str(exc)) + else: + last_error = httpcore.ConnectError(str(exc)) + except BaseException: + sock.close() + raise + + assert last_error is not None + raise last_error + + +class PublicHTTPTransport(httpx.HTTPTransport): + """A direct (non-proxy) httpx transport with an IP-pinning network backend. + + httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we + initialize its compatible pool directly. Pin the httpx/httpcore major APIs + and keep this integration under test when upgrading either dependency. + """ + + def __init__(self, total_timeout: float): + self._pool = httpcore.ConnectionPool( + ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)), + ) + + +def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client: + """Return the hardened HTTP client shared by receiver and sender.""" + headers = {"User-Agent": user_agent} + if accept is not None: + headers["Accept"] = accept + return httpx.Client( + headers=headers, + timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]), + follow_redirects=True, + max_redirects=current_app.config["MAX_REDIRECTS"], + trust_env=False, + transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]), + ) + + +def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """Reject non-public addresses, including IPv4 hidden in IPv6 forms. + + ``is_global`` alone does not reliably exclude IPv6 transition addresses + across Python versions or networks with NAT64/6to4 routing. + """ + if isinstance(address, ipaddress.IPv6Address): + # The well-known NAT64 prefix can translate an otherwise global IPv6 + # address into a private IPv4 destination. IPv4-compatible addresses + # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata. + if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX: + return False + + # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4 + # destination through ipaddress; validate that destination as well. + for embedded in (address.sixtofour, address.ipv4_mapped): + if embedded is not None and not embedded.is_global: + return False + + return address.is_global + + +def public_addresses( + hostname: str, *, lifetime: float | None = None +) -> tuple[str, ...]: + """Resolve *once* and return only checked, globally routable IP addresses. + + A DNS name with a mixture of public and private answers is rejected in its + entirety. IP literals are handled without DNS, including IPv6 literals. + The returned strings must be used for the actual TCP connection; checking + them and then connecting to ``hostname`` reintroduces DNS rebinding. + """ + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + + if literal is not None: + addresses = (literal,) + else: + try: + answers = dns.resolver.resolve_name(hostname, lifetime=lifetime) + addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses()) + except (dns.exception.DNSException, ValueError) as exc: + raise AddressResolutionError( + f"Could not resolve hostname {hostname!r}" + ) from exc + + if not addresses: + raise AddressResolutionError(f"Could not resolve hostname {hostname!r}") + + if any(not is_public_address(address) for address in addresses): + raise UnsafeAddressError(f"Non-public network address for {hostname!r}") + + return tuple(str(address) for address in addresses) + + +def is_http_url(url: str) -> bool: + """ + Check whether a URL is an absolute HTTP or HTTPS URL. + + :param url: URL to validate. + :return: Whether the URL uses HTTP or HTTPS and contains a hostname. + """ + try: + parsed = urlsplit(url) + except ValueError: + return False + + return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None + + +def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes: + """Read a streaming response, rejecting oversized declared or actual bodies.""" + if (content_length := response.headers.get("Content-Length")) is not None: + try: + declared_length = int(content_length) + except ValueError: + pass + else: + if declared_length > max_bytes: + raise ValueError(f"{label} document is too large") + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) + if len(body) > max_bytes: + raise ValueError(f"{label} document is too large") + return bytes(body) + + +def is_temporary_status(status: int) -> bool: + """Return whether an HTTP status is normally retryable.""" + return status in {408, 425, 429} or 500 <= status <= 599 |
