aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/net.py
blob: c9c2beceb2d14179bb965c4a3c1456a92b6bb735 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
#
# SPDX-License-Identifier: BSD-3-Clause

"""Public-only HTTP transport, URL checks, and bounded response reads."""

import ipaddress
import socket
import time
from collections.abc import Iterable, Mapping
from contextlib import ExitStack
from http import HTTPStatus
from urllib.parse import urlsplit

import dns.exception
import dns.resolver
import httpcore
import httpx
from flask import current_app
from httpcore._backends.sync import SyncStream

NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
NETWORK_CONNECT_TIMOUT_ERROR = 599


class AddressResolutionError(Exception):
    """A hostname could not be resolved (potentially transient)."""


class UnsafeAddressError(ValueError):
    """A hostname or IP literal identifies a non-public network address."""


class ResponseTooLargeError(ValueError):
    """A response body exceeds its configured size limit."""


class NetworkDeadline:
    """One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""

    def __init__(self, seconds: float) -> None:
        if seconds <= 0:
            raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero")
        self.expires = time.monotonic() + seconds

    def timeout(self, original: float | None, error: type[Exception]) -> float:
        remaining = self.expires - time.monotonic()
        if remaining <= 0:
            raise error("Overall HTTP request deadline exceeded")
        return min(remaining, original) if original is not None else remaining


class DeadlineStream(httpcore.NetworkStream):
    """Clip every read/write/TLS operation to the remaining overall deadline."""

    def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline):
        self._stream = stream
        self._deadline = deadline

    def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
        return self._stream.read(
            max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout)
        )

    def write(self, buffer: bytes, timeout: float | None = None) -> None:
        self._stream.write(
            buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout)
        )

    def start_tls(
        self,
        ssl_context,
        server_hostname: str | None = None,
        timeout: float | None = None,
    ) -> httpcore.NetworkStream:
        return DeadlineStream(
            self._stream.start_tls(
                ssl_context,
                server_hostname=server_hostname,
                timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout),
            ),
            self._deadline,
        )

    def get_extra_info(self, info: str):
        return self._stream.get_extra_info(info)

    def close(self) -> None:
        self._stream.close()


class PublicNetworkBackend(httpcore.SyncBackend):
    """Resolve and validate in the TCP connection operation, never in a hook."""

    def __init__(self, deadline: NetworkDeadline):
        self._deadline = deadline

    def _connect_one(
        self,
        address: str,
        port: int,
        timeout: float | None,
        local_address: str | None,
        socket_options: Iterable[tuple] | None,
    ) -> httpcore.NetworkStream:
        family = (
            socket.AF_INET6
            if ipaddress.ip_address(address).version == 6
            else socket.AF_INET
        )

        # Close the socket on any error, transferring ownership to SyncStream
        # only after the connection and stream have been created successfully.
        with ExitStack() as cleanup:
            sock = socket.socket(family, socket.SOCK_STREAM)
            cleanup.callback(sock.close)
            sock.settimeout(self._deadline.timeout(timeout, httpcore.ConnectTimeout))

            if local_address is not None:
                sock.bind((local_address, 0))

            for option in socket_options or ():
                sock.setsockopt(*option)

            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

            # This numeric address is the *same* address just validated;
            # no getaddrinfo or DNS resolver runs between check and connect.
            sock.connect((address, port))

            stream = DeadlineStream(SyncStream(sock), self._deadline)
            cleanup.pop_all()
            return stream

    def connect_tcp(
        self,
        host: str,
        port: int,
        timeout: float | None = None,
        local_address: str | None = None,
        socket_options: Iterable[tuple] | None = None,
    ) -> httpcore.NetworkStream:
        # No second lookup of `host` is permitted. Even a DNS response that
        # changes immediately after this check cannot change the socket peer.
        dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout)
        addresses = public_addresses(host, lifetime=dns_timeout)
        last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None

        for address in addresses:
            try:
                return self._connect_one(
                    address, port, timeout, local_address, socket_options
                )
            except OSError as exc:
                if isinstance(exc, TimeoutError):
                    last_error = httpcore.ConnectTimeout(str(exc))
                else:
                    last_error = httpcore.ConnectError(str(exc))

        if last_error is None:
            raise httpcore.ConnectError(f"No public addresses available for {host!r}")
        raise last_error


class PublicHTTPTransport(httpx.HTTPTransport):
    """A direct (non-proxy) httpx transport with an IP-pinning network backend.

    httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we
    initialize its compatible pool directly. Pin the httpx/httpcore major APIs
    and keep this integration under test when upgrading either dependency.
    """

    def __init__(self, total_timeout: float):
        self._pool = httpcore.ConnectionPool(
            ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
            network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
        )


def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
    """Return the hardened HTTP client shared by receiver and sender."""
    headers = {"User-Agent": user_agent}
    if accept is not None:
        headers["Accept"] = accept

    return httpx.Client(
        headers=headers,
        timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]),
        follow_redirects=True,
        max_redirects=current_app.config["MAX_REDIRECTS"],
        trust_env=False,
        transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]),
    )


def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
    """Reject non-public addresses, including IPv4 hidden in IPv6 forms.

    ``is_global`` alone does not reliably exclude IPv6 transition addresses
    across Python versions or networks with NAT64/6to4 routing.
    """
    if isinstance(address, ipaddress.IPv6Address):
        # The well-known NAT64 prefix can translate an otherwise global IPv6
        # address into a private IPv4 destination. IPv4-compatible addresses
        # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
        if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
            return False

        # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
        # destination through ipaddress; validate that destination as well.
        for embedded in (address.sixtofour, address.ipv4_mapped):
            if embedded is not None and not embedded.is_global:
                return False

    return address.is_global


def resolve_hostname(
    hostname: str, *, lifetime: float | None = None
) -> tuple[ipaddress.IPv4Address | ipaddress.IPv6Address, ...]:
    try:
        answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
        return tuple(ipaddress.ip_address(a) for a in answers.addresses())
    except (dns.exception.DNSException, ValueError) as exc:
        raise AddressResolutionError(
            f"Could not resolve hostname {hostname!r}"
        ) from exc


def public_addresses(
    hostname: str, *, lifetime: float | None = None
) -> tuple[str, ...]:
    """Resolve *once* and return only checked, globally routable IP addresses.

    A DNS name with a mixture of public and private answers is rejected in its
    entirety. IP literals are handled without DNS, including IPv6 literals.
    The returned strings must be used for the actual TCP connection; checking
    them and then connecting to ``hostname`` reintroduces DNS rebinding.
    """
    try:
        addresses = (ipaddress.ip_address(hostname),)
    except ValueError:
        addresses = resolve_hostname(hostname, lifetime=lifetime)

    if not addresses:
        raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")

    if any(not is_public_address(address) for address in addresses):
        raise UnsafeAddressError(f"Non-public network address for {hostname!r}")

    return tuple(str(address) for address in addresses)


def hostname_of(url: str) -> str | None:
    """Return a normalized URL hostname, or None for malformed URLs."""
    try:
        hostname = urlsplit(url).hostname
    except TypeError, ValueError:
        return None
    return hostname.rstrip(".") if hostname is not None else None


def content_type_of(headers: Mapping[str, str]) -> str:
    """Return a normalized media type without Content-Type parameters."""
    return headers.get("Content-Type", "").partition(";")[0].strip().lower()


def is_http_url(url: str) -> bool:
    """Check whether a URL is an absolute HTTP or HTTPS URL.

    :param url: URL to validate.
    :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
    """
    try:
        parsed = urlsplit(url)
    except ValueError:
        return False

    return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None


def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
    """Read a streaming response, rejecting oversized declared or actual bodies."""
    if (content_length := response.headers.get("Content-Length")) is not None:
        try:
            declared_length = int(content_length)
        except ValueError:
            pass
        else:
            if declared_length > max_bytes:
                raise ResponseTooLargeError(f"{label} document is too large")

    body = bytearray()
    for chunk in response.iter_bytes(chunk_size=64 * 1024):
        body.extend(chunk)
        if len(body) > max_bytes:
            raise ResponseTooLargeError(f"{label} document is too large")
    return bytes(body)


def is_temporary_status(status: int) -> bool:
    """Return whether an HTTP status is normally retryable."""
    return (
        status
        in {
            HTTPStatus.REQUEST_TIMEOUT,
            HTTPStatus.TOO_EARLY,
            HTTPStatus.TOO_MANY_REQUESTS,
        }
        or HTTPStatus.INTERNAL_SERVER_ERROR <= status <= NETWORK_CONNECT_TIMOUT_ERROR
    )