aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/net.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/net.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/net.py (renamed from webmentions_ssg/public_transport.py)135
1 files changed, 127 insertions, 8 deletions
diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/net.py
index 131cf55..01a9a80 100644
--- a/webmentions_ssg/public_transport.py
+++ b/webmentions_ssg/net.py
@@ -2,23 +2,31 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""HTTP transport that connects only to validated DNS answers.
-
-httpcore receives the original host for Host and TLS SNI/certificate checks,
-but the socket is opened to an IP from a single, bounded DNS lookup. This
-protects the receiver and sender, including redirects and advertised URLs.
-"""
+"""Public-only HTTP transport, URL checks, and bounded response reads."""
import ipaddress
import socket
import time
from collections.abc import Iterable
+from urllib.parse import urlsplit
+import dns.exception
+import dns.resolver
import httpcore
import httpx
-from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation
+from flask import current_app
+from httpcore._backends.sync import SyncStream
+
+NAT64_PREFIX = ipaddress.IPv6Network("64:ff9b::/96")
+IPV4_COMPATIBLE_PREFIX = ipaddress.IPv6Network("::/96")
+
+
+class AddressResolutionError(Exception):
+ """A hostname could not be resolved (potentially transient)."""
-from .url_security import public_addresses
+
+class UnsafeAddressError(ValueError):
+ """A hostname or IP literal identifies a non-public network address."""
class NetworkDeadline:
@@ -142,3 +150,114 @@ class PublicHTTPTransport(httpx.HTTPTransport):
ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
)
+
+
+def build_client(user_agent: str, *, accept: str | None = None) -> httpx.Client:
+ """Return the hardened HTTP client shared by receiver and sender."""
+ headers = {"User-Agent": user_agent}
+ if accept is not None:
+ headers["Accept"] = accept
+ return httpx.Client(
+ headers=headers,
+ timeout=httpx.Timeout(current_app.config["REQUEST_TIMEOUT"]),
+ follow_redirects=True,
+ max_redirects=current_app.config["MAX_REDIRECTS"],
+ trust_env=False,
+ transport=PublicHTTPTransport(current_app.config["REQUEST_TOTAL_TIMEOUT"]),
+ )
+
+
+def is_public_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+ """Reject non-public addresses, including IPv4 hidden in IPv6 forms.
+
+ ``is_global`` alone does not reliably exclude IPv6 transition addresses
+ across Python versions or networks with NAT64/6to4 routing.
+ """
+ if isinstance(address, ipaddress.IPv6Address):
+ # The well-known NAT64 prefix can translate an otherwise global IPv6
+ # address into a private IPv4 destination. IPv4-compatible addresses
+ # similarly embed IPv4 in ::/96, even without ipv4_mapped metadata.
+ if address in NAT64_PREFIX or address in IPV4_COMPATIBLE_PREFIX:
+ return False
+
+ # 6to4 and IPv4-mapped IPv6 addresses expose their embedded IPv4
+ # destination through ipaddress; validate that destination as well.
+ for embedded in (address.sixtofour, address.ipv4_mapped):
+ if embedded is not None and not embedded.is_global:
+ return False
+
+ return address.is_global
+
+
+def public_addresses(
+ hostname: str, *, lifetime: float | None = None
+) -> tuple[str, ...]:
+ """Resolve *once* and return only checked, globally routable IP addresses.
+
+ A DNS name with a mixture of public and private answers is rejected in its
+ entirety. IP literals are handled without DNS, including IPv6 literals.
+ The returned strings must be used for the actual TCP connection; checking
+ them and then connecting to ``hostname`` reintroduces DNS rebinding.
+ """
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ addresses = (literal,)
+ else:
+ try:
+ answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
+ addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
+ except (dns.exception.DNSException, ValueError) as exc:
+ raise AddressResolutionError(
+ f"Could not resolve hostname {hostname!r}"
+ ) from exc
+
+ if not addresses:
+ raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
+
+ if any(not is_public_address(address) for address in addresses):
+ raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
+
+ return tuple(str(address) for address in addresses)
+
+
+def is_http_url(url: str) -> bool:
+ """
+ Check whether a URL is an absolute HTTP or HTTPS URL.
+
+ :param url: URL to validate.
+ :return: Whether the URL uses HTTP or HTTPS and contains a hostname.
+ """
+ try:
+ parsed = urlsplit(url)
+ except ValueError:
+ return False
+
+ return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None
+
+
+def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -> bytes:
+ """Read a streaming response, rejecting oversized declared or actual bodies."""
+ if (content_length := response.headers.get("Content-Length")) is not None:
+ try:
+ declared_length = int(content_length)
+ except ValueError:
+ pass
+ else:
+ if declared_length > max_bytes:
+ raise ValueError(f"{label} document is too large")
+
+ body = bytearray()
+ for chunk in response.iter_bytes(chunk_size=64 * 1024):
+ body.extend(chunk)
+ if len(body) > max_bytes:
+ raise ValueError(f"{label} document is too large")
+ return bytes(body)
+
+
+def is_temporary_status(status: int) -> bool:
+ """Return whether an HTTP status is normally retryable."""
+ return status in {408, 425, 429} or 500 <= status <= 599