aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/config.py2
-rw-r--r--webmentions_ssg/forms/__init__.py4
-rw-r--r--webmentions_ssg/forms/validators.py27
-rw-r--r--webmentions_ssg/public_transport.py144
-rw-r--r--webmentions_ssg/tasks/receiver.py34
-rw-r--r--webmentions_ssg/tasks/sender.py28
-rw-r--r--webmentions_ssg/url_security.py63
7 files changed, 241 insertions, 61 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index e8288db..1c7d44e 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -45,6 +45,8 @@ class DefaultConfig:
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
+ # Absolute wall-clock budget for each verification or delivery attempt.
+ REQUEST_TOTAL_TIMEOUT = 30.0
# Requeue abandoned verification attempts after 15 minutes.
VERIFICATION_TIMEOUT_SECONDS = 900
VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *"
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
index 2e3c72c..f698026 100644
--- a/webmentions_ssg/forms/__init__.py
+++ b/webmentions_ssg/forms/__init__.py
@@ -10,7 +10,7 @@ from flask_wtf import FlaskForm
from wtforms import PasswordField, StringField, SubmitField
from wtforms.validators import URL, InputRequired, Regexp
-from .validators import AllowedHostname, NotEqualTo, PublicURL
+from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral
class LoginForm(FlaskForm):
@@ -42,7 +42,7 @@ class EndpointForm(FlaskForm):
message="source must begin with http or https",
),
NotEqualTo("target"),
- PublicURL(),
+ PublicIPLiteral(),
],
)
target = StringField(
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index dd0ee9b..e8f33eb 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -4,6 +4,7 @@
"""Provide validators for URL safety and form field constraints."""
+import ipaddress
from urllib.parse import urlsplit
from flask import current_app
@@ -85,7 +86,10 @@ class AllowedHostname:
:param field: Field containing the URL to validate.
:raises ValidationError: If the URL hostname is not configured as allowed.
"""
- hostname = urlsplit(field.data).hostname
+ try:
+ hostname = urlsplit(field.data).hostname
+ except (ValueError, TypeError):
+ hostname = None
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
return
raise ValidationError(self.message or field.gettext("Invalid input."))
@@ -125,3 +129,24 @@ class PublicURL:
raise ValidationError(message)
except (AddressResolutionError, ValueError) as exc:
raise ValidationError(message) from exc
+
+
+class PublicIPLiteral:
+ """Reject non-public IP literals without performing DNS in the web worker."""
+
+ def __call__(self, form: BaseForm, field: Field) -> None:
+ try:
+ hostname = urlsplit(field.data).hostname
+ except (ValueError, TypeError) as exc:
+ raise ValidationError(field.gettext("Invalid input.")) from exc
+
+ if hostname is None:
+ raise ValidationError(field.gettext("Invalid input."))
+
+ try:
+ address = ipaddress.ip_address(hostname)
+ except ValueError:
+ return # Named hosts are resolved only during the background fetch.
+
+ if not address.is_global:
+ raise ValidationError(field.gettext("URL must use a public address."))
diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/public_transport.py
new file mode 100644
index 0000000..131cf55
--- /dev/null
+++ b/webmentions_ssg/public_transport.py
@@ -0,0 +1,144 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""HTTP transport that connects only to validated DNS answers.
+
+httpcore receives the original host for Host and TLS SNI/certificate checks,
+but the socket is opened to an IP from a single, bounded DNS lookup. This
+protects the receiver and sender, including redirects and advertised URLs.
+"""
+
+import ipaddress
+import socket
+import time
+from collections.abc import Iterable
+
+import httpcore
+import httpx
+from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation
+
+from .url_security import public_addresses
+
+
+class NetworkDeadline:
+ """One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""
+
+ def __init__(self, seconds: float) -> None:
+ if seconds <= 0:
+ raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero")
+ self.expires = time.monotonic() + seconds
+
+ def timeout(self, original: float | None, error: type[Exception]) -> float:
+ remaining = self.expires - time.monotonic()
+ if remaining <= 0:
+ raise error("Overall HTTP request deadline exceeded")
+ return min(remaining, original) if original is not None else remaining
+
+
+class DeadlineStream(httpcore.NetworkStream):
+ """Clip every read/write/TLS operation to the remaining overall deadline."""
+
+ def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline):
+ self._stream = stream
+ self._deadline = deadline
+
+ def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
+ return self._stream.read(
+ max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout)
+ )
+
+ def write(self, buffer: bytes, timeout: float | None = None) -> None:
+ self._stream.write(
+ buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout)
+ )
+
+ def start_tls(
+ self,
+ ssl_context,
+ server_hostname: str | None = None,
+ timeout: float | None = None,
+ ) -> httpcore.NetworkStream:
+ return DeadlineStream(
+ self._stream.start_tls(
+ ssl_context,
+ server_hostname=server_hostname,
+ timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout),
+ ),
+ self._deadline,
+ )
+
+ def get_extra_info(self, info: str):
+ return self._stream.get_extra_info(info)
+
+ def close(self) -> None:
+ self._stream.close()
+
+
+class PublicNetworkBackend(httpcore.SyncBackend):
+ """Resolve and validate in the TCP connection operation, never in a hook."""
+
+ def __init__(self, deadline: NetworkDeadline):
+ self._deadline = deadline
+
+ def connect_tcp(
+ self,
+ host: str,
+ port: int,
+ timeout: float | None = None,
+ local_address: str | None = None,
+ socket_options: Iterable[tuple] | None = None,
+ ) -> httpcore.NetworkStream:
+ # No second lookup of `host` is permitted. Even a DNS response that
+ # changes immediately after this check cannot change the socket peer.
+ dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout)
+ addresses = public_addresses(host, lifetime=dns_timeout)
+ last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
+
+ for address in addresses:
+ family = (
+ socket.AF_INET6
+ if ipaddress.ip_address(address).version == 6
+ else socket.AF_INET
+ )
+ sock = socket.socket(family, socket.SOCK_STREAM)
+ try:
+ sock.settimeout(
+ self._deadline.timeout(timeout, httpcore.ConnectTimeout)
+ )
+ if local_address is not None:
+ sock.bind((local_address, 0))
+ for option in socket_options or ():
+ sock.setsockopt(*option)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ # This numeric address is the *same* address just validated;
+ # no getaddrinfo or DNS resolver runs between check and connect.
+ sock.connect((address, port))
+ return DeadlineStream(SyncStream(sock), self._deadline)
+ except (TimeoutError, OSError) as exc:
+ sock.close()
+ if isinstance(exc, TimeoutError):
+ last_error = httpcore.ConnectTimeout(str(exc))
+ else:
+ last_error = httpcore.ConnectError(str(exc))
+ except BaseException:
+ sock.close()
+ raise
+
+ assert last_error is not None
+ raise last_error
+
+
+class PublicHTTPTransport(httpx.HTTPTransport):
+ """A direct (non-proxy) httpx transport with an IP-pinning network backend.
+
+ httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we
+ initialize its compatible pool directly. Pin the httpx/httpcore major APIs
+ and keep this integration under test when upgrading either dependency.
+ """
+
+ def __init__(self, total_timeout: float):
+ self._pool = httpcore.ConnectionPool(
+ ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
+ network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
+ )
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 4e8df74..f999d1b 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -18,7 +18,8 @@ from .. import APP_NAME, VERSION
from .. import DATABASE as db
from .. import HUEY as huey
from ..models import ReceivedWebmention
-from ..url_security import AddressResolutionError, is_public_url
+from ..public_transport import PublicHTTPTransport
+from ..url_security import AddressResolutionError, UnsafeAddressError
class VerificationError(Exception):
@@ -124,24 +125,6 @@ def text_mentions_target(body: str, target_url: str) -> bool:
return False
-def ensure_public_request(request: httpx.Request) -> None:
- """
- Ensure that an HTTP request targets a public network address.
-
- :param request: HTTP request to validate.
- :raises VerificationError: If the URL has no hostname or resolves to a
- non-public address.
- :raises TemporaryFetchError: If the hostname cannot be resolved.
- """
- try:
- if not is_public_url(str(request.url)):
- raise VerificationError("Source resolves to a non-public address")
- except ValueError as exc:
- raise VerificationError("Source URL has no hostname") from exc
- except AddressResolutionError as exc:
- raise TemporaryFetchError("Source hostname could not be resolved") from exc
-
-
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
Fetch a webmention source document.
@@ -157,6 +140,15 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
status or exceeds the configured maximum response size.
"""
+ try:
+ return _fetch_source(source_url)
+ except UnsafeAddressError as exc:
+ raise VerificationError("Source resolves to a non-public address") from exc
+ except AddressResolutionError as exc:
+ raise TemporaryFetchError("Source hostname could not be resolved") from exc
+
+
+def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
with (
httpx.Client(
headers={
@@ -167,7 +159,9 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
follow_redirects=True,
max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
trust_env=False,
- event_hooks={"request": [ensure_public_request]},
+ transport=PublicHTTPTransport(
+ current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
+ ),
) as client,
client.stream("GET", source_url) as response,
):
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 725b5c8..7121f6a 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -19,7 +19,8 @@ from .. import APP_NAME, VERSION
from .. import DATABASE as db
from .. import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..url_security import AddressResolutionError, is_http_url, is_public_url
+from ..public_transport import PublicHTTPTransport
+from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
LINK_SPLIT = re.compile(r",\s*(?=<)")
@@ -48,21 +49,6 @@ def temporary_http_status(status: int) -> bool:
return status in {408, 425, 429} or 500 <= status <= 599
-def ensure_public_request(request: httpx.Request) -> None:
- """
- Ensure that an HTTP request targets a public network address.
-
- :param request: HTTP request to validate.
- :raises PermanentSenderError: If the request resolves to a non-public address.
- :raises TemporarySenderError: If the request hostname cannot be resolved.
- """
- try:
- if not is_public_url(str(request.url)):
- raise PermanentSenderError("Request resolves to a non-public address")
- except AddressResolutionError as exc:
- raise TemporarySenderError("Request hostname could not be resolved") from exc
-
-
def resolve_endpoint(response: httpx.Response, href: str) -> str:
"""
Resolve and validate a discovered Webmention endpoint.
@@ -224,7 +210,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
"""
try:
head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
+ except (httpx.RequestError, TemporarySenderError):
# HEAD is only an optimization. Servers and intermediaries may reject
# or drop it even when fetching the document with GET would succeed.
pass
@@ -442,7 +428,9 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
follow_redirects=True,
max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
trust_env=False,
- event_hooks={"request": [ensure_public_request]},
+ transport=PublicHTTPTransport(
+ current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
+ ),
) as client:
endpoint = discover_webmention_endpoint(client, target)
@@ -472,7 +460,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
raise PermanentSenderError(message)
- except (TemporarySenderError, httpx.RequestError) as exc:
+ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
if not attempt_is_current(webmention, revision):
return
@@ -495,7 +483,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
# No Huey retries: only the scanner may requeue after next_retry_at.
return
- except PermanentSenderError as exc:
+ except (PermanentSenderError, UnsafeAddressError) as exc:
if not attempt_is_current(webmention, revision):
return
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
index ade64d6..f943c7e 100644
--- a/webmentions_ssg/url_security.py
+++ b/webmentions_ssg/url_security.py
@@ -12,17 +12,51 @@ import dns.resolver
class AddressResolutionError(Exception):
- pass
+ """A hostname could not be resolved (potentially transient)."""
-def is_public_url(url: str) -> bool:
+class UnsafeAddressError(ValueError):
+ """A hostname or IP literal identifies a non-public network address."""
+
+
+def public_addresses(hostname: str, *, lifetime: float | None = None) -> tuple[str, ...]:
+ """Resolve *once* and return only checked, globally routable IP addresses.
+
+ A DNS name with a mixture of public and private answers is rejected in its
+ entirety. IP literals are handled without DNS, including IPv6 literals.
+ The returned strings must be used for the actual TCP connection; checking
+ them and then connecting to ``hostname`` reintroduces DNS rebinding.
"""
- Check whether a URL resolves exclusively to public IP addresses.
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ addresses = (literal,)
+ else:
+ try:
+ answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
+ addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
+ except (dns.exception.DNSException, ValueError) as exc:
+ raise AddressResolutionError(
+ f"Could not resolve hostname {hostname!r}"
+ ) from exc
+
+ if not addresses:
+ raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
- :param url: URL whose hostname should be resolved.
- :return: Whether all resolved addresses are globally routable.
- :raises ValueError: If the URL does not contain a hostname.
- :raises AddressResolutionError: If the hostname cannot be resolved.
+ if any(not address.is_global for address in addresses):
+ raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
+
+ return tuple(str(address) for address in addresses)
+
+
+def is_public_url(url: str) -> bool:
+ """Return whether the URL resolves entirely to public IP addresses.
+
+ This is useful for diagnostics, but **not** sufficient to authorize a
+ request: network connections must use the exact validated IP address.
"""
hostname = urlsplit(url).hostname
@@ -30,17 +64,10 @@ def is_public_url(url: str) -> bool:
raise ValueError("No hostname was specified")
try:
- answers = dns.resolver.resolve_name(hostname)
- except dns.exception.DNSException as exc:
- raise AddressResolutionError(
- f"Could not resolve hostname {hostname!r}"
- ) from exc
- else:
- for address in answers.addresses():
- if not ipaddress.ip_address(address).is_global:
- return False
-
- return True
+ public_addresses(hostname)
+ except UnsafeAddressError:
+ return False
+ return True
def is_http_url(url: str) -> bool: