aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--contrib/webmentions-ssg.example.cfg8
-rw-r--r--pyproject.toml3
-rw-r--r--tests/forms/test_init.py49
-rw-r--r--tests/forms/test_validators.py6
-rw-r--r--tests/tasks/test_receiver.py58
-rw-r--r--tests/tasks/test_sender.py37
-rw-r--r--tests/test_config.py1
-rw-r--r--tests/test_public_transport.py149
-rw-r--r--tests/test_url_security.py29
-rw-r--r--uv.lock4
-rw-r--r--webmentions_ssg/config.py2
-rw-r--r--webmentions_ssg/forms/__init__.py4
-rw-r--r--webmentions_ssg/forms/validators.py27
-rw-r--r--webmentions_ssg/public_transport.py144
-rw-r--r--webmentions_ssg/tasks/receiver.py34
-rw-r--r--webmentions_ssg/tasks/sender.py28
-rw-r--r--webmentions_ssg/url_security.py63
17 files changed, 495 insertions, 151 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg
index bf363b8..02b0744 100644
--- a/contrib/webmentions-ssg.example.cfg
+++ b/contrib/webmentions-ssg.example.cfg
@@ -44,8 +44,14 @@ LOGFILE_BACKUP_COUNT = 10
# Webmention. Include each hostname you serve; wildcards are not supported.
ALLOWED_HOSTNAMES = {"example.com", "www.example.com"}
+# Defense in depth: restrict the Huey worker service account with outbound
+# firewall/network-policy rules that block loopback, RFC1918, link-local, ULA,
+# and cloud metadata ranges, including IPv6. The application also pins
+# validated DNS IPs, but deployment-level filtering is still necessary.
+
# Limits for fetching/verifying incoming sources.
-REQUEST_TIMEOUT = 5.0 # seconds; also used for outgoing requests
+REQUEST_TIMEOUT = 5.0 # seconds per socket operation (both directions)
+REQUEST_TOTAL_TIMEOUT = 30.0 # seconds overall, including DNS, redirects and body
MAX_REDIRECTS = 20 # also used for outgoing requests
MAX_SOURCE_BYTES = 1_000_000
# Requeue stuck verifications and lost queued tasks after 15 minutes.
diff --git a/pyproject.toml b/pyproject.toml
index f42752a..9b410ff 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,8 @@ dependencies = [
"flask-migrate>=4.1.0",
"flask-sqlalchemy>=3.1.1",
"flask-wtf>=1.3.0",
- "httpx>=0.28.1",
+ "httpx>=0.28.1,<0.29",
+ "httpcore>=1.0.9,<1.1",
"huey>=3.1.0",
"mf2py>=2.0.1",
"rfc3987>=1.3.8",
diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py
index 6fe1849..ffcbd8e 100644
--- a/tests/forms/test_init.py
+++ b/tests/forms/test_init.py
@@ -2,8 +2,6 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-from unittest.mock import Mock
-
import pytest
from flask import Flask
from werkzeug.datastructures import MultiDict
@@ -14,13 +12,6 @@ VALID_SOURCE = "https://source.example/post"
VALID_TARGET = "https://dennisfink.me/blog/example/"
-@pytest.fixture(autouse=True)
-def public_urls(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- "webmentions_ssg.forms.validators.is_public_url", Mock(return_value=True)
- )
-
-
@pytest.mark.parametrize(
("form_data", "invalid_field", "expected_error"),
[
@@ -113,3 +104,43 @@ def test_admin_action_form_accepts_submission(app: Flask) -> None:
with app.test_request_context(method="POST"):
form = AdminActionForm(meta={"csrf": False})
assert form.validate()
+
+
+def test_endpoint_does_not_do_dns_in_web_request(
+ app: Flask, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def unexpected_dns_lookup(*args: object, **kwargs: object) -> None:
+ raise AssertionError("No DNS lookup belongs in form validation")
+
+ monkeypatch.setattr(
+ "webmentions_ssg.url_security.dns.resolver.resolve_name",
+ unexpected_dns_lookup,
+ )
+ with app.test_request_context("/endpoint", method="POST"):
+ for source in (VALID_SOURCE, "ftp://source.example/post", "not a URL"):
+ form = EndpointForm(
+ formdata=MultiDict({"source": source, "target": VALID_TARGET}),
+ meta={"csrf": False},
+ )
+ assert form.validate() is (source == VALID_SOURCE)
+
+
+@pytest.mark.parametrize("source", ["http://127.0.0.1/", "http://[::1]/", "http://169.254.169.254/"])
+def test_endpoint_rejects_private_ip_literals(app: Flask, source: str) -> None:
+ with app.test_request_context("/endpoint", method="POST"):
+ form = EndpointForm(
+ formdata=MultiDict({"source": source, "target": VALID_TARGET}),
+ meta={"csrf": False},
+ )
+ assert not form.validate()
+ assert "URL must use a public address." in form.errors["source"]
+
+
+def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None:
+ with app.test_request_context("/endpoint", method="POST"):
+ form = EndpointForm(
+ formdata=MultiDict({"source": VALID_SOURCE, "target": "http://[::1"}),
+ meta={"csrf": False},
+ )
+ assert not form.validate()
+ assert "Invalid input." in form.errors["target"]
diff --git a/tests/forms/test_validators.py b/tests/forms/test_validators.py
index 6e26039..2858b4c 100644
--- a/tests/forms/test_validators.py
+++ b/tests/forms/test_validators.py
@@ -112,3 +112,9 @@ def test_public_url_uses_custom_message(monkeypatch: pytest.MonkeyPatch) -> None
with pytest.raises(ValidationError, match="Public URL required"):
PublicURL("Public URL required")(form, form.url)
+
+
+def test_allowed_hostname_handles_invalid_ipv6(app: Flask) -> None:
+ form = URLForm(MultiDict({"url": "http://[::1"}))
+ with app.app_context(), pytest.raises(ValidationError, match="Invalid input"):
+ AllowedHostname()(form, form.url)
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 4daca58..102410d 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -236,7 +236,7 @@ def test_fetch_source_returns_response_and_body(
assert options["follow_redirects"] is True
assert options["max_redirects"] == app.config.get("MAX_REDIRECTS", 20)
assert options["trust_env"] is False
- assert options["event_hooks"] == {"request": [receiver.ensure_public_request]}
+ assert options["transport"].__class__.__name__ == "PublicHTTPTransport"
@patch("webmentions_ssg.tasks.receiver.httpx.Client")
@@ -680,52 +680,30 @@ def test_verify_webmention_ignores_unknown_identifier(
assert f"Cannot verify unknown ReceivedWebmention {identifier}" in caplog.text
-@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=True)
-def test_ensure_public_request_accepts_public_url(
- is_public_url: Mock, receiver: ModuleType
+@patch("webmentions_ssg.tasks.receiver.httpx.Client")
+def test_fetch_source_rejects_unsafe_destination(
+ httpx_client: Mock, app: Flask, receiver: ModuleType
) -> None:
- receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL))
- is_public_url.assert_called_once_with(SOURCE_URL)
-
+ from webmentions_ssg.url_security import UnsafeAddressError
-@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=False)
-def test_ensure_public_request_rejects_non_public_address(
- is_public_url: Mock, receiver: ModuleType
-) -> None:
- with pytest.raises(
- receiver.VerificationError, match="Source resolves to a non-public address"
+ client = httpx_client.return_value.__enter__.return_value
+ client.stream.side_effect = UnsafeAddressError("Local address")
+ with app.app_context(), pytest.raises(
+ receiver.VerificationError, match="non-public address"
):
- receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL))
-
- is_public_url.assert_called_once_with(SOURCE_URL)
+ receiver.fetch_source(SOURCE_URL)
-@patch(
- "webmentions_ssg.tasks.receiver.is_public_url",
- side_effect=AddressResolutionError("Could not resolve hostname"),
-)
-def test_ensure_public_request_maps_dns_failure_to_temporary_error(
- is_public_url: Mock, receiver: ModuleType
+@patch("webmentions_ssg.tasks.receiver.httpx.Client")
+def test_fetch_source_treats_dns_failure_as_temporary(
+ httpx_client: Mock, app: Flask, receiver: ModuleType
) -> None:
- with pytest.raises(
- receiver.TemporaryFetchError, match="Source hostname could not be resolved"
+ client = httpx_client.return_value.__enter__.return_value
+ client.stream.side_effect = AddressResolutionError("DNS timeout")
+ with app.app_context(), pytest.raises(
+ receiver.TemporaryFetchError, match="could not be resolved"
):
- receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL))
-
- is_public_url.assert_called_once_with(SOURCE_URL)
-
-
-@patch(
- "webmentions_ssg.tasks.receiver.is_public_url",
- side_effect=ValueError("No hostname was specified"),
-)
-def test_ensure_public_request_rejects_url_without_hostname(
- is_public_url: Mock, receiver: ModuleType
-) -> None:
- with pytest.raises(receiver.VerificationError, match="Source URL has no hostname"):
- receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL))
-
- is_public_url.assert_called_once_with(SOURCE_URL)
+ receiver.fetch_source(SOURCE_URL)
@patch("webmentions_ssg.tasks.receiver.BeautifulSoup")
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index 0db0d8c..38bae11 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -1214,43 +1214,6 @@ def test_send_webmention_ignores_unknown_identifier(
assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text
-def test_ensure_public_request_accepts_public_url(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- is_public_url = Mock(return_value=True)
- monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
- sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
- is_public_url.assert_called_once_with(TARGET_URL)
-
-
-def test_ensure_public_request_rejects_non_public_address(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- is_public_url = Mock(return_value=False)
- monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
- with pytest.raises(
- sender_module.PermanentSenderError,
- match="Request resolves to a non-public address",
- ):
- sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
- is_public_url.assert_called_once_with(TARGET_URL)
-
-
-def test_ensure_public_request_maps_dns_failure_to_temporary_error(
- sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
-) -> None:
- is_public_url = Mock(
- side_effect=AddressResolutionError("Could not resolve hostname")
- )
- monkeypatch.setattr(sender_module, "is_public_url", is_public_url)
- with pytest.raises(
- sender_module.TemporarySenderError,
- match="Request hostname could not be resolved",
- ):
- sender_module.ensure_public_request(httpx.Request("GET", TARGET_URL))
- is_public_url.assert_called_once_with(TARGET_URL)
-
-
def test_resolve_endpoint_rejects_invalid_url(
sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
diff --git a/tests/test_config.py b/tests/test_config.py
index 57ac423..1b45946 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -72,6 +72,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None:
assert config.MAX_REDIRECTS == 20
assert config.MAX_SOURCE_BYTES == 1_000_000
assert config.REQUEST_TIMEOUT == 5.0
+ assert config.REQUEST_TOTAL_TIMEOUT == 30.0
assert DefaultConfig.ALLOWED_HOSTNAMES == set()
diff --git a/tests/test_public_transport.py b/tests/test_public_transport.py
new file mode 100644
index 0000000..e7d77fa
--- /dev/null
+++ b/tests/test_public_transport.py
@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Security regression tests for DNS-pinned network connections."""
+
+import socket
+from unittest.mock import Mock
+
+import httpcore
+import httpx
+import pytest
+
+from webmentions_ssg.public_transport import (
+ DeadlineStream,
+ NetworkDeadline,
+ PublicHTTPTransport,
+ PublicNetworkBackend,
+)
+from webmentions_ssg.url_security import UnsafeAddressError
+
+
+@pytest.mark.parametrize("host", ["127.0.0.1", "169.254.169.254", "::1"])
+def test_backend_rejects_private_ip_without_socket(
+ monkeypatch: pytest.MonkeyPatch, host: str
+) -> None:
+ create_socket = Mock(side_effect=AssertionError("must not open socket"))
+ monkeypatch.setattr(socket, "socket", create_socket)
+ with pytest.raises(UnsafeAddressError):
+ PublicNetworkBackend(NetworkDeadline(5)).connect_tcp(host, 80)
+ create_socket.assert_not_called()
+
+
+def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The address is resolved exactly once and the socket sees only that IP."""
+ dns = Mock(return_value=("93.184.216.34",))
+ monkeypatch.setattr("webmentions_ssg.public_transport.public_addresses", dns)
+ socket_instance = Mock()
+ create_socket = Mock(return_value=socket_instance)
+ monkeypatch.setattr(socket, "socket", create_socket)
+ monkeypatch.setattr(
+ socket, "getaddrinfo", Mock(side_effect=AssertionError("second resolution"))
+ )
+
+ stream = PublicNetworkBackend(NetworkDeadline(5)).connect_tcp(
+ "rebinding.example", 443, timeout=2.0
+ )
+ assert isinstance(stream, DeadlineStream)
+ assert stream.get_extra_info("socket") is socket_instance
+ dns.assert_called_once()
+ assert dns.call_args.args == ("rebinding.example",)
+ assert 0 < dns.call_args.kwargs["lifetime"] <= 2.0
+ socket_instance.connect.assert_called_once_with(("93.184.216.34", 443))
+ create_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
+
+
+class FakeStream(httpcore.NetworkStream):
+ """In-memory TCP/TLS stream so HTTP framing and SNI can be tested."""
+
+ def __init__(self, response: bytes) -> None:
+ self.response = response
+ self.writes = bytearray()
+ self.sni: str | None = None
+
+ def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
+ part, self.response = self.response[:max_bytes], self.response[max_bytes:]
+ return part
+
+ def write(self, buffer: bytes, timeout: float | None = None) -> None:
+ self.writes.extend(buffer)
+
+ def start_tls(
+ self, ssl_context, server_hostname: str | None = None, timeout: float | None = None
+ ) -> httpcore.NetworkStream:
+ self.sni = server_hostname
+ return self
+
+ def get_extra_info(self, info: str):
+ return None
+
+ def close(self) -> None:
+ pass
+
+
+@pytest.mark.parametrize("scheme", ["http", "https"])
+def test_transport_keeps_original_http_host_and_tls_sni(
+ monkeypatch: pytest.MonkeyPatch, scheme: str
+) -> None:
+ stream = FakeStream(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
+ backend = Mock(return_value=stream)
+ monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", backend)
+
+ with httpx.Client(transport=PublicHTTPTransport(5.0), trust_env=False) as client:
+ response = client.get(f"{scheme}://source.example/path")
+
+ assert response.status_code == 200
+ assert response.text == "OK"
+ assert b"Host: source.example\r\n" in stream.writes
+ assert b"GET /path HTTP/1.1\r\n" in stream.writes
+ assert stream.sni == ("source.example" if scheme == "https" else None)
+ backend.assert_called_once()
+ assert backend.call_args.kwargs["host"] == "source.example"
+ assert backend.call_args.kwargs["port"] == (443 if scheme == "https" else 80)
+
+
+def test_deadline_clips_each_read_and_stops_trickle(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ clock = [100.0]
+ monkeypatch.setattr("webmentions_ssg.public_transport.time.monotonic", lambda: clock[0])
+ fake_stream = Mock()
+ fake_stream.read.return_value = b"x"
+ stream = DeadlineStream(fake_stream, NetworkDeadline(5.0))
+
+ clock[0] = 103.0
+ assert stream.read(1, timeout=5.0) == b"x"
+ assert fake_stream.read.call_args.kwargs["timeout"] == pytest.approx(2.0)
+
+ clock[0] = 105.1
+ with pytest.raises(httpcore.ReadTimeout, match="Overall HTTP request deadline"):
+ stream.read(1, timeout=5.0)
+ assert fake_stream.read.call_count == 1
+
+
+def test_redirects_go_through_same_destination_guard(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ redirect = FakeStream(
+ b"HTTP/1.1 302 Found\r\nLocation: http://internal.example/\r\n"
+ b"Content-Length: 0\r\n\r\n"
+ )
+ checked_hosts: list[str] = []
+
+ def connect(
+ self: PublicNetworkBackend, host: str, port: int, **kwargs: object
+ ) -> httpcore.NetworkStream:
+ checked_hosts.append(host)
+ if host == "internal.example":
+ raise UnsafeAddressError("Non-public network address")
+ return redirect
+
+ monkeypatch.setattr(PublicNetworkBackend, "connect_tcp", connect)
+ with (
+ httpx.Client(transport=PublicHTTPTransport(5.0), follow_redirects=True) as client,
+ pytest.raises(UnsafeAddressError, match="Non-public"),
+ ):
+ client.get("http://source.example/post")
+
+ assert checked_hosts == ["source.example", "internal.example"]
diff --git a/tests/test_url_security.py b/tests/test_url_security.py
index 80b45e9..d25589f 100644
--- a/tests/test_url_security.py
+++ b/tests/test_url_security.py
@@ -9,6 +9,8 @@ import pytest
from webmentions_ssg.url_security import (
AddressResolutionError,
+ UnsafeAddressError,
+ public_addresses,
is_http_url,
is_public_url,
is_safe_next_url,
@@ -114,7 +116,7 @@ def test_is_public_url_returns_true_for_public_addresses(
def test_is_public_url_raises_for_resolution_failure(resolve_name: Mock) -> None:
with pytest.raises(AddressResolutionError, match="Could not resolve hostname"):
is_public_url("https://nonexistent.example/")
- resolve_name.assert_called_once_with("nonexistent.example")
+ resolve_name.assert_called_once_with("nonexistent.example", lifetime=None)
@patch("webmentions_ssg.url_security.dns.resolver.resolve_name")
@@ -122,3 +124,28 @@ def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> No
with pytest.raises(ValueError, match="No hostname was specified"):
is_public_url("/relative/url")
resolve_name.assert_not_called()
+
+
+def test_ip_literal_does_not_use_dns() -> None:
+ with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as dns_resolve:
+ with pytest.raises(UnsafeAddressError):
+ public_addresses("169.254.169.254")
+ with pytest.raises(UnsafeAddressError):
+ public_addresses("::1")
+ assert public_addresses("8.8.8.8") == ("8.8.8.8",)
+ dns_resolve.assert_not_called()
+
+
+def test_public_addresses_rejects_mixed_dns_results() -> None:
+ with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
+ resolve.return_value.addresses.return_value = ["93.184.216.34", "127.0.0.1"]
+ with pytest.raises(UnsafeAddressError, match="Non-public"):
+ public_addresses("rebinding.example", lifetime=1.5)
+ resolve.assert_called_once_with("rebinding.example", lifetime=1.5)
+
+
+def test_public_addresses_rejects_empty_dns_result() -> None:
+ with patch("webmentions_ssg.url_security.dns.resolver.resolve_name") as resolve:
+ resolve.return_value.addresses.return_value = []
+ with pytest.raises(AddressResolutionError):
+ public_addresses("empty.example")
diff --git a/uv.lock b/uv.lock
index 1ddffdc..c57bf51 100644
--- a/uv.lock
+++ b/uv.lock
@@ -902,6 +902,7 @@ dependencies = [
{ name = "flask-migrate" },
{ name = "flask-sqlalchemy" },
{ name = "flask-wtf" },
+ { name = "httpcore" },
{ name = "httpx" },
{ name = "huey" },
{ name = "mf2py" },
@@ -934,7 +935,8 @@ requires-dist = [
{ name = "flask-migrate", specifier = ">=4.1.0" },
{ name = "flask-sqlalchemy", specifier = ">=3.1.1" },
{ name = "flask-wtf", specifier = ">=1.3.0" },
- { name = "httpx", specifier = ">=0.28.1" },
+ { name = "httpcore", specifier = ">=1.0.9,<1.1" },
+ { name = "httpx", specifier = ">=0.28.1,<0.29" },
{ name = "huey", specifier = ">=3.1.0" },
{ name = "mf2py", specifier = ">=2.0.1" },
{ name = "rfc3987", specifier = ">=1.3.8" },
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index e8288db..1c7d44e 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -45,6 +45,8 @@ class DefaultConfig:
MAX_REDIRECTS = 20
MAX_SOURCE_BYTES = 1_000_000
REQUEST_TIMEOUT = 5.0
+ # Absolute wall-clock budget for each verification or delivery attempt.
+ REQUEST_TOTAL_TIMEOUT = 30.0
# Requeue abandoned verification attempts after 15 minutes.
VERIFICATION_TIMEOUT_SECONDS = 900
VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *"
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
index 2e3c72c..f698026 100644
--- a/webmentions_ssg/forms/__init__.py
+++ b/webmentions_ssg/forms/__init__.py
@@ -10,7 +10,7 @@ from flask_wtf import FlaskForm
from wtforms import PasswordField, StringField, SubmitField
from wtforms.validators import URL, InputRequired, Regexp
-from .validators import AllowedHostname, NotEqualTo, PublicURL
+from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral
class LoginForm(FlaskForm):
@@ -42,7 +42,7 @@ class EndpointForm(FlaskForm):
message="source must begin with http or https",
),
NotEqualTo("target"),
- PublicURL(),
+ PublicIPLiteral(),
],
)
target = StringField(
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index dd0ee9b..e8f33eb 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -4,6 +4,7 @@
"""Provide validators for URL safety and form field constraints."""
+import ipaddress
from urllib.parse import urlsplit
from flask import current_app
@@ -85,7 +86,10 @@ class AllowedHostname:
:param field: Field containing the URL to validate.
:raises ValidationError: If the URL hostname is not configured as allowed.
"""
- hostname = urlsplit(field.data).hostname
+ try:
+ hostname = urlsplit(field.data).hostname
+ except (ValueError, TypeError):
+ hostname = None
if hostname is not None and hostname in current_app.config["ALLOWED_HOSTNAMES"]:
return
raise ValidationError(self.message or field.gettext("Invalid input."))
@@ -125,3 +129,24 @@ class PublicURL:
raise ValidationError(message)
except (AddressResolutionError, ValueError) as exc:
raise ValidationError(message) from exc
+
+
+class PublicIPLiteral:
+ """Reject non-public IP literals without performing DNS in the web worker."""
+
+ def __call__(self, form: BaseForm, field: Field) -> None:
+ try:
+ hostname = urlsplit(field.data).hostname
+ except (ValueError, TypeError) as exc:
+ raise ValidationError(field.gettext("Invalid input.")) from exc
+
+ if hostname is None:
+ raise ValidationError(field.gettext("Invalid input."))
+
+ try:
+ address = ipaddress.ip_address(hostname)
+ except ValueError:
+ return # Named hosts are resolved only during the background fetch.
+
+ if not address.is_global:
+ raise ValidationError(field.gettext("URL must use a public address."))
diff --git a/webmentions_ssg/public_transport.py b/webmentions_ssg/public_transport.py
new file mode 100644
index 0000000..131cf55
--- /dev/null
+++ b/webmentions_ssg/public_transport.py
@@ -0,0 +1,144 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""HTTP transport that connects only to validated DNS answers.
+
+httpcore receives the original host for Host and TLS SNI/certificate checks,
+but the socket is opened to an IP from a single, bounded DNS lookup. This
+protects the receiver and sender, including redirects and advertised URLs.
+"""
+
+import ipaddress
+import socket
+import time
+from collections.abc import Iterable
+
+import httpcore
+import httpx
+from httpcore._backends.sync import SyncStream # httpcore 1.x socket implementation
+
+from .url_security import public_addresses
+
+
+class NetworkDeadline:
+ """One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""
+
+ def __init__(self, seconds: float) -> None:
+ if seconds <= 0:
+ raise ValueError("REQUEST_TOTAL_TIMEOUT must be greater than zero")
+ self.expires = time.monotonic() + seconds
+
+ def timeout(self, original: float | None, error: type[Exception]) -> float:
+ remaining = self.expires - time.monotonic()
+ if remaining <= 0:
+ raise error("Overall HTTP request deadline exceeded")
+ return min(remaining, original) if original is not None else remaining
+
+
+class DeadlineStream(httpcore.NetworkStream):
+ """Clip every read/write/TLS operation to the remaining overall deadline."""
+
+ def __init__(self, stream: httpcore.NetworkStream, deadline: NetworkDeadline):
+ self._stream = stream
+ self._deadline = deadline
+
+ def read(self, max_bytes: int, timeout: float | None = None) -> bytes:
+ return self._stream.read(
+ max_bytes, timeout=self._deadline.timeout(timeout, httpcore.ReadTimeout)
+ )
+
+ def write(self, buffer: bytes, timeout: float | None = None) -> None:
+ self._stream.write(
+ buffer, timeout=self._deadline.timeout(timeout, httpcore.WriteTimeout)
+ )
+
+ def start_tls(
+ self,
+ ssl_context,
+ server_hostname: str | None = None,
+ timeout: float | None = None,
+ ) -> httpcore.NetworkStream:
+ return DeadlineStream(
+ self._stream.start_tls(
+ ssl_context,
+ server_hostname=server_hostname,
+ timeout=self._deadline.timeout(timeout, httpcore.ConnectTimeout),
+ ),
+ self._deadline,
+ )
+
+ def get_extra_info(self, info: str):
+ return self._stream.get_extra_info(info)
+
+ def close(self) -> None:
+ self._stream.close()
+
+
+class PublicNetworkBackend(httpcore.SyncBackend):
+ """Resolve and validate in the TCP connection operation, never in a hook."""
+
+ def __init__(self, deadline: NetworkDeadline):
+ self._deadline = deadline
+
+ def connect_tcp(
+ self,
+ host: str,
+ port: int,
+ timeout: float | None = None,
+ local_address: str | None = None,
+ socket_options: Iterable[tuple] | None = None,
+ ) -> httpcore.NetworkStream:
+ # No second lookup of `host` is permitted. Even a DNS response that
+ # changes immediately after this check cannot change the socket peer.
+ dns_timeout = self._deadline.timeout(timeout, httpcore.ConnectTimeout)
+ addresses = public_addresses(host, lifetime=dns_timeout)
+ last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
+
+ for address in addresses:
+ family = (
+ socket.AF_INET6
+ if ipaddress.ip_address(address).version == 6
+ else socket.AF_INET
+ )
+ sock = socket.socket(family, socket.SOCK_STREAM)
+ try:
+ sock.settimeout(
+ self._deadline.timeout(timeout, httpcore.ConnectTimeout)
+ )
+ if local_address is not None:
+ sock.bind((local_address, 0))
+ for option in socket_options or ():
+ sock.setsockopt(*option)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ # This numeric address is the *same* address just validated;
+ # no getaddrinfo or DNS resolver runs between check and connect.
+ sock.connect((address, port))
+ return DeadlineStream(SyncStream(sock), self._deadline)
+ except (TimeoutError, OSError) as exc:
+ sock.close()
+ if isinstance(exc, TimeoutError):
+ last_error = httpcore.ConnectTimeout(str(exc))
+ else:
+ last_error = httpcore.ConnectError(str(exc))
+ except BaseException:
+ sock.close()
+ raise
+
+ assert last_error is not None
+ raise last_error
+
+
+class PublicHTTPTransport(httpx.HTTPTransport):
+ """A direct (non-proxy) httpx transport with an IP-pinning network backend.
+
+ httpx 0.28 does not expose httpcore's ``network_backend`` parameter, so we
+ initialize its compatible pool directly. Pin the httpx/httpcore major APIs
+ and keep this integration under test when upgrading either dependency.
+ """
+
+ def __init__(self, total_timeout: float):
+ self._pool = httpcore.ConnectionPool(
+ ssl_context=httpx.create_ssl_context(verify=True, trust_env=False),
+ network_backend=PublicNetworkBackend(NetworkDeadline(total_timeout)),
+ )
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 4e8df74..f999d1b 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -18,7 +18,8 @@ from .. import APP_NAME, VERSION
from .. import DATABASE as db
from .. import HUEY as huey
from ..models import ReceivedWebmention
-from ..url_security import AddressResolutionError, is_public_url
+from ..public_transport import PublicHTTPTransport
+from ..url_security import AddressResolutionError, UnsafeAddressError
class VerificationError(Exception):
@@ -124,24 +125,6 @@ def text_mentions_target(body: str, target_url: str) -> bool:
return False
-def ensure_public_request(request: httpx.Request) -> None:
- """
- Ensure that an HTTP request targets a public network address.
-
- :param request: HTTP request to validate.
- :raises VerificationError: If the URL has no hostname or resolves to a
- non-public address.
- :raises TemporaryFetchError: If the hostname cannot be resolved.
- """
- try:
- if not is_public_url(str(request.url)):
- raise VerificationError("Source resolves to a non-public address")
- except ValueError as exc:
- raise VerificationError("Source URL has no hostname") from exc
- except AddressResolutionError as exc:
- raise TemporaryFetchError("Source hostname could not be resolved") from exc
-
-
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
Fetch a webmention source document.
@@ -157,6 +140,15 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
status or exceeds the configured maximum response size.
"""
+ try:
+ return _fetch_source(source_url)
+ except UnsafeAddressError as exc:
+ raise VerificationError("Source resolves to a non-public address") from exc
+ except AddressResolutionError as exc:
+ raise TemporaryFetchError("Source hostname could not be resolved") from exc
+
+
+def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
with (
httpx.Client(
headers={
@@ -167,7 +159,9 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
follow_redirects=True,
max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
trust_env=False,
- event_hooks={"request": [ensure_public_request]},
+ transport=PublicHTTPTransport(
+ current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
+ ),
) as client,
client.stream("GET", source_url) as response,
):
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index 725b5c8..7121f6a 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -19,7 +19,8 @@ from .. import APP_NAME, VERSION
from .. import DATABASE as db
from .. import HUEY as huey
from ..models import SentWebmention, SentWebmentionStatus
-from ..url_security import AddressResolutionError, is_http_url, is_public_url
+from ..public_transport import PublicHTTPTransport
+from ..url_security import AddressResolutionError, UnsafeAddressError, is_http_url
LINK_SPLIT = re.compile(r",\s*(?=<)")
@@ -48,21 +49,6 @@ def temporary_http_status(status: int) -> bool:
return status in {408, 425, 429} or 500 <= status <= 599
-def ensure_public_request(request: httpx.Request) -> None:
- """
- Ensure that an HTTP request targets a public network address.
-
- :param request: HTTP request to validate.
- :raises PermanentSenderError: If the request resolves to a non-public address.
- :raises TemporarySenderError: If the request hostname cannot be resolved.
- """
- try:
- if not is_public_url(str(request.url)):
- raise PermanentSenderError("Request resolves to a non-public address")
- except AddressResolutionError as exc:
- raise TemporarySenderError("Request hostname could not be resolved") from exc
-
-
def resolve_endpoint(response: httpx.Response, href: str) -> str:
"""
Resolve and validate a discovered Webmention endpoint.
@@ -224,7 +210,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
"""
try:
head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
+ except (httpx.RequestError, TemporarySenderError):
# HEAD is only an optimization. Servers and intermediaries may reject
# or drop it even when fetching the document with GET would succeed.
pass
@@ -442,7 +428,9 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
follow_redirects=True,
max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
trust_env=False,
- event_hooks={"request": [ensure_public_request]},
+ transport=PublicHTTPTransport(
+ current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
+ ),
) as client:
endpoint = discover_webmention_endpoint(client, target)
@@ -472,7 +460,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
raise PermanentSenderError(message)
- except (TemporarySenderError, httpx.RequestError) as exc:
+ except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
if not attempt_is_current(webmention, revision):
return
@@ -495,7 +483,7 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
# No Huey retries: only the scanner may requeue after next_retry_at.
return
- except PermanentSenderError as exc:
+ except (PermanentSenderError, UnsafeAddressError) as exc:
if not attempt_is_current(webmention, revision):
return
diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py
index ade64d6..f943c7e 100644
--- a/webmentions_ssg/url_security.py
+++ b/webmentions_ssg/url_security.py
@@ -12,17 +12,51 @@ import dns.resolver
class AddressResolutionError(Exception):
- pass
+ """A hostname could not be resolved (potentially transient)."""
-def is_public_url(url: str) -> bool:
+class UnsafeAddressError(ValueError):
+ """A hostname or IP literal identifies a non-public network address."""
+
+
+def public_addresses(hostname: str, *, lifetime: float | None = None) -> tuple[str, ...]:
+ """Resolve *once* and return only checked, globally routable IP addresses.
+
+ A DNS name with a mixture of public and private answers is rejected in its
+ entirety. IP literals are handled without DNS, including IPv6 literals.
+ The returned strings must be used for the actual TCP connection; checking
+ them and then connecting to ``hostname`` reintroduces DNS rebinding.
"""
- Check whether a URL resolves exclusively to public IP addresses.
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ addresses = (literal,)
+ else:
+ try:
+ answers = dns.resolver.resolve_name(hostname, lifetime=lifetime)
+ addresses = tuple(ipaddress.ip_address(a) for a in answers.addresses())
+ except (dns.exception.DNSException, ValueError) as exc:
+ raise AddressResolutionError(
+ f"Could not resolve hostname {hostname!r}"
+ ) from exc
+
+ if not addresses:
+ raise AddressResolutionError(f"Could not resolve hostname {hostname!r}")
- :param url: URL whose hostname should be resolved.
- :return: Whether all resolved addresses are globally routable.
- :raises ValueError: If the URL does not contain a hostname.
- :raises AddressResolutionError: If the hostname cannot be resolved.
+ if any(not address.is_global for address in addresses):
+ raise UnsafeAddressError(f"Non-public network address for {hostname!r}")
+
+ return tuple(str(address) for address in addresses)
+
+
+def is_public_url(url: str) -> bool:
+ """Return whether the URL resolves entirely to public IP addresses.
+
+ This is useful for diagnostics, but **not** sufficient to authorize a
+ request: network connections must use the exact validated IP address.
"""
hostname = urlsplit(url).hostname
@@ -30,17 +64,10 @@ def is_public_url(url: str) -> bool:
raise ValueError("No hostname was specified")
try:
- answers = dns.resolver.resolve_name(hostname)
- except dns.exception.DNSException as exc:
- raise AddressResolutionError(
- f"Could not resolve hostname {hostname!r}"
- ) from exc
- else:
- for address in answers.addresses():
- if not ipaddress.ip_address(address).is_global:
- return False
-
- return True
+ public_addresses(hostname)
+ except UnsafeAddressError:
+ return False
+ return True
def is_http_url(url: str) -> bool: