aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDennis Fink2026-11-13 20:03:49 +0100
committerDennis Fink2026-11-13 20:03:49 +0100
commit87117345ed6e99e5a070d919ac17139cba7b8fe4 (patch)
treed7df5d45008dd2959194e2614b87333598b85198
parent2fbe1d383650cb31f3ef8d58c04e4d4095935833 (diff)
downloadwebmentions-ssg-87117345ed6e99e5a070d919ac17139cba7b8fe4.tar.gz
webmentions-ssg-87117345ed6e99e5a070d919ac17139cba7b8fe4.zip
fix(core): make network task failures explicit
Introduce a dedicated response-size error so callers only translate expected limit violations instead of masking unrelated `ValueError` exceptions. Ensure failed sockets are always closed, remove dead discovery error handling, and log unexpected sender failures with tracebacks before recording them.
-rw-r--r--tests/tasks/test_discovery.py26
-rw-r--r--tests/tasks/test_receiver.py19
-rw-r--r--tests/tasks/test_sender.py23
-rw-r--r--tests/test_net.py59
-rw-r--r--webmentions_ssg/net.py81
-rw-r--r--webmentions_ssg/tasks/discovery.py11
-rw-r--r--webmentions_ssg/tasks/receiver.py3
-rw-r--r--webmentions_ssg/tasks/sender.py12
8 files changed, 191 insertions, 43 deletions
diff --git a/tests/tasks/test_discovery.py b/tests/tasks/test_discovery.py
index 2d6c37e..7c28108 100644
--- a/tests/tasks/test_discovery.py
+++ b/tests/tasks/test_discovery.py
@@ -569,7 +569,7 @@ def test_discovery_falls_back_to_get_after_head_request_error(
assert methods == ["HEAD", "GET"]
-def test_discovery_falls_back_to_get_after_head_dns_failure(
+def test_discovery_does_not_suppress_unexpected_head_error(
app: Flask, sender_module: ModuleType
) -> None:
methods: list[str] = []
@@ -588,11 +588,13 @@ def test_discovery_falls_back_to_get_after_head_dns_failure(
transport=httpx.MockTransport(handler),
event_hooks={"request": [request_hook]},
) as client,
+ pytest.raises(
+ sender_module.TemporarySenderError, match="HEAD DNS lookup failed"
+ ),
):
- endpoint = sender_module.discover_webmention_endpoint(client, TARGET_URL)
+ sender_module.discover_webmention_endpoint(client, TARGET_URL)
- assert endpoint == ENDPOINT_URL
- assert methods == ["GET"]
+ assert methods == []
def test_discovery_get_error_after_head_error_is_not_suppressed(
@@ -753,6 +755,22 @@ def test_read_target_body_rejects_declared_oversized_body(
sender_module.read_target_body(response)
+def test_read_target_body_does_not_mask_unexpected_value_error(
+ app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(
+ sender_module,
+ "read_limited_body",
+ Mock(side_effect=ValueError("unrelated processing error")),
+ )
+
+ with (
+ app.app_context(),
+ pytest.raises(ValueError, match="unrelated processing error"),
+ ):
+ sender_module.read_target_body(httpx.Response(200))
+
+
def test_read_target_body_ignores_invalid_content_length(
app: Flask, sender_module: ModuleType
) -> None:
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index 133ab56..51d1bb6 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -198,6 +198,25 @@ def test_fetch_source_rejects_streamed_oversized_body(
receiver.fetch_source(SOURCE_URL)
+@patch("webmentions_ssg.tasks.receiver.httpx.Client")
+def test_fetch_source_does_not_mask_unexpected_value_error(
+ httpx_client: Mock,
+ app: Flask,
+ receiver: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ set_stream_response(
+ httpx_client,
+ httpx.Response(200, request=httpx.Request("GET", SOURCE_URL)),
+ )
+ monkeypatch.setattr(
+ receiver, "read_limited_body", Mock(side_effect=ValueError("unexpected bug"))
+ )
+
+ with app.app_context(), pytest.raises(ValueError, match="unexpected bug"):
+ receiver.fetch_source(SOURCE_URL)
+
+
@patch("webmentions_ssg.tasks.receiver.fetch_source")
@pytest.mark.parametrize(
("content_type", "body", "expected"),
diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py
index 94523e5..818092b 100644
--- a/tests/tasks/test_sender.py
+++ b/tests/tasks/test_sender.py
@@ -431,6 +431,29 @@ def test_sender_malformed_request_is_terminal(
assert post.call_count == (1 if stage == "post" else 0)
+def test_sender_logs_unexpected_value_error_traceback(
+ app: Flask,
+ sender_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ identifier = create_sent_webmention(app)
+ _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch)
+ discover.side_effect = ValueError("unexpected parsing bug")
+
+ with caplog.at_level(logging.ERROR):
+ sender_module.send_webmention.call_local(identifier)
+
+ assert f"Unexpected error sending SentWebmention {identifier}" in caplog.text
+ assert "ValueError: unexpected parsing bug" in caplog.text
+ with app.app_context():
+ webmention = db.session.get(SentWebmention, identifier)
+ assert webmention is not None
+ assert webmention.status == SentWebmentionStatus.FAILED
+ assert webmention.failure_reason == "unexpected parsing bug"
+ assert webmention.processed_revision == 1
+
+
@pytest.mark.parametrize(
("target", "reason"),
[
diff --git a/tests/test_net.py b/tests/test_net.py
index 05ae5a4..8e0744c 100644
--- a/tests/test_net.py
+++ b/tests/test_net.py
@@ -19,10 +19,12 @@ from webmentions_ssg.net import (
NetworkDeadline,
PublicHTTPTransport,
PublicNetworkBackend,
+ ResponseTooLargeError,
UnsafeAddressError,
is_http_url,
is_public_address,
public_addresses,
+ read_limited_body,
)
@@ -60,6 +62,63 @@ def test_rebinding_cannot_change_socket_peer(monkeypatch: pytest.MonkeyPatch) ->
create_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
+def test_backend_closes_failed_socket_and_tries_next_address(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ "webmentions_ssg.net.public_addresses",
+ Mock(return_value=("93.184.216.34", "8.8.8.8")),
+ )
+ first, second = Mock(), Mock()
+ first.connect.side_effect = TimeoutError("connection timed out")
+ monkeypatch.setattr(socket, "socket", Mock(side_effect=[first, second]))
+
+ stream = PublicNetworkBackend(NetworkDeadline(5)).connect_tcp(
+ "example.com", 443
+ )
+
+ assert isinstance(stream, DeadlineStream)
+ first.close.assert_called_once_with()
+ second.close.assert_not_called()
+ first.connect.assert_called_once_with(("93.184.216.34", 443))
+ second.connect.assert_called_once_with(("8.8.8.8", 443))
+ stream.close()
+ second.close.assert_called_once_with()
+
+
+@pytest.mark.parametrize(
+ ("failure", "expected"),
+ [
+ pytest.param(TimeoutError("timed out"), httpcore.ConnectTimeout, id="timeout"),
+ pytest.param(OSError("refused"), httpcore.ConnectError, id="os-error"),
+ pytest.param(ValueError("unexpected"), ValueError, id="unexpected"),
+ ],
+)
+def test_backend_closes_socket_on_failure(
+ monkeypatch: pytest.MonkeyPatch, failure: Exception, expected: type[Exception]
+) -> None:
+ sock = Mock()
+ sock.connect.side_effect = failure
+ monkeypatch.setattr(socket, "socket", Mock(return_value=sock))
+
+ with pytest.raises(expected, match=str(failure)):
+ PublicNetworkBackend(NetworkDeadline(5)).connect_tcp("8.8.8.8", 443)
+
+ sock.close.assert_called_once_with()
+
+
+@pytest.mark.parametrize("mode", ["declared", "streamed"])
+def test_read_limited_body_raises_specific_size_error(mode: str) -> None:
+ response = (
+ httpx.Response(200, headers={"Content-Length": "11"}, content=b"x" * 11)
+ if mode == "declared"
+ else httpx.Response(200, stream=httpx.ByteStream(b"x" * 11))
+ )
+
+ with pytest.raises(ResponseTooLargeError, match="Source document is too large"):
+ read_limited_body(response, 10, label="Source")
+
+
class FakeStream(httpcore.NetworkStream):
"""In-memory TCP/TLS stream so HTTP framing and SNI can be tested."""
diff --git a/webmentions_ssg/net.py b/webmentions_ssg/net.py
index 9a6ecfb..0314810 100644
--- a/webmentions_ssg/net.py
+++ b/webmentions_ssg/net.py
@@ -8,6 +8,7 @@ import ipaddress
import socket
import time
from collections.abc import Iterable
+from contextlib import ExitStack
from urllib.parse import urlsplit
import dns.exception
@@ -29,6 +30,10 @@ class UnsafeAddressError(ValueError):
"""A hostname or IP literal identifies a non-public network address."""
+class ResponseTooLargeError(ValueError):
+ """A response body exceeds its configured size limit."""
+
+
class NetworkDeadline:
"""One monotonic deadline shared by DNS, TCP, TLS, and response streaming."""
@@ -89,6 +94,43 @@ class PublicNetworkBackend(httpcore.SyncBackend):
def __init__(self, deadline: NetworkDeadline):
self._deadline = deadline
+ def _connect_one(
+ self,
+ address: str,
+ port: int,
+ timeout: float | None,
+ local_address: str | None,
+ socket_options: Iterable[tuple] | None,
+ ) -> httpcore.NetworkStream:
+ family = (
+ socket.AF_INET6
+ if ipaddress.ip_address(address).version == 6
+ else socket.AF_INET
+ )
+
+ # Close the socket on any error, transferring ownership to SyncStream
+ # only after the connection and stream have been created successfully.
+ with ExitStack() as cleanup:
+ sock = socket.socket(family, socket.SOCK_STREAM)
+ cleanup.callback(sock.close)
+ sock.settimeout(self._deadline.timeout(timeout, httpcore.ConnectTimeout))
+
+ if local_address is not None:
+ sock.bind((local_address, 0))
+
+ for option in socket_options or ():
+ sock.setsockopt(*option)
+
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ # This numeric address is the *same* address just validated;
+ # no getaddrinfo or DNS resolver runs between check and connect.
+ sock.connect((address, port))
+
+ stream = DeadlineStream(SyncStream(sock), self._deadline)
+ cleanup.pop_all()
+ return stream
+
def connect_tcp(
self,
host: str,
@@ -104,43 +146,18 @@ class PublicNetworkBackend(httpcore.SyncBackend):
last_error: httpcore.ConnectError | httpcore.ConnectTimeout | None = None
for address in addresses:
- family = (
- socket.AF_INET6
- if ipaddress.ip_address(address).version == 6
- else socket.AF_INET
- )
- sock = socket.socket(family, socket.SOCK_STREAM)
-
try:
- sock.settimeout(
- self._deadline.timeout(timeout, httpcore.ConnectTimeout)
+ return self._connect_one(
+ address, port, timeout, local_address, socket_options
)
-
- if local_address is not None:
- sock.bind((local_address, 0))
-
- for option in socket_options or ():
- sock.setsockopt(*option)
-
- sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-
- # This numeric address is the *same* address just validated;
- # no getaddrinfo or DNS resolver runs between check and connect.
- sock.connect((address, port))
-
- return DeadlineStream(SyncStream(sock), self._deadline)
- except (TimeoutError, OSError) as exc:
- sock.close()
-
+ except OSError as exc:
if isinstance(exc, TimeoutError):
last_error = httpcore.ConnectTimeout(str(exc))
else:
last_error = httpcore.ConnectError(str(exc))
- except BaseException:
- sock.close()
- raise
- assert last_error is not None
+ if last_error is None:
+ raise httpcore.ConnectError(f"No public addresses available for {host!r}")
raise last_error
@@ -256,13 +273,13 @@ def read_limited_body(response: httpx.Response, max_bytes: int, *, label: str) -
pass
else:
if declared_length > max_bytes:
- raise ValueError(f"{label} document is too large")
+ raise ResponseTooLargeError(f"{label} document is too large")
body = bytearray()
for chunk in response.iter_bytes(chunk_size=64 * 1024):
body.extend(chunk)
if len(body) > max_bytes:
- raise ValueError(f"{label} document is too large")
+ raise ResponseTooLargeError(f"{label} document is too large")
return bytes(body)
diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py
index 29c3e81..456e47e 100644
--- a/webmentions_ssg/tasks/discovery.py
+++ b/webmentions_ssg/tasks/discovery.py
@@ -11,7 +11,12 @@ import httpx
from bs4 import BeautifulSoup
from flask import current_app
-from ..net import is_http_url, is_temporary_status, read_limited_body
+from ..net import (
+ ResponseTooLargeError,
+ is_http_url,
+ is_temporary_status,
+ read_limited_body,
+)
LINK_SPLIT = re.compile(r",\s*(?=<)")
DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"}
@@ -130,7 +135,7 @@ def read_target_body(response: httpx.Response) -> bytes:
return read_limited_body(
response, current_app.config["MAX_TARGET_BYTES"], label="Target"
)
- except ValueError as exc:
+ except ResponseTooLargeError as exc:
raise PermanentSenderError(str(exc)) from exc
@@ -149,7 +154,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non
"""
try:
head_response = client.head(target, headers=DISCOVERY_HEADERS)
- except httpx.RequestError, TemporarySenderError:
+ except httpx.RequestError:
# HEAD is only an optimization. Servers and intermediaries may reject
# or drop it even when fetching the document with GET would succeed.
pass
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index 2963dc7..cbb42b6 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -16,6 +16,7 @@ from ..extensions import db, huey
from ..models import ReceivedStatus, ReceivedWebmention
from ..net import (
AddressResolutionError,
+ ResponseTooLargeError,
UnsafeAddressError,
build_client,
is_temporary_status,
@@ -76,7 +77,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
return response, read_limited_body(
response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
)
- except ValueError as exc:
+ except ResponseTooLargeError as exc:
raise VerificationError(str(exc)) from exc
except UnsafeAddressError as exc:
diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py
index db99476..cd6c64a 100644
--- a/webmentions_ssg/tasks/sender.py
+++ b/webmentions_ssg/tasks/sender.py
@@ -210,10 +210,8 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
httpx.InvalidURL,
httpx.UnsupportedProtocol,
httpx.LocalProtocolError,
- ValueError,
- UnicodeError,
) as exc:
- # Invalid URLs and local protocol errors cannot improve with retries.
+ # Known permanent failures cannot improve with retries.
outcome = SentWebmentionStatus.FAILED
reason = str(exc) or type(exc).__name__
except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc:
@@ -221,6 +219,14 @@ def send_webmention_task(webmention_uuid: uuid.UUID) -> None:
reason = str(exc) or "Webmention request failed"
terminal = attempt >= max_attempts
temporary_failure = True
+ except Exception as exc:
+ # Unexpected errors must not disappear into a generic permanent-failure
+ # handler without a traceback (including programming ValueErrors).
+ current_app.logger.exception(
+ "Unexpected error sending SentWebmention %s", webmention_uuid
+ )
+ outcome = SentWebmentionStatus.FAILED
+ reason = str(exc) or type(exc).__name__
# A scanner may have reopened or superseded the revision during I/O.
# Refresh only once, and persist one outcome atomically with its metadata.