# SPDX-FileCopyrightText: 2026 Dennis Fink # # SPDX-License-Identifier: BSD-3-Clause import logging import uuid from datetime import UTC, datetime, timedelta from http import HTTPStatus from types import ModuleType from unittest.mock import MagicMock, Mock import httpx import pytest import sqlalchemy as sa from flask import Flask from webmentions_ssg import db from webmentions_ssg.models.sources import ( SentWebmention, SentWebmentionStatus, SentWebmentionStatusEvent, SourceDocument, ) SOURCE_URL = "https://dennisfink.me/blog/example/" TARGET_URL = "https://example.com/post" ENDPOINT_URL = "https://example.com/webmention" STATUS_URL = "https://example.com/webmention/status/123" def create_sent_webmention( app: Flask, *, linked: bool = True, desired_revision: int = 1, processed_revision: int | None = None, sent_revision: int | None = None, status: SentWebmentionStatus | None = None, failure_reason: str | None = None, endpoint: str | None = None, response_status: int | None = None, status_url: str | None = None, attempt_count: int = 0, next_retry_at: datetime | None = None, ) -> uuid.UUID: now = datetime.now(UTC) with app.app_context(): source = SourceDocument( path="example/index.html", url=SOURCE_URL, content_hash="0" * 32, revision=desired_revision, last_seen_at=now, revised_at=now, ) webmention = SentWebmention( target=TARGET_URL, linked=linked, desired_revision=desired_revision, processed_revision=processed_revision, sent_revision=sent_revision, status=status, failure_reason=failure_reason, webmention_endpoint=endpoint, endpoint_response_status=response_status, status_url=status_url, attempt_count=attempt_count, next_retry_at=next_retry_at, ) source.sent_webmentions.append(webmention) db.session.add(source) db.session.commit() return webmention.uuid def mock_sender_requests( sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch, *, endpoint: str | None = ENDPOINT_URL, result: tuple[int, str | None] = (HTTPStatus.ACCEPTED, None), ) -> tuple[Mock, Mock, Mock, Mock]: httpx_client = MagicMock() client = httpx_client.return_value.__enter__.return_value discover = Mock(return_value=endpoint) post = Mock(return_value=result) monkeypatch.setattr(sender_module.httpx, "Client", httpx_client) monkeypatch.setattr(sender_module, "discover_webmention_endpoint", discover) monkeypatch.setattr(sender_module, "post_webmention", post) return httpx_client, client, discover, post @pytest.fixture def sender_module(app: Flask) -> ModuleType: _ = app from webmentions_ssg.tasks import sender return sender @pytest.mark.parametrize( ("desired_revision", "processed_revision", "revision", "expected"), [ pytest.param(2, None, 2, True, id="unprocessed-current"), pytest.param(2, 1, 2, True, id="older-processed-current"), pytest.param(2, 2, 2, False, id="already-processed"), pytest.param(2, 3, 2, False, id="newer-processed"), pytest.param(3, 1, 2, False, id="superseded"), ], ) def test_revision_still_pending( app: Flask, sender_module: ModuleType, desired_revision: int, processed_revision: int | None, revision: int, expected: bool, ) -> None: identifier = create_sent_webmention( app, desired_revision=desired_revision, processed_revision=processed_revision ) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert sender_module.revision_still_pending(webmention, revision) is expected def test_attempt_delivery_returns_retryable_outcome( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: mock_sender_requests( sender_module, monkeypatch, result=(HTTPStatus.SERVICE_UNAVAILABLE, None) ) with app.app_context(): result = sender_module.attempt_delivery( SOURCE_URL, TARGET_URL, webmention_uuid=uuid.uuid7() ) assert result == sender_module.DeliveryResult( status=SentWebmentionStatus.FAILED, reason="Webmention endpoint returned HTTP 503", endpoint=ENDPOINT_URL, response_status=HTTPStatus.SERVICE_UNAVAILABLE, retryable=True, ) def test_send_webmention_persists_success( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention(app) _, client, discover, post = mock_sender_requests( sender_module, monkeypatch, result=(HTTPStatus.CREATED, STATUS_URL) ) sender_module.send_webmention.call_local(identifier) discover.assert_called_once_with(client, TARGET_URL) post.assert_called_once_with( client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL ) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.processed_revision == 1 assert webmention.sent_revision == 1 assert webmention.status == SentWebmentionStatus.SENT assert webmention.failure_reason is None assert webmention.webmention_endpoint == ENDPOINT_URL assert webmention.endpoint_response_status == HTTPStatus.CREATED assert webmention.status_url == STATUS_URL assert webmention.last_attempted_at is not None assert webmention.last_sent_at is not None assert webmention.attempt_count == 1 assert webmention.next_retry_at is None history = db.session.scalars( sa.select(SentWebmentionStatusEvent).where( SentWebmentionStatusEvent.webmention_uuid == identifier ) ).all() assert len(history) == 1 assert history[0].action == "Delivery succeeded" assert history[0].status == SentWebmentionStatus.SENT assert history[0].attempt_count == 1 assert history[0].endpoint_response_status == HTTPStatus.CREATED def test_send_webmention_marks_unsupported_target_processed( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( app, desired_revision=2, processed_revision=1, sent_revision=1, status=SentWebmentionStatus.SENT, endpoint="https://old.example/webmention", response_status=HTTPStatus.CREATED, status_url="https://old.example/status/123", ) _, client, discover, post = mock_sender_requests( sender_module, monkeypatch, endpoint=None ) sender_module.send_webmention.call_local(identifier) discover.assert_called_once_with(client, TARGET_URL) post.assert_not_called() with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.processed_revision == 2 assert webmention.sent_revision == 1 assert webmention.status == SentWebmentionStatus.NO_ENDPOINT assert webmention.failure_reason == "No Webmention endpoint discovered" assert webmention.webmention_endpoint is None assert webmention.endpoint_response_status is None assert webmention.status_url is None assert webmention.last_attempted_at is not None history = db.session.scalars(sa.select(SentWebmentionStatusEvent)).all() assert len(history) == 1 assert history[0].action == "No endpoint discovered" assert history[0].status == SentWebmentionStatus.NO_ENDPOINT def test_send_webmention_persists_permanent_failure( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( app, desired_revision=2, processed_revision=1, sent_revision=1, status=SentWebmentionStatus.SENT, status_url="https://old.example/status/123", ) mock_sender_requests( sender_module, monkeypatch, result=(HTTPStatus.BAD_REQUEST, None) ) sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.processed_revision == 2 assert webmention.sent_revision == 1 assert webmention.status == SentWebmentionStatus.FAILED assert webmention.failure_reason == "Webmention endpoint returned HTTP 400" assert webmention.webmention_endpoint == ENDPOINT_URL assert webmention.endpoint_response_status == HTTPStatus.BAD_REQUEST assert webmention.status_url is None history = db.session.scalars(sa.select(SentWebmentionStatusEvent)).all() assert len(history) == 1 assert history[0].action == "Delivery failed" assert history[0].failure_reason == webmention.failure_reason def test_send_webmention_persists_temporary_failure_with_backoff( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( app, desired_revision=2, processed_revision=1, sent_revision=1, status=SentWebmentionStatus.SENT, status_url="https://old.example/status/123", ) mock_sender_requests( sender_module, monkeypatch, result=(HTTPStatus.SERVICE_UNAVAILABLE, None) ) sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.processed_revision == 1 assert webmention.sent_revision == 1 assert webmention.status == SentWebmentionStatus.FAILED assert webmention.failure_reason == "Webmention endpoint returned HTTP 503" assert webmention.webmention_endpoint == ENDPOINT_URL assert webmention.endpoint_response_status == HTTPStatus.SERVICE_UNAVAILABLE assert webmention.status_url is None assert webmention.attempt_count == 1 assert webmention.next_retry_at is not None assert webmention.last_attempted_at is not None # SQLite drops timezone info when loading timestamp columns. assert ( webmention.next_retry_at.replace(tzinfo=None) - webmention.last_attempted_at.replace(tzinfo=None) ).total_seconds() == 3600 history = db.session.scalars(sa.select(SentWebmentionStatusEvent)).all() assert len(history) == 1 assert history[0].action == "Delivery failed; automatic retry scheduled" assert history[0].attempt_count == 1 def test_backoff_delay_is_exponential_and_capped( app: Flask, sender_module: ModuleType ) -> None: with app.app_context(): assert [ sender_module.backoff_delay(n).total_seconds() for n in range(1, 8) ] == [3600, 7200, 14400, 28800, 57600, 86400, 86400] def test_temporary_failure_backoff_suppresses_duplicate_tasks_and_caps_attempts( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: app.config["SEND_MAX_ATTEMPTS"] = 3 identifier = create_sent_webmention(app) _, _, discover, post = mock_sender_requests( sender_module, monkeypatch, result=(HTTPStatus.SERVICE_UNAVAILABLE, None) ) for attempt in range(1, 4): sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.attempt_count == attempt assert webmention.status == SentWebmentionStatus.FAILED assert webmention.processed_revision == (1 if attempt == 3 else None) if attempt < 3: assert webmention.next_retry_at is not None sender_module.send_webmention.call_local(identifier) assert webmention.attempt_count == attempt webmention.next_retry_at = datetime.now(UTC) - timedelta(seconds=1) db.session.commit() else: assert webmention.next_retry_at is None # A terminal failure stays terminal across scans or stray queued tasks. sender_module.send_webmention.call_local(identifier) assert discover.call_count == 3 assert post.call_count == 3 def test_sender_does_not_retry_before_due_even_after_restart( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( app, attempt_count=2, next_retry_at=datetime.now(UTC) + timedelta(days=1) ) httpx_client, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) sender_module.send_webmention.call_local(identifier) httpx_client.assert_not_called() discover.assert_not_called() with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.attempt_count == 2 assert webmention.processed_revision is None def test_sender_finishes_abandoned_final_attempt_without_new_request( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: app.config["SEND_MAX_ATTEMPTS"] = 2 identifier = create_sent_webmention( app, attempt_count=2, next_retry_at=datetime.now(UTC) - timedelta(seconds=1), status=SentWebmentionStatus.FAILED, failure_reason="Network unreachable", ) httpx_client, _, _, _ = mock_sender_requests(sender_module, monkeypatch) sender_module.send_webmention.call_local(identifier) httpx_client.assert_not_called() with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.processed_revision == 1 assert webmention.failure_reason == "Network unreachable" assert webmention.next_retry_at is None def test_sender_request_error_uses_same_retry_budget( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: app.config["SEND_MAX_ATTEMPTS"] = 1 identifier = create_sent_webmention(app) _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) discover.side_effect = httpx.ConnectError("DNS lookup failed") sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.status == SentWebmentionStatus.FAILED assert webmention.failure_reason == "DNS lookup failed" assert webmention.processed_revision == 1 assert webmention.attempt_count == 1 assert webmention.next_retry_at is None @pytest.mark.parametrize( ("failure", "reason"), [ pytest.param( httpx.InvalidURL("Invalid IDNA hostname"), "Invalid IDNA hostname", id="invalid-url", ), pytest.param( httpx.UnsupportedProtocol("Unsupported URL scheme"), "Unsupported URL scheme", id="unsupported-protocol", ), pytest.param( httpx.LocalProtocolError("Invalid HTTP Host header"), "Invalid HTTP Host header", id="local-protocol-error", ), pytest.param(ValueError("Invalid port"), "Invalid port", id="value-error"), pytest.param( UnicodeError("Invalid URL encoding"), "Invalid URL encoding", id="unicode-error", ), pytest.param(httpx.InvalidURL(""), "InvalidURL", id="no-message"), ], ) @pytest.mark.parametrize("stage", ["discovery", "post"]) def test_sender_malformed_request_is_terminal( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch, failure: Exception, reason: str, stage: str, ) -> None: app.config["SEND_MAX_ATTEMPTS"] = 3 identifier = create_sent_webmention(app) httpx_client, _, discover, post = mock_sender_requests(sender_module, monkeypatch) if stage == "discovery": discover.side_effect = failure else: post.side_effect = failure sender_module.send_webmention.call_local(identifier) sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.status == SentWebmentionStatus.FAILED assert webmention.failure_reason == reason assert webmention.processed_revision == 1 assert webmention.attempt_count == 1 assert webmention.next_retry_at is None assert webmention.webmention_endpoint == ( ENDPOINT_URL if stage == "post" else None ) assert webmention.endpoint_response_status is None assert webmention.last_attempted_at is not None assert httpx_client.call_count == 1 assert discover.call_count == 1 assert post.call_count == (1 if stage == "post" else 0) def test_sender_logs_unexpected_value_error_traceback( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: identifier = create_sent_webmention(app) _, _, discover, _ = mock_sender_requests(sender_module, monkeypatch) discover.side_effect = ValueError("unexpected parsing bug") with caplog.at_level(logging.ERROR): sender_module.send_webmention.call_local(identifier) assert f"Unexpected error sending SentWebmention {identifier}" in caplog.text assert "ValueError: unexpected parsing bug" in caplog.text with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.status == SentWebmentionStatus.FAILED assert webmention.failure_reason == "unexpected parsing bug" assert webmention.processed_revision == 1 @pytest.mark.parametrize( ("target", "reason"), [ pytest.param( "https://ex☃mple.test/", "Invalid IDNA hostname", id="invalid-idna" ), pytest.param("https://example.com:abc/post", "Invalid port", id="invalid-port"), pytest.param( "ftp://example.com/post", "Invalid target URL", id="non-http-scheme" ), pytest.param( "http://[invalid/post", "Invalid target URL", id="invalid-ip-literal" ), ], ) def test_sender_rejects_malformed_stored_target_without_retry( app: Flask, sender_module: ModuleType, target: str, reason: str ) -> None: identifier = create_sent_webmention(app) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None webmention.target = target db.session.commit() sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.status == SentWebmentionStatus.FAILED assert reason in (webmention.failure_reason or "") assert webmention.processed_revision == 1 assert webmention.attempt_count == 1 assert webmention.next_retry_at is None def test_send_webmention_skips_duplicate_while_locked( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention(app) _, _, discover, post = mock_sender_requests(sender_module, monkeypatch) # Simulate a first worker already sending this Webmention. The second # worker must not make HTTP requests or consume an attempt. with sender_module.huey.lock_task(f"send-webmention-{identifier}"): sender_module.send_webmention.call_local(identifier) discover.assert_not_called() post.assert_not_called() with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.attempt_count == 0 assert webmention.processed_revision is None # Once the other worker releases the lock, a queued task can complete. sender_module.send_webmention.call_local(identifier) sender_module.send_webmention.call_local(identifier) discover.assert_called_once() post.assert_called_once() with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.attempt_count == 1 assert webmention.processed_revision == 1 def test_send_webmention_lock_is_scoped_to_uuid( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: first_id = create_sent_webmention(app) other_target = "https://example.com/another-post" with app.app_context(): first = db.session.get(SentWebmention, first_id) assert first is not None second = SentWebmention( source_uuid=first.source_uuid, target=other_target, desired_revision=1 ) db.session.add(second) db.session.commit() second_id = second.uuid _, _, discover, post = mock_sender_requests(sender_module, monkeypatch) with sender_module.huey.lock_task(f"send-webmention-{first_id}"): sender_module.send_webmention.call_local(first_id) sender_module.send_webmention.call_local(second_id) discover.assert_called_once() post.assert_called_once() assert post.call_args.kwargs["target"] == other_target with app.app_context(): first = db.session.get(SentWebmention, first_id) second = db.session.get(SentWebmention, second_id) assert first is not None and second is not None assert first.attempt_count == 0 assert first.processed_revision is None assert second.attempt_count == 1 assert second.processed_revision == 1 def test_send_webmention_ignores_processed_revision( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention( app, desired_revision=2, processed_revision=2, sent_revision=2, status=SentWebmentionStatus.SENT, ) httpx_client, _, discover, post = mock_sender_requests(sender_module, monkeypatch) sender_module.send_webmention.call_local(identifier) httpx_client.assert_not_called() discover.assert_not_called() post.assert_not_called() @pytest.mark.parametrize( "outcome", ["success", "no_endpoint", "temporary-failure", "permanent-failure"] ) def test_send_webmention_ignores_superseded_attempt( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch, outcome: str ) -> None: identifier = create_sent_webmention(app) _, _, discover, post = mock_sender_requests(sender_module, monkeypatch) def supersede_revision() -> None: webmention = db.session.get(SentWebmention, identifier) assert webmention is not None webmention.desired_revision = 2 db.session.commit() match outcome: case "success": def post_success(*args, **kwargs) -> tuple[int, None]: supersede_revision() return (HTTPStatus.ACCEPTED, None) post.side_effect = post_success case "no_endpoint": def discover_no_endpoint(*args, **kwargs) -> None: supersede_revision() discover.side_effect = discover_no_endpoint case "temporary-failure": def post_temporary_failure(*args, **kwargs) -> None: supersede_revision() raise sender_module.RetryableDeliveryError("Temporary failure") post.side_effect = post_temporary_failure case "permanent-failure": def post_permanent_failure(*args, **kwargs) -> tuple[int, None]: supersede_revision() return (HTTPStatus.BAD_REQUEST, None) post.side_effect = post_permanent_failure case _: raise AssertionError(f"Unexpected outcome: {outcome}") sender_module.send_webmention.call_local(identifier) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert webmention.desired_revision == 2 assert webmention.processed_revision is None assert webmention.sent_revision is None assert webmention.status is None def test_send_webmention_ignores_unknown_identifier( sender_module: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: identifier = uuid.uuid7() with caplog.at_level(logging.WARNING): sender_module.send_webmention.call_local(identifier) assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text @pytest.mark.parametrize( ("status_url", "expected"), [ ("javascript:alert(1)", None), ("data:text/html,evil", None), ("ftp://example.com/status", None), ("https://example.com/status", "https://example.com/status"), ], ) def test_send_does_not_persist_unsafe_status_url( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch, status_url: str, expected: str | None, ) -> None: identifier = create_sent_webmention(app) client_factory = MagicMock() monkeypatch.setattr(sender_module.httpx, "Client", client_factory) monkeypatch.setattr( sender_module, "discover_webmention_endpoint", Mock(return_value=ENDPOINT_URL) ) monkeypatch.setattr( sender_module, "post_webmention", Mock(return_value=(HTTPStatus.CREATED, status_url)), ) sender_module.send_webmention.call_local(identifier) with app.app_context(): mention = db.session.get(SentWebmention, identifier) assert mention is not None assert mention.status == SentWebmentionStatus.SENT assert mention.endpoint_response_status == HTTPStatus.CREATED assert mention.status_url == expected def test_reserve_send_attempt_rejects_lost_claim( app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch ) -> None: identifier = create_sent_webmention(app) monkeypatch.setattr(sender_module, "rows_affected", lambda result: 0) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None assert sender_module.reserve_send_attempt(webmention, datetime.now(UTC)) is None