aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--contrib/webmentions-ssg.example.cfg3
-rw-r--r--tests/cli/test_database.py8
-rw-r--r--tests/forms/test_init.py37
-rw-r--r--tests/tasks/test_receiver.py102
-rw-r--r--tests/test_views.py32
-rw-r--r--webmentions_ssg/config.py2
-rw-r--r--webmentions_ssg/forms/__init__.py10
-rw-r--r--webmentions_ssg/forms/validators.py30
-rw-r--r--webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py31
-rw-r--r--webmentions_ssg/models.py5
-rw-r--r--webmentions_ssg/tasks/receiver.py43
-rw-r--r--webmentions_ssg/views.py8
12 files changed, 306 insertions, 5 deletions
diff --git a/contrib/webmentions-ssg.example.cfg b/contrib/webmentions-ssg.example.cfg
index 45ae297..577eeef 100644
--- a/contrib/webmentions-ssg.example.cfg
+++ b/contrib/webmentions-ssg.example.cfg
@@ -73,6 +73,9 @@ MAX_REDIRECTS = 20 # also used for outgoing requests
MAX_SOURCE_BYTES = 1_000_000
# Requeue stuck verifications and lost queued tasks after 15 minutes.
VERIFICATION_TIMEOUT_SECONDS = 900
+# Max stale-row requeues before marking the Webmention failed. Manual
+# reverification or a new submission after the cooldown resets this counter.
+VERIFICATION_MAX_RECOVERIES = 3
# Huey cron expression: every five minutes (in UTC).
VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *"
diff --git a/tests/cli/test_database.py b/tests/cli/test_database.py
index 7095d51..7d325aa 100644
--- a/tests/cli/test_database.py
+++ b/tests/cli/test_database.py
@@ -85,11 +85,15 @@ def test_create_db_builds_schema_and_records_latest_migration(
columns = {
column["name"] for column in inspector.get_columns("sent_webmentions")
}
+ received_columns = {
+ column["name"] for column in inspector.get_columns("received_webmentions")
+ }
received_indexes = {
index["name"] for index in inspector.get_indexes("received_webmentions")
}
assert {"alembic_version", "users", "received_webmentions"} <= table_names
assert {"attempt_count", "next_retry_at"} <= columns
+ assert "recovery_attempts" in received_columns
assert "ix_received_webmentions_status_updated_at" in received_indexes
@@ -247,6 +251,10 @@ def test_upgrade_backfills_portable_digest_for_existing_received_rows(
)
).one()
assert result.status == "verified"
+ recovery_attempts = connection.execute(
+ sa.text("SELECT recovery_attempts FROM received_webmentions")
+ ).scalar_one()
+ assert recovery_attempts == 0
assert result.url_digest == received_url_digest(
result.source, result.target
)
diff --git a/tests/forms/test_init.py b/tests/forms/test_init.py
index 2a68509..42e6a2c 100644
--- a/tests/forms/test_init.py
+++ b/tests/forms/test_init.py
@@ -144,7 +144,7 @@ def test_endpoint_handles_malformed_ipv6_without_crashing(app: Flask) -> None:
meta={"csrf": False},
)
assert not form.validate()
- assert "Invalid input." in form.errors["target"]
+ assert "Invalid URL." in form.errors["target"]
def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields(
@@ -168,3 +168,38 @@ def test_endpoint_form_applies_configurable_url_byte_limit_to_both_fields(
assert not form.validate()
assert "target" in form.errors
assert any("bytes" in message for message in form.errors["target"])
+
+
+@pytest.mark.parametrize("field_name", ["source", "target"])
+@pytest.mark.parametrize(
+ "invalid_url",
+ [
+ pytest.param("http://ex☃mple.test/", id="invalid-idn"),
+ pytest.param("https://example.com/a\nb", id="newline"),
+ pytest.param("https://example.com/a\rb", id="carriage-return"),
+ pytest.param("https://example.com/\x00b", id="nul"),
+ pytest.param("https://example.com/a b", id="ascii-space"),
+ pytest.param("https://example.com/a\u00a0b", id="unicode-whitespace"),
+ pytest.param("https://example.com/\x7fb", id="del"),
+ ],
+)
+def test_endpoint_rejects_invalid_httpx_urls_and_whitespace(
+ app: Flask, field_name: str, invalid_url: str
+) -> None:
+ data = {"source": VALID_SOURCE, "target": VALID_TARGET}
+ data[field_name] = invalid_url
+ with app.test_request_context("/endpoint", method="POST"):
+ form = EndpointForm(formdata=MultiDict(data), meta={"csrf": False})
+ assert not form.validate()
+ assert "Invalid URL." in form.errors[field_name]
+
+
+def test_endpoint_accepts_encoded_path_space(app: Flask) -> None:
+ with app.test_request_context("/endpoint", method="POST"):
+ form = EndpointForm(
+ formdata=MultiDict(
+ {"source": "https://source.example/a%20b", "target": VALID_TARGET}
+ ),
+ meta={"csrf": False},
+ )
+ assert form.validate()
diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py
index c795115..44d1cc2 100644
--- a/tests/tasks/test_receiver.py
+++ b/tests/tasks/test_receiver.py
@@ -35,6 +35,7 @@ def create_webmention(
target: str = TARGET_URL,
status: str = "received",
failure_reason: str | None = None,
+ recovery_attempts: int = 0,
) -> uuid.UUID:
identifier = uuid.uuid7()
@@ -45,6 +46,7 @@ def create_webmention(
target=target,
status=status,
failure_reason=failure_reason,
+ recovery_attempts=recovery_attempts,
)
db.session.add(webmention)
@@ -669,6 +671,106 @@ def test_reaper_requeues_and_verifies_integration(
check.assert_called_once_with(SOURCE_URL, TARGET_URL)
+@patch("webmentions_ssg.tasks.receiver.source_mentions_target")
+@pytest.mark.parametrize(
+ "unexpected_error",
+ [
+ pytest.param(httpx.InvalidURL("Invalid IDNA hostname"), id="invalid-url"),
+ pytest.param(
+ RuntimeError("Unexpected HTML parser error"), id="unexpected-error"
+ ),
+ ],
+)
+def test_verify_webmention_records_unexpected_error_as_terminal_failure(
+ source_mentions_target: Mock,
+ app: Flask,
+ receiver: ModuleType,
+ unexpected_error: Exception,
+) -> None:
+ identifier = create_webmention(app, source="http://ex☃mple.test/")
+ source_mentions_target.side_effect = unexpected_error
+
+ receiver.verify_webmention.call_local(identifier)
+
+ assert get_webmention_state(app, identifier) == ("failed", str(unexpected_error))
+ source_mentions_target.assert_called_once()
+ # Failed rows are not eligible for automatic reaper recovery.
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 0
+
+
+def test_reaper_caps_stale_verification_recoveries(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["VERIFICATION_MAX_RECOVERIES"] = 3
+ identifier = create_webmention(app, status="verifying")
+ enqueue = Mock()
+ monkeypatch.setattr(receiver, "verify_webmention", enqueue)
+
+ # Simulate workers that repeatedly crash before completing the task.
+ for attempt in range(3):
+ set_webmention_updated_at(
+ app, identifier, datetime.now(UTC) - timedelta(hours=1)
+ )
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 1
+ row = db.session.get(ReceivedWebmention, identifier)
+ assert row is not None
+ assert row.status == "received"
+ assert row.recovery_attempts == attempt + 1
+
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 0
+ row = db.session.get(ReceivedWebmention, identifier)
+ assert row is not None
+ assert row.status == "failed"
+ assert row.failure_reason == receiver.RECOVERY_EXHAUSTED_REASON
+ assert row.recovery_attempts == 3
+ assert receiver.recover_stale_webmentions() == 0
+ assert enqueue.call_count == 3
+
+
+def test_reaper_caps_received_rows_lost_before_first_task(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ app.config["VERIFICATION_MAX_RECOVERIES"] = 1
+ identifier = create_webmention(app, status="received", recovery_attempts=1)
+ set_webmention_updated_at(app, identifier, datetime.now(UTC) - timedelta(hours=1))
+ enqueue = Mock()
+ monkeypatch.setattr(receiver, "verify_webmention", enqueue)
+
+ with app.app_context():
+ assert receiver.recover_stale_webmentions() == 0
+ assert get_webmention_state(app, identifier) == (
+ "failed",
+ receiver.RECOVERY_EXHAUSTED_REASON,
+ )
+ enqueue.assert_not_called()
+
+
+def test_exhausted_row_cannot_be_revived_by_late_huey_retry(
+ app: Flask, receiver: ModuleType, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ identifier = create_webmention(
+ app,
+ status="failed",
+ failure_reason=receiver.RECOVERY_EXHAUSTED_REASON,
+ recovery_attempts=3,
+ )
+ source_mentions_target = Mock(return_value=True)
+ monkeypatch.setattr(receiver, "source_mentions_target", source_mentions_target)
+
+ receiver.verify_webmention.call_local(identifier)
+
+ assert get_webmention_state(app, identifier) == (
+ "failed",
+ receiver.RECOVERY_EXHAUSTED_REASON,
+ )
+ source_mentions_target.assert_not_called()
+
+
def test_verify_webmention_ignores_unknown_identifier(
receiver: ModuleType, caplog: pytest.LogCaptureFixture
) -> None:
diff --git a/tests/test_views.py b/tests/test_views.py
index f5e301f..2108b38 100644
--- a/tests/test_views.py
+++ b/tests/test_views.py
@@ -576,6 +576,11 @@ def test_reverify_received_webmention(
identifier = create_received_webmention(
app, status="failed", failure_reason="Previous failure"
)
+ with app.app_context():
+ row = db.session.get(ReceivedWebmention, identifier)
+ assert row is not None
+ row.recovery_attempts = 3
+ db.session.commit()
mock_admin_action_form(views_module, monkeypatch, valid=True)
verify_webmention = Mock()
monkeypatch.setattr(views_module, "verify_webmention", verify_webmention)
@@ -590,6 +595,7 @@ def test_reverify_received_webmention(
assert webmention is not None
assert webmention.status == "received"
assert webmention.failure_reason is None
+ assert webmention.recovery_attempts == 0
verify_webmention.assert_called_once_with(identifier)
@@ -911,6 +917,7 @@ def test_endpoint_reverifies_after_cooldown(
target=TARGET_URL,
status="failed",
failure_reason="Previous failure",
+ recovery_attempts=3,
updated_at=datetime.now(UTC) - timedelta(minutes=11),
)
db.session.add(webmention)
@@ -926,6 +933,7 @@ def test_endpoint_reverifies_after_cooldown(
assert stored is not None
assert stored.status == "received"
assert stored.failure_reason is None
+ assert stored.recovery_attempts == 0
verify.assert_called_once_with(identifier)
@@ -1194,3 +1202,27 @@ def test_endpoint_rejects_request_bodies_over_configured_maximum(
"/endpoint", data={"source": SOURCE_URL + "x" * 150, "target": TARGET_URL}
)
assert response.status_code == 413
+
+
+def test_endpoint_rejects_bad_url_before_inserting_row(
+ app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from webmentions_ssg import views
+
+ verify_webmention = Mock()
+ monkeypatch.setattr(views, "verify_webmention", verify_webmention)
+
+ response = client.post(
+ "/endpoint", data={"source": "http://ex☃mple.test/", "target": TARGET_URL}
+ )
+
+ assert response.status_code == 400
+ assert "source" in response.get_json()
+ verify_webmention.assert_not_called()
+ with app.app_context():
+ assert (
+ db.session.scalar(
+ sa.select(sa.func.count()).select_from(ReceivedWebmention)
+ )
+ == 0
+ )
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index 97dd682..e91c651 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -56,6 +56,8 @@ class DefaultConfig:
REQUEST_TOTAL_TIMEOUT = 30.0
# Requeue abandoned verification attempts after 15 minutes.
VERIFICATION_TIMEOUT_SECONDS = 900
+ # Maximum automatic recoveries of a stale received/verifying row.
+ VERIFICATION_MAX_RECOVERIES = 3
VERIFICATION_REAPER_SCHEDULE = "*/5 * * * *"
SOURCE_DIRECTORY = None
diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py
index 5006365..589cbd9 100644
--- a/webmentions_ssg/forms/__init__.py
+++ b/webmentions_ssg/forms/__init__.py
@@ -10,7 +10,13 @@ from flask_wtf import FlaskForm
from wtforms import PasswordField, StringField, SubmitField
from wtforms.validators import URL, InputRequired, Regexp
-from .validators import AllowedHostname, NotEqualTo, PublicIPLiteral, URLMaxBytes
+from .validators import (
+ AllowedHostname,
+ HTTPXURL,
+ NotEqualTo,
+ PublicIPLiteral,
+ URLMaxBytes,
+)
class LoginForm(FlaskForm):
@@ -36,6 +42,7 @@ class EndpointForm(FlaskForm):
validators=[
InputRequired(),
URLMaxBytes(),
+ HTTPXURL(),
URL(),
Regexp(
"^https?://.*",
@@ -51,6 +58,7 @@ class EndpointForm(FlaskForm):
validators=[
InputRequired(),
URLMaxBytes(),
+ HTTPXURL(),
URL(),
Regexp(
"^https?://.*",
diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py
index c1f1c97..384c7a1 100644
--- a/webmentions_ssg/forms/validators.py
+++ b/webmentions_ssg/forms/validators.py
@@ -5,10 +5,13 @@
"""Provide validators for URL safety and form field constraints."""
import ipaddress
+import unicodedata
from urllib.parse import urlsplit
+import httpx
from flask import current_app
from wtforms import Field, ValidationError
+from wtforms.validators import StopValidation
from wtforms.form import BaseForm
from ..url_security import AddressResolutionError, is_public_address, is_public_url
@@ -40,6 +43,33 @@ class URLMaxBytes:
)
+class HTTPXURL:
+ """Reject URLs that cannot be fetched by httpx or contain whitespace/controls.
+
+ httpx automatically percent-encodes some whitespace. Reject it instead of
+ silently changing the submitted URL (which may be used in exact URL matches).
+ """
+
+ def __call__(self, form: BaseForm, field: Field) -> None:
+ value = field.data
+ if not isinstance(value, str):
+ raise StopValidation(field.gettext("Invalid URL."))
+
+ if any(
+ character.isspace() or unicodedata.category(character) == "Cc"
+ for character in value
+ ):
+ raise StopValidation(field.gettext("Invalid URL."))
+
+ try:
+ parsed = httpx.URL(value)
+ except (httpx.InvalidURL, ValueError, UnicodeError) as exc:
+ raise StopValidation(field.gettext("Invalid URL.")) from exc
+
+ if not parsed.host:
+ raise StopValidation(field.gettext("Invalid URL."))
+
+
class NotEqualTo:
"""
Validate that a field does not equal another field.
diff --git a/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py
new file mode 100644
index 0000000..c516636
--- /dev/null
+++ b/webmentions_ssg/migrations/versions/e62b1a7c4d90_cap_received_verification_recoveries.py
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Cap stale received Webmention verification recovery per row.
+
+Revision ID: e62b1a7c4d90
+Revises: d845aa7fbc19
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "e62b1a7c4d90"
+down_revision = "d845aa7fbc19"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ with op.batch_alter_table("received_webmentions") as batch_op:
+ batch_op.add_column(
+ sa.Column(
+ "recovery_attempts", sa.Integer(), nullable=False, server_default="0"
+ )
+ )
+
+
+def downgrade():
+ with op.batch_alter_table("received_webmentions") as batch_op:
+ batch_op.drop_column("recovery_attempts")
diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py
index 0dfbb78..31516d5 100644
--- a/webmentions_ssg/models.py
+++ b/webmentions_ssg/models.py
@@ -121,6 +121,11 @@ class ReceivedWebmention(Base):
return value
status: Mapped[str] = mapped_column(Text(), nullable=False, default="received")
+ # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh
+ # explicit submission resets this counter.
+ recovery_attempts: Mapped[int] = mapped_column(
+ Integer, nullable=False, default=0, server_default="0"
+ )
failure_reason: Mapped[str | None] = mapped_column(
Text(), nullable=True, default=None
)
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index f999d1b..acd66a1 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -22,6 +22,9 @@ from ..public_transport import PublicHTTPTransport
from ..url_security import AddressResolutionError, UnsafeAddressError
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+
+
class VerificationError(Exception):
"""The source permanently failed ReceivedWebmention verification."""
@@ -261,6 +264,12 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
# should not start another concurrent verification.
if webmention.status not in {"received", "failed"}:
return
+ # A reaper-exhausted row must not be revived by a late Huey retry.
+ if (
+ webmention.status == "failed"
+ and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
+ ):
+ return
previous_status = webmention.status
previous_updated_at = webmention.updated_at
@@ -308,6 +317,13 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
except (TemporaryFetchError, httpx.RequestError) as exc:
finish("failed", str(exc) or "Source could not be fetched")
raise
+ except Exception as exc:
+ # InvalidURL is not a RequestError; other unexpected source errors must
+ # also leave the row in a terminal state, without an endless retry loop.
+ current_app.logger.exception(
+ "Unexpected error verifying ReceivedWebmention %s", webmention_uuid
+ )
+ finish("failed", str(exc) or type(exc).__name__)
else:
if mentions_target:
finish("verified", None)
@@ -324,6 +340,8 @@ def recover_stale_webmentions() -> int:
updates prevent overlapping reapers from enqueueing the same row. Each
reclaimed row is committed *before* enqueueing, so the worker can claim it.
A crash between commit and enqueue is recovered on a later sweep.
+ Persist a recovery count so repeatedly lost or crashing tasks eventually
+ terminate rather than being requeued forever.
:return: Number of webmentions requeued.
"""
@@ -334,6 +352,7 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.uuid,
ReceivedWebmention.status,
ReceivedWebmention.updated_at,
+ ReceivedWebmention.recovery_attempts,
).where(
ReceivedWebmention.status.in_(("verifying", "received")),
ReceivedWebmention.updated_at <= cutoff,
@@ -341,24 +360,44 @@ def recover_stale_webmentions() -> int:
).all()
recovered = 0
- for identifier, status, updated_at in stale:
+ exhausted = 0
+ max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+ for identifier, status, updated_at, recovery_attempts in stale:
+ at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
sa.update(ReceivedWebmention)
.where(
ReceivedWebmention.uuid == identifier,
ReceivedWebmention.status == status,
ReceivedWebmention.updated_at == updated_at,
+ ReceivedWebmention.recovery_attempts == recovery_attempts,
+ )
+ .values(
+ status="failed" if at_limit else "received",
+ failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
+ updated_at=now,
+ recovery_attempts=(
+ recovery_attempts if at_limit else recovery_attempts + 1
+ ),
)
- .values(status="received", failure_reason=None, updated_at=now)
)
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+ if at_limit:
+ exhausted += 1
+ continue
verify_webmention(identifier)
recovered += 1
+ if exhausted:
+ current_app.logger.warning(
+ "Failed %d stale webmentions after maximum verification recoveries",
+ exhausted,
+ )
+
if recovered:
current_app.logger.warning(
"Requeued %d stale received webmention verifications", recovered
diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py
index fcb5f2d..0836869 100644
--- a/webmentions_ssg/views.py
+++ b/webmentions_ssg/views.py
@@ -185,6 +185,7 @@ def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue:
webmention.status = "received"
webmention.failure_reason = None
+ webmention.recovery_attempts = 0
db.session.commit()
@@ -395,7 +396,12 @@ def endpoint() -> ResponseReturnValue:
ReceivedWebmention.uuid == webmention.uuid,
ReceivedWebmention.updated_at <= cutoff,
)
- .values(status="received", failure_reason=None, updated_at=now)
+ .values(
+ status="received",
+ failure_reason=None,
+ recovery_attempts=0,
+ updated_at=now,
+ )
.execution_options(synchronize_session=False)
),
)