aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tests/forms/test_sources.py79
-rw-r--r--tests/test_config.py2
-rw-r--r--tests/views/test_sources.py366
-rw-r--r--webmentions_ssg/config.py3
-rw-r--r--webmentions_ssg/forms/sources.py81
-rw-r--r--webmentions_ssg/templates/sent.html103
-rw-r--r--webmentions_ssg/templates/sent_source.html23
-rw-r--r--webmentions_ssg/views/sources.py166
8 files changed, 785 insertions, 38 deletions
diff --git a/tests/forms/test_sources.py b/tests/forms/test_sources.py
new file mode 100644
index 0000000..44b4096
--- /dev/null
+++ b/tests/forms/test_sources.py
@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Validation of source-list GET filters."""
+
+import pytest
+from flask import Flask
+from werkzeug.datastructures import MultiDict
+
+from webmentions_ssg.forms.sources import SourceFilterForm
+
+
+def test_source_filter_form_accepts_get_params(app: Flask) -> None:
+ with app.test_request_context("/sources"):
+ form = SourceFilterForm(
+ formdata=MultiDict(
+ {
+ "status": "retrying",
+ "q": "example.org",
+ "date_from": "2026-11-08",
+ "date_to": "2026-11-14",
+ "sort": "webmentions",
+ "order": "asc",
+ "per_page": "37",
+ }
+ ),
+ meta={"csrf": False},
+ )
+ assert form.validate()
+ assert form.status.data == "retrying"
+ assert form.date_from.data.isoformat() == "2026-11-08"
+ assert form.date_to.data.isoformat() == "2026-11-14"
+ assert (form.sort.data, form.order.data, form.per_page.data) == (
+ "webmentions",
+ "asc",
+ 37,
+ )
+ assert "csrf_token" not in form._fields
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ {"status": "unknown"},
+ {"sort": "unknown"},
+ {"order": "up"},
+ {"per_page": "0"},
+ {"per_page": "101"},
+ {"per_page": "nope"},
+ {"date_from": "2026-11-8"},
+ {"date_to": "not-a-date"},
+ {"date_from": "2026-11-15", "date_to": "2026-11-08"},
+ ],
+)
+def test_source_filter_form_rejects_bad_get_params(
+ app: Flask, data: dict[str, str]
+) -> None:
+ with app.test_request_context("/sources"):
+ form = SourceFilterForm(formdata=MultiDict(data), meta={"csrf": False})
+ assert not form.validate()
+ assert form.errors
+
+
+def test_source_filter_form_defaults_and_configured_maximum(app: Flask) -> None:
+ app.config["SOURCES_MAX_PAGE_SIZE"] = 10
+ with app.test_request_context("/sources"):
+ form = SourceFilterForm(formdata=MultiDict(), meta={"csrf": False})
+ assert form.validate()
+ assert (
+ form.status.data,
+ form.sort.data,
+ form.order.data,
+ form.per_page.data,
+ ) == ("", "last_seen", "desc", 10)
+ form = SourceFilterForm(
+ formdata=MultiDict({"per_page": "11"}), meta={"csrf": False}
+ )
+ assert not form.validate()
diff --git a/tests/test_config.py b/tests/test_config.py
index ca87403..75b8a4a 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -382,6 +382,7 @@ def test_resource_limits_have_safe_defaults() -> None:
assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192
assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30
assert DefaultConfig.RECEIVED_MAX_PAGE_SIZE == 100
+ assert DefaultConfig.SOURCES_MAX_PAGE_SIZE == 100
assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *"
@@ -392,6 +393,7 @@ def test_resource_limits_have_safe_defaults() -> None:
"MAX_WEBMENTION_URL_BYTES",
"RECEIVED_RETENTION_DAYS",
"RECEIVED_MAX_PAGE_SIZE",
+ "SOURCES_MAX_PAGE_SIZE",
],
)
@pytest.mark.parametrize("value", [None, "100", 0, -1, 1.5])
diff --git a/tests/views/test_sources.py b/tests/views/test_sources.py
index 673fc9f..c851833 100644
--- a/tests/views/test_sources.py
+++ b/tests/views/test_sources.py
@@ -2,11 +2,14 @@
#
# SPDX-License-Identifier: BSD-3-Clause
+import re
import uuid
from datetime import UTC, datetime, timedelta
+from html import unescape
from http import HTTPStatus
from types import ModuleType
from unittest.mock import Mock
+from urllib.parse import parse_qs, urlsplit
import pytest
from bs4 import BeautifulSoup
@@ -14,7 +17,11 @@ from flask import Flask
from flask.testing import FlaskClient
from webmentions_ssg import db
-from webmentions_ssg.models.sources import SentWebmention
+from webmentions_ssg.models.sources import (
+ SentWebmention,
+ SentWebmentionStatus,
+ SourceDocument,
+)
from webmentions_ssg.models.user import User
from .helpers import (
@@ -72,12 +79,12 @@ def test_sent_sources_table_hides_secondary_columns_on_mobile(
headers = table.select("thead th")
cells = table.select("tbody tr:first-child td")
assert len(headers) == len(cells) == 7
- for index in (1, 3, 4, 5): # URL, revision, created, last seen
+ for index in (2, 4, 5, 6): # URL, revision, created, last seen
for cell in (headers[index], cells[index]):
assert {"d-none", "d-md-table-cell"} <= set(cell.get("class", []))
- for index in (0, 2, 6): # Path, count, status stay visible
+ for index in (0, 1, 3): # Path, count, status stay visible
assert "d-none" not in cells[index].get("class", [])
- assert "text-break" in cells[0].get("class", [])
+ assert "text-break" in cells[1].get("class", [])
def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None:
@@ -298,3 +305,354 @@ def test_admin_only_links_safe_existing_status_urls(
html = response.get_data(as_text=True)
assert (f'href="{status_url}"' in html) is safe
assert status_url in html
+
+
+def make_source(
+ app: Flask,
+ name: str,
+ *,
+ statuses: tuple[SentWebmentionStatus | None, ...] = (SentWebmentionStatus.SENT,),
+ retrying: bool = False,
+ processed: bool = True,
+ deleted: bool = False,
+ last_seen_at: datetime | None = None,
+ revision: int = 1,
+) -> uuid.UUID:
+ """Create distinguishable sources with outgoing rows for list filtering."""
+ now = last_seen_at or datetime(2026, 11, 12, 12, tzinfo=UTC)
+ with app.app_context():
+ source = SourceDocument(
+ path=f"blog/{name}/index.html",
+ url=f"https://source.example/{name}/",
+ content_hash="a" * 32,
+ revision=revision,
+ last_seen_at=now,
+ revised_at=now,
+ deleted_at=now if deleted else None,
+ )
+ for number, status in enumerate(statuses):
+ source.sent_webmentions.append(
+ SentWebmention(
+ target=f"https://target.example/{name}/{number}",
+ linked=True,
+ desired_revision=revision,
+ processed_revision=revision if processed else None,
+ status=status,
+ attempt_count=2 if retrying else 0,
+ next_retry_at=now + timedelta(days=10) if retrying else None,
+ )
+ )
+ db.session.add(source)
+ db.session.commit()
+ return source.uuid
+
+
+@pytest.mark.parametrize(
+ ("status", "expected"),
+ [
+ ("failing", {"failed", "retrying", "stale", "multiple"}),
+ ("retrying", {"retrying"}),
+ ("deleted", {"deleted"}),
+ ],
+)
+def test_sources_filter_by_delivery_state_without_duplicate_rows(
+ app: Flask, client: FlaskClient, status: str, expected: set[str]
+) -> None:
+ log_in(app, client)
+ make_source(app, "failed", statuses=(SentWebmentionStatus.FAILED,))
+ make_source(
+ app,
+ "retrying",
+ statuses=(SentWebmentionStatus.FAILED,),
+ retrying=True,
+ processed=False,
+ )
+ make_source(app, "stale", statuses=(SentWebmentionStatus.FAILED,), retrying=True)
+ make_source(
+ app,
+ "multiple",
+ statuses=(SentWebmentionStatus.FAILED, SentWebmentionStatus.FAILED),
+ )
+ make_source(app, "deleted", deleted=True)
+ make_source(app, "sent")
+ make_source(app, "pending", statuses=(None,), processed=False)
+
+ response = client.get("/sources", query_string={"status": status})
+ assert response.status_code == HTTPStatus.OK
+ soup = BeautifulSoup(response.get_data(as_text=True), "html.parser")
+ paths = [
+ row.select_one("td a").get_text(strip=True) for row in soup.select("tbody tr")
+ ]
+ assert set(paths) == {f"blog/{name}/index.html" for name in expected}
+ assert len(paths) == len(expected)
+ assert (
+ soup.select_one(f'select[name="status"] option[value="{status}"][selected]')
+ is not None
+ )
+
+
+@pytest.mark.parametrize(
+ ("q", "expected"),
+ [
+ ("blog/alpha", "alpha"),
+ ("source.example/alpha", "alpha"),
+ ("target.example/alpha", "alpha"),
+ ("alpha/0", "alpha"),
+ ("alpha%", None),
+ ],
+)
+def test_sources_search_matches_path_source_url_and_sent_target(
+ app: Flask, client: FlaskClient, q: str, expected: str | None
+) -> None:
+ log_in(app, client)
+ make_source(app, "alpha")
+ make_source(app, "beta")
+
+ response = client.get("/sources", query_string={"q": f" {q} "})
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ assert ("blog/alpha/index.html" in html) == (expected == "alpha")
+ assert "blog/beta/index.html" not in html
+ if expected is None:
+ assert "No sources match these filters." in html
+
+
+def test_sources_filter_by_inclusive_last_seen_dates(
+ app: Flask, client: FlaskClient
+) -> None:
+ log_in(app, client)
+ make_source(app, "old", last_seen_at=datetime(2026, 11, 7, tzinfo=UTC))
+ make_source(app, "included", last_seen_at=datetime(2026, 11, 10, 22, tzinfo=UTC))
+ make_source(app, "new", last_seen_at=datetime(2026, 11, 15, tzinfo=UTC))
+ response = client.get(
+ "/sources", query_string={"date_from": "2026-11-08", "date_to": "2026-11-10"}
+ )
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ assert "blog/included/index.html" in html
+ assert "blog/old/index.html" not in html
+ assert "blog/new/index.html" not in html
+
+
+@pytest.mark.parametrize(
+ "sort", ["path", "url", "webmentions", "revision", "created", "last_seen", "status"]
+)
+def test_sources_sort_links_preserve_filters(
+ app: Flask, client: FlaskClient, sort: str
+) -> None:
+ log_in(app, client)
+ make_source(app, "alpha", statuses=(SentWebmentionStatus.FAILED,))
+ response = client.get(
+ "/sources",
+ query_string={
+ "status": "failing",
+ "q": "alpha",
+ "date_from": "2026-11-01",
+ "sort": sort,
+ "order": "asc",
+ "per_page": 37,
+ },
+ )
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ soup = BeautifulSoup(html, "html.parser")
+ assert soup.select_one('th[aria-sort="ascending"]') is not None
+ links = [
+ parse_qs(urlsplit(unescape(a["href"])).query)
+ for a in soup.select("th a[href]")
+ if urlsplit(unescape(a["href"])).path == "/sources"
+ ]
+ assert len(links) == 7
+ for link in links:
+ assert link["status"] == ["failing"]
+ assert link["q"] == ["alpha"]
+ assert link["date_from"] == ["2026-11-01"]
+ assert link["per_page"] == ["37"]
+ assert link["sort"][0] in (
+ "path",
+ "url",
+ "webmentions",
+ "revision",
+ "created",
+ "last_seen",
+ "status",
+ )
+ assert any(link["sort"] == [sort] and link["order"] == ["desc"] for link in links)
+
+
+def test_sources_sort_orders_by_count_and_path(app: Flask, client: FlaskClient) -> None:
+ log_in(app, client)
+ make_source(app, "z", statuses=(SentWebmentionStatus.SENT,))
+ make_source(
+ app, "a", statuses=(SentWebmentionStatus.SENT, SentWebmentionStatus.FAILED)
+ )
+ for sort, order, expected_first in (
+ ("webmentions", "desc", "a"),
+ ("webmentions", "asc", "z"),
+ ("path", "asc", "a"),
+ ("path", "desc", "z"),
+ ):
+ response = client.get("/sources", query_string={"sort": sort, "order": order})
+ assert response.status_code == HTTPStatus.OK
+ soup = BeautifulSoup(response.get_data(as_text=True), "html.parser")
+ assert (
+ soup.select_one("tbody tr td a").get_text(strip=True)
+ == f"blog/{expected_first}/index.html"
+ )
+
+
+def test_sources_filter_pagination_and_navigation_preserve_parameters(
+ app: Flask, client: FlaskClient
+) -> None:
+ log_in(app, client)
+ for number in range(28):
+ make_source(app, f"failed-{number:02}", statuses=(SentWebmentionStatus.FAILED,))
+ filters = {
+ "status": "failing",
+ "q": "failed",
+ "sort": "path",
+ "order": "asc",
+ "per_page": "10",
+ }
+ response = client.get("/sources", query_string={**filters, "page": "2"})
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ assert "28 sources" in html
+ soup = BeautifulSoup(html, "html.parser")
+ assert len(soup.select("tbody tr")) == 10
+ urls = [unescape(a["href"]) for a in soup.select("a[href]")]
+ assert any(
+ urlsplit(url).path == "/sources"
+ and parse_qs(urlsplit(url).query)
+ == {**{k: [v] for k, v in filters.items()}, "page": ["3"]}
+ for url in urls
+ )
+ detail_urls = [
+ url for url in urls if re.search(r"/sources/[0-9a-f-]{36}", urlsplit(url).path)
+ ]
+ assert detail_urls
+ assert parse_qs(urlsplit(detail_urls[0]).query) == {
+ **{k: [v] for k, v in filters.items()},
+ "source_page": ["2"],
+ }
+ detail = client.get(detail_urls[0])
+ assert detail.status_code == HTTPStatus.OK
+ back = BeautifulSoup(detail.get_data(as_text=True), "html.parser").select_one(
+ 'a[href*="/sources?"]'
+ )
+ assert back is not None
+ assert parse_qs(urlsplit(unescape(back["href"])).query) == {
+ **{k: [v] for k, v in filters.items()},
+ "page": ["2"],
+ }
+
+
+@pytest.mark.parametrize(
+ "invalid",
+ [
+ "status=unknown",
+ "sort=unsupported",
+ "order=reverse",
+ "per_page=0",
+ "per_page=101",
+ "per_page=abc",
+ "date_from=2026-11-8",
+ "date_to=bad",
+ "date_from=2026-11-10&date_to=2026-11-01",
+ ],
+)
+def test_sources_bad_filters_return_400(
+ app: Flask, client: FlaskClient, invalid: str
+) -> None:
+ log_in(app, client)
+ assert client.get(f"/sources?{invalid}").status_code == HTTPStatus.BAD_REQUEST
+
+
+def test_rescan_sources_preserves_filter_query(
+ app: Flask,
+ client: FlaskClient,
+ sources_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ mock_admin_action_form(sources_module, monkeypatch, valid=True)
+ scan = Mock()
+ monkeypatch.setattr(sources_module, "manual_scan_sources", scan)
+ response = client.post(
+ "/sources/rescan",
+ query_string={"status": "retrying", "per_page": 37, "page": 2},
+ )
+ assert response.status_code == HTTPStatus.FOUND
+ assert parse_qs(urlsplit(response.headers["Location"]).query) == {
+ "status": ["retrying"],
+ "per_page": ["37"],
+ "page": ["2"],
+ }
+ scan.assert_called_once_with()
+
+
+def test_sources_show_failed_and_retrying_counts_and_retry_deadline(
+ app: Flask, client: FlaskClient
+) -> None:
+ log_in(app, client)
+ identifier = make_source(
+ app,
+ "retry-badges",
+ statuses=(SentWebmentionStatus.FAILED,),
+ retrying=True,
+ processed=False,
+ )
+ response = client.get("/sources", query_string={"status": "retrying"})
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ assert "1 failed" in html
+ assert "1 awaiting retry" in html
+
+ response = client.get(f"/sources/{identifier}")
+ assert response.status_code == HTTPStatus.OK
+ assert "Retry scheduled after attempt 2:" in response.get_data(as_text=True)
+
+
+def test_resend_preserves_source_filter_context(
+ app: Flask,
+ client: FlaskClient,
+ sources_module: ModuleType,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ log_in(app, client)
+ source_id, mention_id = create_resendable_sent_webmention(app)
+ mock_admin_action_form(sources_module, monkeypatch, valid=True)
+ send = Mock()
+ monkeypatch.setattr(sources_module, "send_webmention", send)
+
+ response = client.post(
+ f"/sources/{source_id}/{mention_id}/resend",
+ query_string={
+ "status": "failing",
+ "q": "resend",
+ "sort": "path",
+ "source_page": 2,
+ "page": 3,
+ },
+ )
+ assert response.status_code == HTTPStatus.FOUND
+ assert urlsplit(response.headers["Location"]).path == f"/sources/{source_id}"
+ assert parse_qs(urlsplit(response.headers["Location"]).query) == {
+ "status": ["failing"],
+ "q": ["resend"],
+ "sort": ["path"],
+ "source_page": ["2"],
+ "page": ["3"],
+ }
+ send.assert_called_once_with(mention_id)
+
+
+def test_sources_empty_filter_result_keeps_rescan_modal(
+ app: Flask, client: FlaskClient
+) -> None:
+ log_in(app, client)
+ response = client.get("/sources", query_string={"status": "failing"})
+ assert response.status_code == HTTPStatus.OK
+ html = response.get_data(as_text=True)
+ assert "No sources match these filters." in html
+ assert 'id="confirmation-modal"' in html
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index be54989..3d05e5d 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -91,6 +91,8 @@ class DefaultConfig:
RECEIVED_RETENTION_DAYS = 30
# Maximum rows that an admin can request per received Webmentions page.
RECEIVED_MAX_PAGE_SIZE = 100
+ # Maximum rows that an admin can request per source document page.
+ SOURCES_MAX_PAGE_SIZE = 100
RECEIVED_RETENTION_SCHEDULE = "0 3 * * *"
# Scanner
@@ -246,6 +248,7 @@ def validated_config_overrides(
"MAX_WEBMENTION_URL_BYTES",
"RECEIVED_RETENTION_DAYS",
"RECEIVED_MAX_PAGE_SIZE",
+ "SOURCES_MAX_PAGE_SIZE",
):
value = config[key]
if type(value) is not int or value <= 0:
diff --git a/webmentions_ssg/forms/sources.py b/webmentions_ssg/forms/sources.py
new file mode 100644
index 0000000..04afe77
--- /dev/null
+++ b/webmentions_ssg/forms/sources.py
@@ -0,0 +1,81 @@
+# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Source document filtering form."""
+
+from flask import current_app
+from flask_wtf import FlaskForm
+from wtforms import (
+ DateField,
+ HiddenField,
+ IntegerField,
+ SearchField,
+ SelectField,
+ ValidationError,
+)
+from wtforms.validators import AnyOf, NumberRange, Optional
+
+from .validators import CanonicalDate
+
+SOURCE_STATUS_CHOICES = {
+ "": "All sources",
+ "failing": "Failing sends",
+ "retrying": "Awaiting retry",
+ "deleted": "Deleted sources",
+}
+
+
+def default_source_page_size() -> int:
+ """Use 25 rows unless the configured maximum is lower."""
+ return min(25, current_app.config["SOURCES_MAX_PAGE_SIZE"])
+
+
+class SourceFilterForm(FlaskForm):
+ """GET filters for the source document list."""
+
+ status = SelectField("Status", choices=SOURCE_STATUS_CHOICES, default="")
+ q = SearchField("Path or URL")
+ date_from = DateField(
+ "Last seen from (UTC)", validators=[Optional(), CanonicalDate()]
+ )
+ date_to = DateField("Last seen to (UTC)", validators=[Optional(), CanonicalDate()])
+ sort = HiddenField(
+ default="last_seen",
+ validators=[
+ AnyOf(
+ (
+ "path",
+ "url",
+ "webmentions",
+ "revision",
+ "created",
+ "last_seen",
+ "status",
+ )
+ )
+ ],
+ )
+ order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))])
+ per_page = IntegerField(
+ "Rows per page",
+ default=default_source_page_size,
+ validators=[NumberRange(min=1)],
+ )
+
+ def validate_per_page(self, field: IntegerField) -> None:
+ """Enforce the deployment's maximum source-list page size."""
+ if (
+ field.data is not None
+ and field.data > current_app.config["SOURCES_MAX_PAGE_SIZE"]
+ ):
+ raise ValidationError("Rows per page exceeds the configured maximum.")
+
+ def validate_date_to(self, field: DateField) -> None:
+ """Require an end date on or after the start date."""
+ if (
+ self.date_from.data is not None
+ and field.data is not None
+ and field.data < self.date_from.data
+ ):
+ raise ValidationError("End date must be on or after start date.")
diff --git a/webmentions_ssg/templates/sent.html b/webmentions_ssg/templates/sent.html
index d2a319d..6febdb9 100644
--- a/webmentions_ssg/templates/sent.html
+++ b/webmentions_ssg/templates/sent.html
@@ -7,6 +7,19 @@ SPDX-License-Identifier: BSD-3-Clause
{% from "bootstrap5/pagination.html" import render_pagination %}
+{% macro sortable_header(label, column, responsive_class="") %}
+ {% set active = sort_key == column %}
+ <th scope="col"
+ class="{{ responsive_class }}"
+ aria-sort="{{ ('ascending' if sort_order == 'asc' else 'descending') if active else 'none' }}">
+ <a href="{{ sort_urls[column] }}"
+ class="link-body-emphasis text-decoration-none">
+ {{ label }}
+ {% if active %}<span aria-hidden="true">{{ "↑" if sort_order == "asc" else "↓" }}</span>{% endif %}
+ </a>
+ </th>
+{% endmacro %}
+
{% block content %}
<div class="d-flex justify-content-between align-items-center mb-3">
<h1 class="mb-0">Sources</h1>
@@ -16,29 +29,72 @@ SPDX-License-Identifier: BSD-3-Clause
class="btn btn-sm btn-primary"
data-bs-toggle="modal"
data-bs-target="#confirmation-modal"
- data-action="{{ url_for('webmentions.rescan_sources') }}"
+ data-action="{{ url_for('webmentions.rescan_sources', page=sources.page, **filters) }}"
data-title="Rescan sources?"
data-message="All sources will be scanned again for changes and webmentions."
data-confirm-label="Rescan"
data-confirm-class="btn-primary">Rescan</button>
</div>
</div>
+ <form action="{{ url_for('webmentions.source_list') }}"
+ method="get"
+ class="row g-2 align-items-end mb-4"
+ role="search">
+ {{ form.sort }}
+ {{ form.order }}
+ <div class="col-sm-6 col-lg">
+ {{ form.status.label(class_="form-label") }}
+ {{ form.status(class_="form-select") }}
+ </div>
+ <div class="col-sm-6 col-lg">
+ {{ form.q.label(class_="form-label") }}
+ {{ form.q(class_="form-control", placeholder="Search paths and URLs") }}
+ </div>
+ <div class="col-sm-6 col-lg">
+ {{ form.date_from.label(class_="form-label") }}
+ {{ form.date_from(class_="form-control") }}
+ </div>
+ <div class="col-sm-6 col-lg">
+ {{ form.date_to.label(class_="form-label") }}
+ {{ form.date_to(class_="form-control") }}
+ </div>
+ <div class="col-sm-6 col-lg-auto">
+ {{ form.per_page.label(class_="form-label") }}
+ {{ form.per_page(class_="form-control", min=1, max=max_source_page_size, step=1) }}
+ </div>
+ <div class="col-auto d-flex gap-2 ms-auto">
+ <button type="submit" class="btn btn-primary">Filter</button>
+ <a href="{{ url_for('webmentions.source_list') }}"
+ class="btn btn-outline-secondary">Clear</a>
+ </div>
+ </form>
{% if sources.items %}
<div class="table-responsive">
<table class="table table-bordered table-striped table-hover align-middle">
<thead>
<tr>
- <th scope="col">Path</th>
- <th scope="col" class="d-none d-md-table-cell">URL</th>
- <th scope="col">Webmentions</th>
- <th scope="col" class="d-none d-md-table-cell">Revision</th>
- <th scope="col" class="d-none d-md-table-cell">Created</th>
- <th scope="col" class="d-none d-md-table-cell">Last seen</th>
- <th scope="col">Status</th>
+ {{ sortable_header('Status', 'status') }}
+ {{ sortable_header('Path', 'path') }}
+ {{ sortable_header('URL', 'url', 'd-none d-md-table-cell') }}
+ {{ sortable_header('Webmentions', 'webmentions') }}
+ {{ sortable_header('Revision', 'revision', 'd-none d-md-table-cell') }}
+ {{ sortable_header('Created', 'created', 'd-none d-md-table-cell') }}
+ {{ sortable_header('Last seen', 'last_seen', 'd-none d-md-table-cell') }}
</tr>
</thead>
<tbody>
{% for source in sources %}
+ {% set statistics = namespace(sent=0, failed=0, retrying=0) %}
+ {% for webmention in source.sent_webmentions %}
+ {% if webmention.status == "sent" %}
+ {% set statistics.sent = statistics.sent + 1 %}
+ {% elif webmention.status == "failed" %}
+ {% set statistics.failed = statistics.failed + 1 %}
+ {% endif %}
+ {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %}
+ {% set statistics.retrying = statistics.retrying + 1 %}
+ {% endif %}
+ {% endfor %}
{% if source.deleted_at is none %}
{% set row_class = "table-success" %}
{% set badge_class = "text-bg-success" %}
@@ -49,13 +105,27 @@ SPDX-License-Identifier: BSD-3-Clause
{% set status_text = "Deleted" %}
{% endif %}
<tr class="{{ row_class }}">
+ <td>
+ <span class="badge {{ badge_class }}">{{ status_text }}</span>
+ </td>
<td class="text-break">
- <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid) }}">{{ source.path }}</a>
+ <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid, source_page=sources.page, **filters) }}">{{ source.path }}</a>
</td>
<td class="text-break d-none d-md-table-cell">
<a href="{{ source.url }}">{{ source.url }}</a>
</td>
- <td>{{ source.sent_webmentions | length }}</td>
+ <td>
+ <span class="badge text-bg-info">{{ source.sent_webmentions | length }} Total</span>
+ {% if statistics.sent %}
+ <span class="badge text-bg-success">{{ statistics.sent }} sent</span>
+ {% endif %}
+ {% if statistics.failed %}
+ <span class="badge text-bg-danger">{{ statistics.failed }} failed</span>
+ {% endif %}
+ {% if statistics.retrying %}
+ <span class="badge text-bg-warning">{{ statistics.retrying }} awaiting retry</span>
+ {% endif %}
+ </td>
<td class="d-none d-md-table-cell">{{ source.revision }}</td>
<td class="text-nowrap d-none d-md-table-cell">
<time datetime="{{ source.created_at | utc_isoformat }}"
@@ -69,21 +139,22 @@ SPDX-License-Identifier: BSD-3-Clause
{{ source.last_seen_at | relative_time }}
</time>
</td>
- <td>
- <span class="badge {{ badge_class }}">{{ status_text }}</span>
- </td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% if sources.pages > 1 %}
- <div class="mt-3">{{ render_pagination(sources) }}</div>
+ <div class="mt-3">{{ render_pagination(sources, args=filters) }}</div>
{% endif %}
- {% include "includes/confirmation_modal.html" %}
{% else %}
- <p class="text-body-secondary">No Webmention sources have been discovered yet.</p>
+ {% if filters %}
+ <p class="text-body-secondary">No sources match these filters.</p>
+ {% else %}
+ <p class="text-body-secondary">No Webmention sources have been discovered yet.</p>
+ {% endif %}
{% endif %}
+ {% include "includes/confirmation_modal.html" %}
{% endblock %}
{% block scripts %}
diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html
index 33e26a2..04f9d07 100644
--- a/webmentions_ssg/templates/sent_source.html
+++ b/webmentions_ssg/templates/sent_source.html
@@ -26,7 +26,7 @@ SPDX-License-Identifier: BSD-3-Clause
{% block content %}
<div class="mb-3">
- <a href="{{ url_for('webmentions.source_list') }}">← Back to sources</a>
+ <a href="{{ url_for('webmentions.source_list', page=source_page, **filters) }}">← Back to sources</a>
</div>
<div class="d-flex justify-content-between align-items-start mb-3">
<h1 class="mb-0">Source Details</h1>
@@ -96,8 +96,8 @@ SPDX-License-Identifier: BSD-3-Clause
<table class="table table-bordered table-striped table-hover align-middle">
<thead>
<tr>
- <th scope="col">Target</th>
<th scope="col">Status</th>
+ <th scope="col">Target</th>
<th scope="col" class="d-none d-md-table-cell">Active</th>
<th scope="col" class="d-none d-md-table-cell">Revisions</th>
<th scope="col" class="d-none d-md-table-cell">Delivery</th>
@@ -110,13 +110,22 @@ SPDX-License-Identifier: BSD-3-Clause
{% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %}
<tr class="{{ style.row_class }}">
<td class="text-break">
- <a href="{{ webmention.target }}">{{ webmention.target }}</a>
- </td>
- <td class="text-break">
<span class="badge {{ style.badge_class }}">{{ style.label }}</span>
{% if webmention.failure_reason is not none %}
<div class="small mt-2">{{ webmention.failure_reason }}</div>
{% endif %}
+ {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %}
+ <div class="small mt-2">
+ Retry scheduled after attempt {{ webmention.attempt_count }}:
+ <time datetime="{{ webmention.next_retry_at | utc_isoformat }}"
+ title="{{ webmention.next_retry_at | utc_timestamp }}">
+ {{ webmention.next_retry_at | relative_time }}
+ </time>
+ </div>
+ {% endif %}
+ </td>
+ <td class="text-break">
+ <a href="{{ webmention.target }}">{{ webmention.target }}</a>
</td>
<td class="d-none d-md-table-cell">
{% if webmention.linked %}
@@ -199,7 +208,7 @@ SPDX-License-Identifier: BSD-3-Clause
class="btn btn-sm btn-primary"
data-bs-toggle="modal"
data-bs-target="#confirmation-modal"
- data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid) }}"
+ data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid, source_page=source_page, page=webmentions.page, **filters) }}"
data-title="Resend Webmention?"
data-message="This Webmention will be queued again for delivery to {{ webmention.target }}."
data-confirm-label="Resend"
@@ -211,7 +220,7 @@ SPDX-License-Identifier: BSD-3-Clause
</table>
</div>
{% if webmentions.pages > 1 %}
- <div class="mt-3">{{ render_pagination(webmentions) }}</div>
+ <div class="mt-3">{{ render_pagination(webmentions, args=dict(filters, source_page=source_page)) }}</div>
{% endif %}
{% include "includes/confirmation_modal.html" %}
{% else %}
diff --git a/webmentions_ssg/views/sources.py b/webmentions_ssg/views/sources.py
index b68d2fd..1c9cc58 100644
--- a/webmentions_ssg/views/sources.py
+++ b/webmentions_ssg/views/sources.py
@@ -5,39 +5,166 @@
"""Authenticated source document and sent Webmention views."""
import uuid
+from datetime import UTC, date, datetime, time
from http import HTTPStatus
+from typing import TypedDict, cast
import sqlalchemy as sa
-from flask import abort, flash, redirect, render_template, url_for
+from flask import abort, current_app, flash, redirect, render_template, request, url_for
from flask.typing import ResponseReturnValue
from flask_login import login_required
from sqlalchemy.orm import selectinload
from ..extensions import db
from ..forms.admin import ConfirmActionForm
-from ..models.sources import SentWebmention, SourceDocument
+from ..forms.sources import (
+ SOURCE_STATUS_CHOICES,
+ SourceFilterForm,
+ default_source_page_size,
+)
+from ..models.sources import SentWebmention, SentWebmentionStatus, SourceDocument
from ..tasks.scanner import manual_scan_sources
from ..tasks.sender import send_webmention
from .blueprint import webmentions_bp
+class SourceFilterArgs(TypedDict, total=False):
+ """Validated source-list query parameters; keys match GET form fields."""
+
+ status: str
+ q: str
+ date_from: str
+ date_to: str
+ sort: str
+ order: str
+ per_page: str
+
+
+def source_filter_form() -> SourceFilterForm:
+ """Validate GET filters without requiring CSRF on the search form."""
+ form = SourceFilterForm(formdata=request.args, meta={"csrf": False})
+ if not form.validate():
+ abort(HTTPStatus.BAD_REQUEST)
+ form.q.data = (form.q.data or "").strip()
+ return form
+
+
+def source_filter_args(form: SourceFilterForm) -> SourceFilterArgs:
+ """Serialize non-default values for pagination, sorting, and navigation."""
+ values: dict[str, str] = {}
+ for name in SourceFilterArgs.__annotations__:
+ field = form[name]
+ value = field.data
+ default = field.default() if callable(field.default) else field.default
+ if value is None or value == "" or value == default:
+ continue
+ values[name] = value.isoformat() if isinstance(value, date) else str(value)
+ return cast(SourceFilterArgs, values)
+
+
@webmentions_bp.route("/sources")
@login_required
def source_list() -> ResponseReturnValue:
- """Display sources with sent Webmentions.
+ """Display source documents with filters for outgoing delivery problems."""
+ form = source_filter_form()
+ filters = source_filter_args(form)
+ query = sa.select(SourceDocument).options(
+ selectinload(SourceDocument.sent_webmentions)
+ )
- :returns: Rendered page containing the paginated source list.
- """
+ # EXISTS avoids duplicate source rows when several Webmentions match.
+ sent = sa.select(SentWebmention.uuid).where(
+ SentWebmention.source_uuid == SourceDocument.uuid
+ )
+ if status := filters.get("status"):
+ if status == "failing":
+ query = query.where(
+ sent.where(
+ SentWebmention.status == SentWebmentionStatus.FAILED
+ ).exists()
+ )
+ elif status == "retrying":
+ query = query.where(
+ sent.where(
+ SentWebmention.awaiting_processing,
+ SentWebmention.attempt_count > 0,
+ SentWebmention.next_retry_at.is_not(None),
+ ).exists()
+ )
+ else: # deleted
+ query = query.where(SourceDocument.deleted_at.is_not(None))
+
+ if text := filters.get("q"):
+ query = query.where(
+ sa.or_(
+ SourceDocument.path.icontains(text, autoescape=True),
+ SourceDocument.url.icontains(text, autoescape=True),
+ sent.where(
+ SentWebmention.target.icontains(text, autoescape=True)
+ ).exists(),
+ )
+ )
+
+ if start := filters.get("date_from"):
+ query = query.where(
+ SourceDocument.last_seen_at
+ >= datetime.combine(date.fromisoformat(start), time.min, UTC)
+ )
+ if end := filters.get("date_to"):
+ query = query.where(
+ SourceDocument.last_seen_at
+ <= datetime.combine(date.fromisoformat(end), time.max, UTC)
+ )
+
+ count = (
+ sa.select(sa.func.count(SentWebmention.uuid))
+ .where(SentWebmention.source_uuid == SourceDocument.uuid)
+ .scalar_subquery()
+ )
+ sort_columns = {
+ "path": SourceDocument.path,
+ "url": SourceDocument.url,
+ "webmentions": count,
+ "revision": SourceDocument.revision,
+ "created": SourceDocument.uuid,
+ "last_seen": SourceDocument.last_seen_at,
+ "status": sa.case((SourceDocument.deleted_at.is_(None), 0), else_=1),
+ }
+ sort_key = filters.get("sort", "last_seen")
+ sort_order = filters.get("order", "desc")
+ column = sort_columns[sort_key]
+ query = query.order_by(column.asc() if sort_order == "asc" else column.desc())
+ if sort_key != "created":
+ query = query.order_by(SourceDocument.uuid.desc())
+
+ page_size = int(filters.get("per_page", default_source_page_size()))
sources = db.paginate(
- sa.select(SourceDocument)
- .options(selectinload(SourceDocument.sent_webmentions))
- .order_by(SourceDocument.last_seen_at.desc()),
- per_page=25,
+ query,
+ per_page=page_size,
+ max_per_page=current_app.config["SOURCES_MAX_PAGE_SIZE"],
)
+
+ sort_urls = {}
+ for key in sort_columns:
+ if key == sort_key:
+ next_order = "asc" if sort_order == "desc" else "desc"
+ else:
+ next_order = "asc" if key in ("path", "url", "status") else "desc"
+ sort_filters: SourceFilterArgs = {**filters, "sort": key, "order": next_order}
+ sort_urls[key] = url_for("webmentions.source_list", **sort_filters)
+
return render_template(
"sent.html",
title="Sent Webmentions",
sources=sources,
+ form=form,
+ filters=filters,
+ sort_key=sort_key,
+ sort_order=sort_order,
+ sort_urls=sort_urls,
+ status_choices=SOURCE_STATUS_CHOICES,
+ page_size=page_size,
+ max_source_page_size=current_app.config["SOURCES_MAX_PAGE_SIZE"],
action_form=ConfirmActionForm(),
)
@@ -52,10 +179,15 @@ def rescan_sources() -> ResponseReturnValue:
if not ConfirmActionForm().validate_on_submit():
abort(HTTPStatus.BAD_REQUEST)
+ filters = source_filter_args(source_filter_form())
manual_scan_sources()
flash("Source rescan queued.", "success")
- return redirect(url_for("webmentions.source_list"))
+ query_args: dict[str, str | int] = dict(filters)
+ page = request.args.get("page", 1, type=int)
+ if page > 1:
+ query_args["page"] = page
+ return redirect(url_for("webmentions.source_list", **query_args))
@webmentions_bp.route("/sources/<uuid:source_uuid>")
@@ -66,6 +198,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue:
:param source_uuid: UUID of the source to display.
:returns: Rendered source details page.
"""
+ filters = source_filter_args(source_filter_form())
+ source_page = request.args.get("source_page", 1, type=int)
source = db.get_or_404(SourceDocument, source_uuid)
webmentions = db.paginate(
@@ -80,6 +214,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue:
title="Sent Webmentions",
source=source,
webmentions=webmentions,
+ filters=filters,
+ source_page=source_page,
action_form=ConfirmActionForm(),
)
@@ -103,6 +239,7 @@ def resend_webmention(
if not ConfirmActionForm().validate_on_submit():
abort(HTTPStatus.BAD_REQUEST)
+ filters = source_filter_args(source_filter_form())
webmention = db.get_or_404(SentWebmention, webmention_uuid)
if webmention.source_uuid != source_uuid:
@@ -119,4 +256,11 @@ def resend_webmention(
flash(f"Webmention {webmention.uuid} queued for resend.", "success")
- return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid))
+ detail_args: dict[str, str | int] = dict(filters)
+ for name in ("page", "source_page"):
+ page = request.args.get(name, 1, type=int)
+ if page > 1:
+ detail_args[name] = page
+ return redirect(
+ url_for("webmentions.source_detail", source_uuid=source_uuid, **detail_args)
+ )