diff options
| author | Dennis Fink | 2026-11-18 21:13:34 +0100 |
|---|---|---|
| committer | Dennis Fink | 2026-11-18 21:13:34 +0100 |
| commit | b2b5429bcf008db357d9d65db77ba37f7af908a8 (patch) | |
| tree | ed913ee050817609b56d2f5515b2c3fbc9945a75 | |
| parent | beb0131b089293512c2dab2243b9b2ff314e2387 (diff) | |
| download | webmentions-ssg-b2b5429bcf008db357d9d65db77ba37f7af908a8.tar.gz webmentions-ssg-b2b5429bcf008db357d9d65db77ba37f7af908a8.zip | |
feat(sender): add source list filters and sorting
Add status, search, date, sorting, and page-size controls to the source
list with validated query parameters and stable pagination.
Preserve list context across source details, rescans, resends, and
pagination, and surface failed and retrying delivery counts with retry
deadlines.
| -rw-r--r-- | tests/forms/test_sources.py | 79 | ||||
| -rw-r--r-- | tests/test_config.py | 2 | ||||
| -rw-r--r-- | tests/views/test_sources.py | 366 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 3 | ||||
| -rw-r--r-- | webmentions_ssg/forms/sources.py | 81 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent.html | 103 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent_source.html | 23 | ||||
| -rw-r--r-- | webmentions_ssg/views/sources.py | 166 |
8 files changed, 785 insertions, 38 deletions
diff --git a/tests/forms/test_sources.py b/tests/forms/test_sources.py new file mode 100644 index 0000000..44b4096 --- /dev/null +++ b/tests/forms/test_sources.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Validation of source-list GET filters.""" + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.sources import SourceFilterForm + + +def test_source_filter_form_accepts_get_params(app: Flask) -> None: + with app.test_request_context("/sources"): + form = SourceFilterForm( + formdata=MultiDict( + { + "status": "retrying", + "q": "example.org", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + "sort": "webmentions", + "order": "asc", + "per_page": "37", + } + ), + meta={"csrf": False}, + ) + assert form.validate() + assert form.status.data == "retrying" + assert form.date_from.data.isoformat() == "2026-11-08" + assert form.date_to.data.isoformat() == "2026-11-14" + assert (form.sort.data, form.order.data, form.per_page.data) == ( + "webmentions", + "asc", + 37, + ) + assert "csrf_token" not in form._fields + + +@pytest.mark.parametrize( + "data", + [ + {"status": "unknown"}, + {"sort": "unknown"}, + {"order": "up"}, + {"per_page": "0"}, + {"per_page": "101"}, + {"per_page": "nope"}, + {"date_from": "2026-11-8"}, + {"date_to": "not-a-date"}, + {"date_from": "2026-11-15", "date_to": "2026-11-08"}, + ], +) +def test_source_filter_form_rejects_bad_get_params( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/sources"): + form = SourceFilterForm(formdata=MultiDict(data), meta={"csrf": False}) + assert not form.validate() + assert form.errors + + +def test_source_filter_form_defaults_and_configured_maximum(app: Flask) -> None: + app.config["SOURCES_MAX_PAGE_SIZE"] = 10 + with app.test_request_context("/sources"): + form = SourceFilterForm(formdata=MultiDict(), meta={"csrf": False}) + assert form.validate() + assert ( + form.status.data, + form.sort.data, + form.order.data, + form.per_page.data, + ) == ("", "last_seen", "desc", 10) + form = SourceFilterForm( + formdata=MultiDict({"per_page": "11"}), meta={"csrf": False} + ) + assert not form.validate() diff --git a/tests/test_config.py b/tests/test_config.py index ca87403..75b8a4a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -382,6 +382,7 @@ def test_resource_limits_have_safe_defaults() -> None: assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192 assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30 assert DefaultConfig.RECEIVED_MAX_PAGE_SIZE == 100 + assert DefaultConfig.SOURCES_MAX_PAGE_SIZE == 100 assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *" @@ -392,6 +393,7 @@ def test_resource_limits_have_safe_defaults() -> None: "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS", "RECEIVED_MAX_PAGE_SIZE", + "SOURCES_MAX_PAGE_SIZE", ], ) @pytest.mark.parametrize("value", [None, "100", 0, -1, 1.5]) diff --git a/tests/views/test_sources.py b/tests/views/test_sources.py index 673fc9f..c851833 100644 --- a/tests/views/test_sources.py +++ b/tests/views/test_sources.py @@ -2,11 +2,14 @@ # # SPDX-License-Identifier: BSD-3-Clause +import re import uuid from datetime import UTC, datetime, timedelta +from html import unescape from http import HTTPStatus from types import ModuleType from unittest.mock import Mock +from urllib.parse import parse_qs, urlsplit import pytest from bs4 import BeautifulSoup @@ -14,7 +17,11 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models.sources import SentWebmention +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) from webmentions_ssg.models.user import User from .helpers import ( @@ -72,12 +79,12 @@ def test_sent_sources_table_hides_secondary_columns_on_mobile( headers = table.select("thead th") cells = table.select("tbody tr:first-child td") assert len(headers) == len(cells) == 7 - for index in (1, 3, 4, 5): # URL, revision, created, last seen + for index in (2, 4, 5, 6): # URL, revision, created, last seen for cell in (headers[index], cells[index]): assert {"d-none", "d-md-table-cell"} <= set(cell.get("class", [])) - for index in (0, 2, 6): # Path, count, status stay visible + for index in (0, 1, 3): # Path, count, status stay visible assert "d-none" not in cells[index].get("class", []) - assert "text-break" in cells[0].get("class", []) + assert "text-break" in cells[1].get("class", []) def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: @@ -298,3 +305,354 @@ def test_admin_only_links_safe_existing_status_urls( html = response.get_data(as_text=True) assert (f'href="{status_url}"' in html) is safe assert status_url in html + + +def make_source( + app: Flask, + name: str, + *, + statuses: tuple[SentWebmentionStatus | None, ...] = (SentWebmentionStatus.SENT,), + retrying: bool = False, + processed: bool = True, + deleted: bool = False, + last_seen_at: datetime | None = None, + revision: int = 1, +) -> uuid.UUID: + """Create distinguishable sources with outgoing rows for list filtering.""" + now = last_seen_at or datetime(2026, 11, 12, 12, tzinfo=UTC) + with app.app_context(): + source = SourceDocument( + path=f"blog/{name}/index.html", + url=f"https://source.example/{name}/", + content_hash="a" * 32, + revision=revision, + last_seen_at=now, + revised_at=now, + deleted_at=now if deleted else None, + ) + for number, status in enumerate(statuses): + source.sent_webmentions.append( + SentWebmention( + target=f"https://target.example/{name}/{number}", + linked=True, + desired_revision=revision, + processed_revision=revision if processed else None, + status=status, + attempt_count=2 if retrying else 0, + next_retry_at=now + timedelta(days=10) if retrying else None, + ) + ) + db.session.add(source) + db.session.commit() + return source.uuid + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("failing", {"failed", "retrying", "stale", "multiple"}), + ("retrying", {"retrying"}), + ("deleted", {"deleted"}), + ], +) +def test_sources_filter_by_delivery_state_without_duplicate_rows( + app: Flask, client: FlaskClient, status: str, expected: set[str] +) -> None: + log_in(app, client) + make_source(app, "failed", statuses=(SentWebmentionStatus.FAILED,)) + make_source( + app, + "retrying", + statuses=(SentWebmentionStatus.FAILED,), + retrying=True, + processed=False, + ) + make_source(app, "stale", statuses=(SentWebmentionStatus.FAILED,), retrying=True) + make_source( + app, + "multiple", + statuses=(SentWebmentionStatus.FAILED, SentWebmentionStatus.FAILED), + ) + make_source(app, "deleted", deleted=True) + make_source(app, "sent") + make_source(app, "pending", statuses=(None,), processed=False) + + response = client.get("/sources", query_string={"status": status}) + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + paths = [ + row.select_one("td a").get_text(strip=True) for row in soup.select("tbody tr") + ] + assert set(paths) == {f"blog/{name}/index.html" for name in expected} + assert len(paths) == len(expected) + assert ( + soup.select_one(f'select[name="status"] option[value="{status}"][selected]') + is not None + ) + + +@pytest.mark.parametrize( + ("q", "expected"), + [ + ("blog/alpha", "alpha"), + ("source.example/alpha", "alpha"), + ("target.example/alpha", "alpha"), + ("alpha/0", "alpha"), + ("alpha%", None), + ], +) +def test_sources_search_matches_path_source_url_and_sent_target( + app: Flask, client: FlaskClient, q: str, expected: str | None +) -> None: + log_in(app, client) + make_source(app, "alpha") + make_source(app, "beta") + + response = client.get("/sources", query_string={"q": f" {q} "}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert ("blog/alpha/index.html" in html) == (expected == "alpha") + assert "blog/beta/index.html" not in html + if expected is None: + assert "No sources match these filters." in html + + +def test_sources_filter_by_inclusive_last_seen_dates( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + make_source(app, "old", last_seen_at=datetime(2026, 11, 7, tzinfo=UTC)) + make_source(app, "included", last_seen_at=datetime(2026, 11, 10, 22, tzinfo=UTC)) + make_source(app, "new", last_seen_at=datetime(2026, 11, 15, tzinfo=UTC)) + response = client.get( + "/sources", query_string={"date_from": "2026-11-08", "date_to": "2026-11-10"} + ) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "blog/included/index.html" in html + assert "blog/old/index.html" not in html + assert "blog/new/index.html" not in html + + +@pytest.mark.parametrize( + "sort", ["path", "url", "webmentions", "revision", "created", "last_seen", "status"] +) +def test_sources_sort_links_preserve_filters( + app: Flask, client: FlaskClient, sort: str +) -> None: + log_in(app, client) + make_source(app, "alpha", statuses=(SentWebmentionStatus.FAILED,)) + response = client.get( + "/sources", + query_string={ + "status": "failing", + "q": "alpha", + "date_from": "2026-11-01", + "sort": sort, + "order": "asc", + "per_page": 37, + }, + ) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + soup = BeautifulSoup(html, "html.parser") + assert soup.select_one('th[aria-sort="ascending"]') is not None + links = [ + parse_qs(urlsplit(unescape(a["href"])).query) + for a in soup.select("th a[href]") + if urlsplit(unescape(a["href"])).path == "/sources" + ] + assert len(links) == 7 + for link in links: + assert link["status"] == ["failing"] + assert link["q"] == ["alpha"] + assert link["date_from"] == ["2026-11-01"] + assert link["per_page"] == ["37"] + assert link["sort"][0] in ( + "path", + "url", + "webmentions", + "revision", + "created", + "last_seen", + "status", + ) + assert any(link["sort"] == [sort] and link["order"] == ["desc"] for link in links) + + +def test_sources_sort_orders_by_count_and_path(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + make_source(app, "z", statuses=(SentWebmentionStatus.SENT,)) + make_source( + app, "a", statuses=(SentWebmentionStatus.SENT, SentWebmentionStatus.FAILED) + ) + for sort, order, expected_first in ( + ("webmentions", "desc", "a"), + ("webmentions", "asc", "z"), + ("path", "asc", "a"), + ("path", "desc", "z"), + ): + response = client.get("/sources", query_string={"sort": sort, "order": order}) + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + assert ( + soup.select_one("tbody tr td a").get_text(strip=True) + == f"blog/{expected_first}/index.html" + ) + + +def test_sources_filter_pagination_and_navigation_preserve_parameters( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + for number in range(28): + make_source(app, f"failed-{number:02}", statuses=(SentWebmentionStatus.FAILED,)) + filters = { + "status": "failing", + "q": "failed", + "sort": "path", + "order": "asc", + "per_page": "10", + } + response = client.get("/sources", query_string={**filters, "page": "2"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "28 sources" in html + soup = BeautifulSoup(html, "html.parser") + assert len(soup.select("tbody tr")) == 10 + urls = [unescape(a["href"]) for a in soup.select("a[href]")] + assert any( + urlsplit(url).path == "/sources" + and parse_qs(urlsplit(url).query) + == {**{k: [v] for k, v in filters.items()}, "page": ["3"]} + for url in urls + ) + detail_urls = [ + url for url in urls if re.search(r"/sources/[0-9a-f-]{36}", urlsplit(url).path) + ] + assert detail_urls + assert parse_qs(urlsplit(detail_urls[0]).query) == { + **{k: [v] for k, v in filters.items()}, + "source_page": ["2"], + } + detail = client.get(detail_urls[0]) + assert detail.status_code == HTTPStatus.OK + back = BeautifulSoup(detail.get_data(as_text=True), "html.parser").select_one( + 'a[href*="/sources?"]' + ) + assert back is not None + assert parse_qs(urlsplit(unescape(back["href"])).query) == { + **{k: [v] for k, v in filters.items()}, + "page": ["2"], + } + + +@pytest.mark.parametrize( + "invalid", + [ + "status=unknown", + "sort=unsupported", + "order=reverse", + "per_page=0", + "per_page=101", + "per_page=abc", + "date_from=2026-11-8", + "date_to=bad", + "date_from=2026-11-10&date_to=2026-11-01", + ], +) +def test_sources_bad_filters_return_400( + app: Flask, client: FlaskClient, invalid: str +) -> None: + log_in(app, client) + assert client.get(f"/sources?{invalid}").status_code == HTTPStatus.BAD_REQUEST + + +def test_rescan_sources_preserves_filter_query( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + scan = Mock() + monkeypatch.setattr(sources_module, "manual_scan_sources", scan) + response = client.post( + "/sources/rescan", + query_string={"status": "retrying", "per_page": 37, "page": 2}, + ) + assert response.status_code == HTTPStatus.FOUND + assert parse_qs(urlsplit(response.headers["Location"]).query) == { + "status": ["retrying"], + "per_page": ["37"], + "page": ["2"], + } + scan.assert_called_once_with() + + +def test_sources_show_failed_and_retrying_counts_and_retry_deadline( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + identifier = make_source( + app, + "retry-badges", + statuses=(SentWebmentionStatus.FAILED,), + retrying=True, + processed=False, + ) + response = client.get("/sources", query_string={"status": "retrying"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "1 failed" in html + assert "1 awaiting retry" in html + + response = client.get(f"/sources/{identifier}") + assert response.status_code == HTTPStatus.OK + assert "Retry scheduled after attempt 2:" in response.get_data(as_text=True) + + +def test_resend_preserves_source_filter_context( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_id, mention_id = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + send = Mock() + monkeypatch.setattr(sources_module, "send_webmention", send) + + response = client.post( + f"/sources/{source_id}/{mention_id}/resend", + query_string={ + "status": "failing", + "q": "resend", + "sort": "path", + "source_page": 2, + "page": 3, + }, + ) + assert response.status_code == HTTPStatus.FOUND + assert urlsplit(response.headers["Location"]).path == f"/sources/{source_id}" + assert parse_qs(urlsplit(response.headers["Location"]).query) == { + "status": ["failing"], + "q": ["resend"], + "sort": ["path"], + "source_page": ["2"], + "page": ["3"], + } + send.assert_called_once_with(mention_id) + + +def test_sources_empty_filter_result_keeps_rescan_modal( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.get("/sources", query_string={"status": "failing"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "No sources match these filters." in html + assert 'id="confirmation-modal"' in html diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index be54989..3d05e5d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -91,6 +91,8 @@ class DefaultConfig: RECEIVED_RETENTION_DAYS = 30 # Maximum rows that an admin can request per received Webmentions page. RECEIVED_MAX_PAGE_SIZE = 100 + # Maximum rows that an admin can request per source document page. + SOURCES_MAX_PAGE_SIZE = 100 RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" # Scanner @@ -246,6 +248,7 @@ def validated_config_overrides( "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS", "RECEIVED_MAX_PAGE_SIZE", + "SOURCES_MAX_PAGE_SIZE", ): value = config[key] if type(value) is not int or value <= 0: diff --git a/webmentions_ssg/forms/sources.py b/webmentions_ssg/forms/sources.py new file mode 100644 index 0000000..04afe77 --- /dev/null +++ b/webmentions_ssg/forms/sources.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source document filtering form.""" + +from flask import current_app +from flask_wtf import FlaskForm +from wtforms import ( + DateField, + HiddenField, + IntegerField, + SearchField, + SelectField, + ValidationError, +) +from wtforms.validators import AnyOf, NumberRange, Optional + +from .validators import CanonicalDate + +SOURCE_STATUS_CHOICES = { + "": "All sources", + "failing": "Failing sends", + "retrying": "Awaiting retry", + "deleted": "Deleted sources", +} + + +def default_source_page_size() -> int: + """Use 25 rows unless the configured maximum is lower.""" + return min(25, current_app.config["SOURCES_MAX_PAGE_SIZE"]) + + +class SourceFilterForm(FlaskForm): + """GET filters for the source document list.""" + + status = SelectField("Status", choices=SOURCE_STATUS_CHOICES, default="") + q = SearchField("Path or URL") + date_from = DateField( + "Last seen from (UTC)", validators=[Optional(), CanonicalDate()] + ) + date_to = DateField("Last seen to (UTC)", validators=[Optional(), CanonicalDate()]) + sort = HiddenField( + default="last_seen", + validators=[ + AnyOf( + ( + "path", + "url", + "webmentions", + "revision", + "created", + "last_seen", + "status", + ) + ) + ], + ) + order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) + per_page = IntegerField( + "Rows per page", + default=default_source_page_size, + validators=[NumberRange(min=1)], + ) + + def validate_per_page(self, field: IntegerField) -> None: + """Enforce the deployment's maximum source-list page size.""" + if ( + field.data is not None + and field.data > current_app.config["SOURCES_MAX_PAGE_SIZE"] + ): + raise ValidationError("Rows per page exceeds the configured maximum.") + + def validate_date_to(self, field: DateField) -> None: + """Require an end date on or after the start date.""" + if ( + self.date_from.data is not None + and field.data is not None + and field.data < self.date_from.data + ): + raise ValidationError("End date must be on or after start date.") diff --git a/webmentions_ssg/templates/sent.html b/webmentions_ssg/templates/sent.html index d2a319d..6febdb9 100644 --- a/webmentions_ssg/templates/sent.html +++ b/webmentions_ssg/templates/sent.html @@ -7,6 +7,19 @@ SPDX-License-Identifier: BSD-3-Clause {% from "bootstrap5/pagination.html" import render_pagination %} +{% macro sortable_header(label, column, responsive_class="") %} + {% set active = sort_key == column %} + <th scope="col" + class="{{ responsive_class }}" + aria-sort="{{ ('ascending' if sort_order == 'asc' else 'descending') if active else 'none' }}"> + <a href="{{ sort_urls[column] }}" + class="link-body-emphasis text-decoration-none"> + {{ label }} + {% if active %}<span aria-hidden="true">{{ "↑" if sort_order == "asc" else "↓" }}</span>{% endif %} + </a> + </th> +{% endmacro %} + {% block content %} <div class="d-flex justify-content-between align-items-center mb-3"> <h1 class="mb-0">Sources</h1> @@ -16,29 +29,72 @@ SPDX-License-Identifier: BSD-3-Clause class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#confirmation-modal" - data-action="{{ url_for('webmentions.rescan_sources') }}" + data-action="{{ url_for('webmentions.rescan_sources', page=sources.page, **filters) }}" data-title="Rescan sources?" data-message="All sources will be scanned again for changes and webmentions." data-confirm-label="Rescan" data-confirm-class="btn-primary">Rescan</button> </div> </div> + <form action="{{ url_for('webmentions.source_list') }}" + method="get" + class="row g-2 align-items-end mb-4" + role="search"> + {{ form.sort }} + {{ form.order }} + <div class="col-sm-6 col-lg"> + {{ form.status.label(class_="form-label") }} + {{ form.status(class_="form-select") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.q.label(class_="form-label") }} + {{ form.q(class_="form-control", placeholder="Search paths and URLs") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.date_from.label(class_="form-label") }} + {{ form.date_from(class_="form-control") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.date_to.label(class_="form-label") }} + {{ form.date_to(class_="form-control") }} + </div> + <div class="col-sm-6 col-lg-auto"> + {{ form.per_page.label(class_="form-label") }} + {{ form.per_page(class_="form-control", min=1, max=max_source_page_size, step=1) }} + </div> + <div class="col-auto d-flex gap-2 ms-auto"> + <button type="submit" class="btn btn-primary">Filter</button> + <a href="{{ url_for('webmentions.source_list') }}" + class="btn btn-outline-secondary">Clear</a> + </div> + </form> {% if sources.items %} <div class="table-responsive"> <table class="table table-bordered table-striped table-hover align-middle"> <thead> <tr> - <th scope="col">Path</th> - <th scope="col" class="d-none d-md-table-cell">URL</th> - <th scope="col">Webmentions</th> - <th scope="col" class="d-none d-md-table-cell">Revision</th> - <th scope="col" class="d-none d-md-table-cell">Created</th> - <th scope="col" class="d-none d-md-table-cell">Last seen</th> - <th scope="col">Status</th> + {{ sortable_header('Status', 'status') }} + {{ sortable_header('Path', 'path') }} + {{ sortable_header('URL', 'url', 'd-none d-md-table-cell') }} + {{ sortable_header('Webmentions', 'webmentions') }} + {{ sortable_header('Revision', 'revision', 'd-none d-md-table-cell') }} + {{ sortable_header('Created', 'created', 'd-none d-md-table-cell') }} + {{ sortable_header('Last seen', 'last_seen', 'd-none d-md-table-cell') }} </tr> </thead> <tbody> {% for source in sources %} + {% set statistics = namespace(sent=0, failed=0, retrying=0) %} + {% for webmention in source.sent_webmentions %} + {% if webmention.status == "sent" %} + {% set statistics.sent = statistics.sent + 1 %} + {% elif webmention.status == "failed" %} + {% set statistics.failed = statistics.failed + 1 %} + {% endif %} + {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %} + {% set statistics.retrying = statistics.retrying + 1 %} + {% endif %} + {% endfor %} {% if source.deleted_at is none %} {% set row_class = "table-success" %} {% set badge_class = "text-bg-success" %} @@ -49,13 +105,27 @@ SPDX-License-Identifier: BSD-3-Clause {% set status_text = "Deleted" %} {% endif %} <tr class="{{ row_class }}"> + <td> + <span class="badge {{ badge_class }}">{{ status_text }}</span> + </td> <td class="text-break"> - <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid) }}">{{ source.path }}</a> + <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid, source_page=sources.page, **filters) }}">{{ source.path }}</a> </td> <td class="text-break d-none d-md-table-cell"> <a href="{{ source.url }}">{{ source.url }}</a> </td> - <td>{{ source.sent_webmentions | length }}</td> + <td> + <span class="badge text-bg-info">{{ source.sent_webmentions | length }} Total</span> + {% if statistics.sent %} + <span class="badge text-bg-success">{{ statistics.sent }} sent</span> + {% endif %} + {% if statistics.failed %} + <span class="badge text-bg-danger">{{ statistics.failed }} failed</span> + {% endif %} + {% if statistics.retrying %} + <span class="badge text-bg-warning">{{ statistics.retrying }} awaiting retry</span> + {% endif %} + </td> <td class="d-none d-md-table-cell">{{ source.revision }}</td> <td class="text-nowrap d-none d-md-table-cell"> <time datetime="{{ source.created_at | utc_isoformat }}" @@ -69,21 +139,22 @@ SPDX-License-Identifier: BSD-3-Clause {{ source.last_seen_at | relative_time }} </time> </td> - <td> - <span class="badge {{ badge_class }}">{{ status_text }}</span> - </td> </tr> {% endfor %} </tbody> </table> </div> {% if sources.pages > 1 %} - <div class="mt-3">{{ render_pagination(sources) }}</div> + <div class="mt-3">{{ render_pagination(sources, args=filters) }}</div> {% endif %} - {% include "includes/confirmation_modal.html" %} {% else %} - <p class="text-body-secondary">No Webmention sources have been discovered yet.</p> + {% if filters %} + <p class="text-body-secondary">No sources match these filters.</p> + {% else %} + <p class="text-body-secondary">No Webmention sources have been discovered yet.</p> + {% endif %} {% endif %} + {% include "includes/confirmation_modal.html" %} {% endblock %} {% block scripts %} diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 33e26a2..04f9d07 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -26,7 +26,7 @@ SPDX-License-Identifier: BSD-3-Clause {% block content %} <div class="mb-3"> - <a href="{{ url_for('webmentions.source_list') }}">← Back to sources</a> + <a href="{{ url_for('webmentions.source_list', page=source_page, **filters) }}">← Back to sources</a> </div> <div class="d-flex justify-content-between align-items-start mb-3"> <h1 class="mb-0">Source Details</h1> @@ -96,8 +96,8 @@ SPDX-License-Identifier: BSD-3-Clause <table class="table table-bordered table-striped table-hover align-middle"> <thead> <tr> - <th scope="col">Target</th> <th scope="col">Status</th> + <th scope="col">Target</th> <th scope="col" class="d-none d-md-table-cell">Active</th> <th scope="col" class="d-none d-md-table-cell">Revisions</th> <th scope="col" class="d-none d-md-table-cell">Delivery</th> @@ -110,13 +110,22 @@ SPDX-License-Identifier: BSD-3-Clause {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %} <tr class="{{ style.row_class }}"> <td class="text-break"> - <a href="{{ webmention.target }}">{{ webmention.target }}</a> - </td> - <td class="text-break"> <span class="badge {{ style.badge_class }}">{{ style.label }}</span> {% if webmention.failure_reason is not none %} <div class="small mt-2">{{ webmention.failure_reason }}</div> {% endif %} + {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %} + <div class="small mt-2"> + Retry scheduled after attempt {{ webmention.attempt_count }}: + <time datetime="{{ webmention.next_retry_at | utc_isoformat }}" + title="{{ webmention.next_retry_at | utc_timestamp }}"> + {{ webmention.next_retry_at | relative_time }} + </time> + </div> + {% endif %} + </td> + <td class="text-break"> + <a href="{{ webmention.target }}">{{ webmention.target }}</a> </td> <td class="d-none d-md-table-cell"> {% if webmention.linked %} @@ -199,7 +208,7 @@ SPDX-License-Identifier: BSD-3-Clause class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#confirmation-modal" - data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid) }}" + data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid, source_page=source_page, page=webmentions.page, **filters) }}" data-title="Resend Webmention?" data-message="This Webmention will be queued again for delivery to {{ webmention.target }}." data-confirm-label="Resend" @@ -211,7 +220,7 @@ SPDX-License-Identifier: BSD-3-Clause </table> </div> {% if webmentions.pages > 1 %} - <div class="mt-3">{{ render_pagination(webmentions) }}</div> + <div class="mt-3">{{ render_pagination(webmentions, args=dict(filters, source_page=source_page)) }}</div> {% endif %} {% include "includes/confirmation_modal.html" %} {% else %} diff --git a/webmentions_ssg/views/sources.py b/webmentions_ssg/views/sources.py index b68d2fd..1c9cc58 100644 --- a/webmentions_ssg/views/sources.py +++ b/webmentions_ssg/views/sources.py @@ -5,39 +5,166 @@ """Authenticated source document and sent Webmention views.""" import uuid +from datetime import UTC, date, datetime, time from http import HTTPStatus +from typing import TypedDict, cast import sqlalchemy as sa -from flask import abort, flash, redirect, render_template, url_for +from flask import abort, current_app, flash, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from flask_login import login_required from sqlalchemy.orm import selectinload from ..extensions import db from ..forms.admin import ConfirmActionForm -from ..models.sources import SentWebmention, SourceDocument +from ..forms.sources import ( + SOURCE_STATUS_CHOICES, + SourceFilterForm, + default_source_page_size, +) +from ..models.sources import SentWebmention, SentWebmentionStatus, SourceDocument from ..tasks.scanner import manual_scan_sources from ..tasks.sender import send_webmention from .blueprint import webmentions_bp +class SourceFilterArgs(TypedDict, total=False): + """Validated source-list query parameters; keys match GET form fields.""" + + status: str + q: str + date_from: str + date_to: str + sort: str + order: str + per_page: str + + +def source_filter_form() -> SourceFilterForm: + """Validate GET filters without requiring CSRF on the search form.""" + form = SourceFilterForm(formdata=request.args, meta={"csrf": False}) + if not form.validate(): + abort(HTTPStatus.BAD_REQUEST) + form.q.data = (form.q.data or "").strip() + return form + + +def source_filter_args(form: SourceFilterForm) -> SourceFilterArgs: + """Serialize non-default values for pagination, sorting, and navigation.""" + values: dict[str, str] = {} + for name in SourceFilterArgs.__annotations__: + field = form[name] + value = field.data + default = field.default() if callable(field.default) else field.default + if value is None or value == "" or value == default: + continue + values[name] = value.isoformat() if isinstance(value, date) else str(value) + return cast(SourceFilterArgs, values) + + @webmentions_bp.route("/sources") @login_required def source_list() -> ResponseReturnValue: - """Display sources with sent Webmentions. + """Display source documents with filters for outgoing delivery problems.""" + form = source_filter_form() + filters = source_filter_args(form) + query = sa.select(SourceDocument).options( + selectinload(SourceDocument.sent_webmentions) + ) - :returns: Rendered page containing the paginated source list. - """ + # EXISTS avoids duplicate source rows when several Webmentions match. + sent = sa.select(SentWebmention.uuid).where( + SentWebmention.source_uuid == SourceDocument.uuid + ) + if status := filters.get("status"): + if status == "failing": + query = query.where( + sent.where( + SentWebmention.status == SentWebmentionStatus.FAILED + ).exists() + ) + elif status == "retrying": + query = query.where( + sent.where( + SentWebmention.awaiting_processing, + SentWebmention.attempt_count > 0, + SentWebmention.next_retry_at.is_not(None), + ).exists() + ) + else: # deleted + query = query.where(SourceDocument.deleted_at.is_not(None)) + + if text := filters.get("q"): + query = query.where( + sa.or_( + SourceDocument.path.icontains(text, autoescape=True), + SourceDocument.url.icontains(text, autoescape=True), + sent.where( + SentWebmention.target.icontains(text, autoescape=True) + ).exists(), + ) + ) + + if start := filters.get("date_from"): + query = query.where( + SourceDocument.last_seen_at + >= datetime.combine(date.fromisoformat(start), time.min, UTC) + ) + if end := filters.get("date_to"): + query = query.where( + SourceDocument.last_seen_at + <= datetime.combine(date.fromisoformat(end), time.max, UTC) + ) + + count = ( + sa.select(sa.func.count(SentWebmention.uuid)) + .where(SentWebmention.source_uuid == SourceDocument.uuid) + .scalar_subquery() + ) + sort_columns = { + "path": SourceDocument.path, + "url": SourceDocument.url, + "webmentions": count, + "revision": SourceDocument.revision, + "created": SourceDocument.uuid, + "last_seen": SourceDocument.last_seen_at, + "status": sa.case((SourceDocument.deleted_at.is_(None), 0), else_=1), + } + sort_key = filters.get("sort", "last_seen") + sort_order = filters.get("order", "desc") + column = sort_columns[sort_key] + query = query.order_by(column.asc() if sort_order == "asc" else column.desc()) + if sort_key != "created": + query = query.order_by(SourceDocument.uuid.desc()) + + page_size = int(filters.get("per_page", default_source_page_size())) sources = db.paginate( - sa.select(SourceDocument) - .options(selectinload(SourceDocument.sent_webmentions)) - .order_by(SourceDocument.last_seen_at.desc()), - per_page=25, + query, + per_page=page_size, + max_per_page=current_app.config["SOURCES_MAX_PAGE_SIZE"], ) + + sort_urls = {} + for key in sort_columns: + if key == sort_key: + next_order = "asc" if sort_order == "desc" else "desc" + else: + next_order = "asc" if key in ("path", "url", "status") else "desc" + sort_filters: SourceFilterArgs = {**filters, "sort": key, "order": next_order} + sort_urls[key] = url_for("webmentions.source_list", **sort_filters) + return render_template( "sent.html", title="Sent Webmentions", sources=sources, + form=form, + filters=filters, + sort_key=sort_key, + sort_order=sort_order, + sort_urls=sort_urls, + status_choices=SOURCE_STATUS_CHOICES, + page_size=page_size, + max_source_page_size=current_app.config["SOURCES_MAX_PAGE_SIZE"], action_form=ConfirmActionForm(), ) @@ -52,10 +179,15 @@ def rescan_sources() -> ResponseReturnValue: if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) + filters = source_filter_args(source_filter_form()) manual_scan_sources() flash("Source rescan queued.", "success") - return redirect(url_for("webmentions.source_list")) + query_args: dict[str, str | int] = dict(filters) + page = request.args.get("page", 1, type=int) + if page > 1: + query_args["page"] = page + return redirect(url_for("webmentions.source_list", **query_args)) @webmentions_bp.route("/sources/<uuid:source_uuid>") @@ -66,6 +198,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: :param source_uuid: UUID of the source to display. :returns: Rendered source details page. """ + filters = source_filter_args(source_filter_form()) + source_page = request.args.get("source_page", 1, type=int) source = db.get_or_404(SourceDocument, source_uuid) webmentions = db.paginate( @@ -80,6 +214,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: title="Sent Webmentions", source=source, webmentions=webmentions, + filters=filters, + source_page=source_page, action_form=ConfirmActionForm(), ) @@ -103,6 +239,7 @@ def resend_webmention( if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) + filters = source_filter_args(source_filter_form()) webmention = db.get_or_404(SentWebmention, webmention_uuid) if webmention.source_uuid != source_uuid: @@ -119,4 +256,11 @@ def resend_webmention( flash(f"Webmention {webmention.uuid} queued for resend.", "success") - return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) + detail_args: dict[str, str | int] = dict(filters) + for name in ("page", "source_page"): + page = request.args.get(name, 1, type=int) + if page > 1: + detail_args[name] = page + return redirect( + url_for("webmentions.source_detail", source_uuid=source_uuid, **detail_args) + ) |
