diff options
| -rw-r--r-- | tests/forms/test_sources.py | 79 | ||||
| -rw-r--r-- | tests/test_config.py | 2 | ||||
| -rw-r--r-- | tests/views/test_sources.py | 366 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 3 | ||||
| -rw-r--r-- | webmentions_ssg/forms/sources.py | 81 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent.html | 103 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent_source.html | 23 | ||||
| -rw-r--r-- | webmentions_ssg/views/sources.py | 166 |
8 files changed, 785 insertions, 38 deletions
diff --git a/tests/forms/test_sources.py b/tests/forms/test_sources.py new file mode 100644 index 0000000..44b4096 --- /dev/null +++ b/tests/forms/test_sources.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Validation of source-list GET filters.""" + +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms.sources import SourceFilterForm + + +def test_source_filter_form_accepts_get_params(app: Flask) -> None: + with app.test_request_context("/sources"): + form = SourceFilterForm( + formdata=MultiDict( + { + "status": "retrying", + "q": "example.org", + "date_from": "2026-11-08", + "date_to": "2026-11-14", + "sort": "webmentions", + "order": "asc", + "per_page": "37", + } + ), + meta={"csrf": False}, + ) + assert form.validate() + assert form.status.data == "retrying" + assert form.date_from.data.isoformat() == "2026-11-08" + assert form.date_to.data.isoformat() == "2026-11-14" + assert (form.sort.data, form.order.data, form.per_page.data) == ( + "webmentions", + "asc", + 37, + ) + assert "csrf_token" not in form._fields + + +@pytest.mark.parametrize( + "data", + [ + {"status": "unknown"}, + {"sort": "unknown"}, + {"order": "up"}, + {"per_page": "0"}, + {"per_page": "101"}, + {"per_page": "nope"}, + {"date_from": "2026-11-8"}, + {"date_to": "not-a-date"}, + {"date_from": "2026-11-15", "date_to": "2026-11-08"}, + ], +) +def test_source_filter_form_rejects_bad_get_params( + app: Flask, data: dict[str, str] +) -> None: + with app.test_request_context("/sources"): + form = SourceFilterForm(formdata=MultiDict(data), meta={"csrf": False}) + assert not form.validate() + assert form.errors + + +def test_source_filter_form_defaults_and_configured_maximum(app: Flask) -> None: + app.config["SOURCES_MAX_PAGE_SIZE"] = 10 + with app.test_request_context("/sources"): + form = SourceFilterForm(formdata=MultiDict(), meta={"csrf": False}) + assert form.validate() + assert ( + form.status.data, + form.sort.data, + form.order.data, + form.per_page.data, + ) == ("", "last_seen", "desc", 10) + form = SourceFilterForm( + formdata=MultiDict({"per_page": "11"}), meta={"csrf": False} + ) + assert not form.validate() diff --git a/tests/test_config.py b/tests/test_config.py index ca87403..75b8a4a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -382,6 +382,7 @@ def test_resource_limits_have_safe_defaults() -> None: assert DefaultConfig.MAX_WEBMENTION_URL_BYTES == 8192 assert DefaultConfig.RECEIVED_RETENTION_DAYS == 30 assert DefaultConfig.RECEIVED_MAX_PAGE_SIZE == 100 + assert DefaultConfig.SOURCES_MAX_PAGE_SIZE == 100 assert DefaultConfig.RECEIVED_RETENTION_SCHEDULE == "0 3 * * *" @@ -392,6 +393,7 @@ def test_resource_limits_have_safe_defaults() -> None: "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS", "RECEIVED_MAX_PAGE_SIZE", + "SOURCES_MAX_PAGE_SIZE", ], ) @pytest.mark.parametrize("value", [None, "100", 0, -1, 1.5]) diff --git a/tests/views/test_sources.py b/tests/views/test_sources.py index 673fc9f..c851833 100644 --- a/tests/views/test_sources.py +++ b/tests/views/test_sources.py @@ -2,11 +2,14 @@ # # SPDX-License-Identifier: BSD-3-Clause +import re import uuid from datetime import UTC, datetime, timedelta +from html import unescape from http import HTTPStatus from types import ModuleType from unittest.mock import Mock +from urllib.parse import parse_qs, urlsplit import pytest from bs4 import BeautifulSoup @@ -14,7 +17,11 @@ from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import db -from webmentions_ssg.models.sources import SentWebmention +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) from webmentions_ssg.models.user import User from .helpers import ( @@ -72,12 +79,12 @@ def test_sent_sources_table_hides_secondary_columns_on_mobile( headers = table.select("thead th") cells = table.select("tbody tr:first-child td") assert len(headers) == len(cells) == 7 - for index in (1, 3, 4, 5): # URL, revision, created, last seen + for index in (2, 4, 5, 6): # URL, revision, created, last seen for cell in (headers[index], cells[index]): assert {"d-none", "d-md-table-cell"} <= set(cell.get("class", [])) - for index in (0, 2, 6): # Path, count, status stay visible + for index in (0, 1, 3): # Path, count, status stay visible assert "d-none" not in cells[index].get("class", []) - assert "text-break" in cells[0].get("class", []) + assert "text-break" in cells[1].get("class", []) def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: @@ -298,3 +305,354 @@ def test_admin_only_links_safe_existing_status_urls( html = response.get_data(as_text=True) assert (f'href="{status_url}"' in html) is safe assert status_url in html + + +def make_source( + app: Flask, + name: str, + *, + statuses: tuple[SentWebmentionStatus | None, ...] = (SentWebmentionStatus.SENT,), + retrying: bool = False, + processed: bool = True, + deleted: bool = False, + last_seen_at: datetime | None = None, + revision: int = 1, +) -> uuid.UUID: + """Create distinguishable sources with outgoing rows for list filtering.""" + now = last_seen_at or datetime(2026, 11, 12, 12, tzinfo=UTC) + with app.app_context(): + source = SourceDocument( + path=f"blog/{name}/index.html", + url=f"https://source.example/{name}/", + content_hash="a" * 32, + revision=revision, + last_seen_at=now, + revised_at=now, + deleted_at=now if deleted else None, + ) + for number, status in enumerate(statuses): + source.sent_webmentions.append( + SentWebmention( + target=f"https://target.example/{name}/{number}", + linked=True, + desired_revision=revision, + processed_revision=revision if processed else None, + status=status, + attempt_count=2 if retrying else 0, + next_retry_at=now + timedelta(days=10) if retrying else None, + ) + ) + db.session.add(source) + db.session.commit() + return source.uuid + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("failing", {"failed", "retrying", "stale", "multiple"}), + ("retrying", {"retrying"}), + ("deleted", {"deleted"}), + ], +) +def test_sources_filter_by_delivery_state_without_duplicate_rows( + app: Flask, client: FlaskClient, status: str, expected: set[str] +) -> None: + log_in(app, client) + make_source(app, "failed", statuses=(SentWebmentionStatus.FAILED,)) + make_source( + app, + "retrying", + statuses=(SentWebmentionStatus.FAILED,), + retrying=True, + processed=False, + ) + make_source(app, "stale", statuses=(SentWebmentionStatus.FAILED,), retrying=True) + make_source( + app, + "multiple", + statuses=(SentWebmentionStatus.FAILED, SentWebmentionStatus.FAILED), + ) + make_source(app, "deleted", deleted=True) + make_source(app, "sent") + make_source(app, "pending", statuses=(None,), processed=False) + + response = client.get("/sources", query_string={"status": status}) + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + paths = [ + row.select_one("td a").get_text(strip=True) for row in soup.select("tbody tr") + ] + assert set(paths) == {f"blog/{name}/index.html" for name in expected} + assert len(paths) == len(expected) + assert ( + soup.select_one(f'select[name="status"] option[value="{status}"][selected]') + is not None + ) + + +@pytest.mark.parametrize( + ("q", "expected"), + [ + ("blog/alpha", "alpha"), + ("source.example/alpha", "alpha"), + ("target.example/alpha", "alpha"), + ("alpha/0", "alpha"), + ("alpha%", None), + ], +) +def test_sources_search_matches_path_source_url_and_sent_target( + app: Flask, client: FlaskClient, q: str, expected: str | None +) -> None: + log_in(app, client) + make_source(app, "alpha") + make_source(app, "beta") + + response = client.get("/sources", query_string={"q": f" {q} "}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert ("blog/alpha/index.html" in html) == (expected == "alpha") + assert "blog/beta/index.html" not in html + if expected is None: + assert "No sources match these filters." in html + + +def test_sources_filter_by_inclusive_last_seen_dates( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + make_source(app, "old", last_seen_at=datetime(2026, 11, 7, tzinfo=UTC)) + make_source(app, "included", last_seen_at=datetime(2026, 11, 10, 22, tzinfo=UTC)) + make_source(app, "new", last_seen_at=datetime(2026, 11, 15, tzinfo=UTC)) + response = client.get( + "/sources", query_string={"date_from": "2026-11-08", "date_to": "2026-11-10"} + ) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "blog/included/index.html" in html + assert "blog/old/index.html" not in html + assert "blog/new/index.html" not in html + + +@pytest.mark.parametrize( + "sort", ["path", "url", "webmentions", "revision", "created", "last_seen", "status"] +) +def test_sources_sort_links_preserve_filters( + app: Flask, client: FlaskClient, sort: str +) -> None: + log_in(app, client) + make_source(app, "alpha", statuses=(SentWebmentionStatus.FAILED,)) + response = client.get( + "/sources", + query_string={ + "status": "failing", + "q": "alpha", + "date_from": "2026-11-01", + "sort": sort, + "order": "asc", + "per_page": 37, + }, + ) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + soup = BeautifulSoup(html, "html.parser") + assert soup.select_one('th[aria-sort="ascending"]') is not None + links = [ + parse_qs(urlsplit(unescape(a["href"])).query) + for a in soup.select("th a[href]") + if urlsplit(unescape(a["href"])).path == "/sources" + ] + assert len(links) == 7 + for link in links: + assert link["status"] == ["failing"] + assert link["q"] == ["alpha"] + assert link["date_from"] == ["2026-11-01"] + assert link["per_page"] == ["37"] + assert link["sort"][0] in ( + "path", + "url", + "webmentions", + "revision", + "created", + "last_seen", + "status", + ) + assert any(link["sort"] == [sort] and link["order"] == ["desc"] for link in links) + + +def test_sources_sort_orders_by_count_and_path(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + make_source(app, "z", statuses=(SentWebmentionStatus.SENT,)) + make_source( + app, "a", statuses=(SentWebmentionStatus.SENT, SentWebmentionStatus.FAILED) + ) + for sort, order, expected_first in ( + ("webmentions", "desc", "a"), + ("webmentions", "asc", "z"), + ("path", "asc", "a"), + ("path", "desc", "z"), + ): + response = client.get("/sources", query_string={"sort": sort, "order": order}) + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + assert ( + soup.select_one("tbody tr td a").get_text(strip=True) + == f"blog/{expected_first}/index.html" + ) + + +def test_sources_filter_pagination_and_navigation_preserve_parameters( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + for number in range(28): + make_source(app, f"failed-{number:02}", statuses=(SentWebmentionStatus.FAILED,)) + filters = { + "status": "failing", + "q": "failed", + "sort": "path", + "order": "asc", + "per_page": "10", + } + response = client.get("/sources", query_string={**filters, "page": "2"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "28 sources" in html + soup = BeautifulSoup(html, "html.parser") + assert len(soup.select("tbody tr")) == 10 + urls = [unescape(a["href"]) for a in soup.select("a[href]")] + assert any( + urlsplit(url).path == "/sources" + and parse_qs(urlsplit(url).query) + == {**{k: [v] for k, v in filters.items()}, "page": ["3"]} + for url in urls + ) + detail_urls = [ + url for url in urls if re.search(r"/sources/[0-9a-f-]{36}", urlsplit(url).path) + ] + assert detail_urls + assert parse_qs(urlsplit(detail_urls[0]).query) == { + **{k: [v] for k, v in filters.items()}, + "source_page": ["2"], + } + detail = client.get(detail_urls[0]) + assert detail.status_code == HTTPStatus.OK + back = BeautifulSoup(detail.get_data(as_text=True), "html.parser").select_one( + 'a[href*="/sources?"]' + ) + assert back is not None + assert parse_qs(urlsplit(unescape(back["href"])).query) == { + **{k: [v] for k, v in filters.items()}, + "page": ["2"], + } + + +@pytest.mark.parametrize( + "invalid", + [ + "status=unknown", + "sort=unsupported", + "order=reverse", + "per_page=0", + "per_page=101", + "per_page=abc", + "date_from=2026-11-8", + "date_to=bad", + "date_from=2026-11-10&date_to=2026-11-01", + ], +) +def test_sources_bad_filters_return_400( + app: Flask, client: FlaskClient, invalid: str +) -> None: + log_in(app, client) + assert client.get(f"/sources?{invalid}").status_code == HTTPStatus.BAD_REQUEST + + +def test_rescan_sources_preserves_filter_query( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + scan = Mock() + monkeypatch.setattr(sources_module, "manual_scan_sources", scan) + response = client.post( + "/sources/rescan", + query_string={"status": "retrying", "per_page": 37, "page": 2}, + ) + assert response.status_code == HTTPStatus.FOUND + assert parse_qs(urlsplit(response.headers["Location"]).query) == { + "status": ["retrying"], + "per_page": ["37"], + "page": ["2"], + } + scan.assert_called_once_with() + + +def test_sources_show_failed_and_retrying_counts_and_retry_deadline( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + identifier = make_source( + app, + "retry-badges", + statuses=(SentWebmentionStatus.FAILED,), + retrying=True, + processed=False, + ) + response = client.get("/sources", query_string={"status": "retrying"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "1 failed" in html + assert "1 awaiting retry" in html + + response = client.get(f"/sources/{identifier}") + assert response.status_code == HTTPStatus.OK + assert "Retry scheduled after attempt 2:" in response.get_data(as_text=True) + + +def test_resend_preserves_source_filter_context( + app: Flask, + client: FlaskClient, + sources_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + log_in(app, client) + source_id, mention_id = create_resendable_sent_webmention(app) + mock_admin_action_form(sources_module, monkeypatch, valid=True) + send = Mock() + monkeypatch.setattr(sources_module, "send_webmention", send) + + response = client.post( + f"/sources/{source_id}/{mention_id}/resend", + query_string={ + "status": "failing", + "q": "resend", + "sort": "path", + "source_page": 2, + "page": 3, + }, + ) + assert response.status_code == HTTPStatus.FOUND + assert urlsplit(response.headers["Location"]).path == f"/sources/{source_id}" + assert parse_qs(urlsplit(response.headers["Location"]).query) == { + "status": ["failing"], + "q": ["resend"], + "sort": ["path"], + "source_page": ["2"], + "page": ["3"], + } + send.assert_called_once_with(mention_id) + + +def test_sources_empty_filter_result_keeps_rescan_modal( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + response = client.get("/sources", query_string={"status": "failing"}) + assert response.status_code == HTTPStatus.OK + html = response.get_data(as_text=True) + assert "No sources match these filters." in html + assert 'id="confirmation-modal"' in html diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index be54989..3d05e5d 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -91,6 +91,8 @@ class DefaultConfig: RECEIVED_RETENTION_DAYS = 30 # Maximum rows that an admin can request per received Webmentions page. RECEIVED_MAX_PAGE_SIZE = 100 + # Maximum rows that an admin can request per source document page. + SOURCES_MAX_PAGE_SIZE = 100 RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" # Scanner @@ -246,6 +248,7 @@ def validated_config_overrides( "MAX_WEBMENTION_URL_BYTES", "RECEIVED_RETENTION_DAYS", "RECEIVED_MAX_PAGE_SIZE", + "SOURCES_MAX_PAGE_SIZE", ): value = config[key] if type(value) is not int or value <= 0: diff --git a/webmentions_ssg/forms/sources.py b/webmentions_ssg/forms/sources.py new file mode 100644 index 0000000..04afe77 --- /dev/null +++ b/webmentions_ssg/forms/sources.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source document filtering form.""" + +from flask import current_app +from flask_wtf import FlaskForm +from wtforms import ( + DateField, + HiddenField, + IntegerField, + SearchField, + SelectField, + ValidationError, +) +from wtforms.validators import AnyOf, NumberRange, Optional + +from .validators import CanonicalDate + +SOURCE_STATUS_CHOICES = { + "": "All sources", + "failing": "Failing sends", + "retrying": "Awaiting retry", + "deleted": "Deleted sources", +} + + +def default_source_page_size() -> int: + """Use 25 rows unless the configured maximum is lower.""" + return min(25, current_app.config["SOURCES_MAX_PAGE_SIZE"]) + + +class SourceFilterForm(FlaskForm): + """GET filters for the source document list.""" + + status = SelectField("Status", choices=SOURCE_STATUS_CHOICES, default="") + q = SearchField("Path or URL") + date_from = DateField( + "Last seen from (UTC)", validators=[Optional(), CanonicalDate()] + ) + date_to = DateField("Last seen to (UTC)", validators=[Optional(), CanonicalDate()]) + sort = HiddenField( + default="last_seen", + validators=[ + AnyOf( + ( + "path", + "url", + "webmentions", + "revision", + "created", + "last_seen", + "status", + ) + ) + ], + ) + order = HiddenField(default="desc", validators=[AnyOf(("asc", "desc"))]) + per_page = IntegerField( + "Rows per page", + default=default_source_page_size, + validators=[NumberRange(min=1)], + ) + + def validate_per_page(self, field: IntegerField) -> None: + """Enforce the deployment's maximum source-list page size.""" + if ( + field.data is not None + and field.data > current_app.config["SOURCES_MAX_PAGE_SIZE"] + ): + raise ValidationError("Rows per page exceeds the configured maximum.") + + def validate_date_to(self, field: DateField) -> None: + """Require an end date on or after the start date.""" + if ( + self.date_from.data is not None + and field.data is not None + and field.data < self.date_from.data + ): + raise ValidationError("End date must be on or after start date.") diff --git a/webmentions_ssg/templates/sent.html b/webmentions_ssg/templates/sent.html index d2a319d..6febdb9 100644 --- a/webmentions_ssg/templates/sent.html +++ b/webmentions_ssg/templates/sent.html @@ -7,6 +7,19 @@ SPDX-License-Identifier: BSD-3-Clause {% from "bootstrap5/pagination.html" import render_pagination %} +{% macro sortable_header(label, column, responsive_class="") %} + {% set active = sort_key == column %} + <th scope="col" + class="{{ responsive_class }}" + aria-sort="{{ ('ascending' if sort_order == 'asc' else 'descending') if active else 'none' }}"> + <a href="{{ sort_urls[column] }}" + class="link-body-emphasis text-decoration-none"> + {{ label }} + {% if active %}<span aria-hidden="true">{{ "↑" if sort_order == "asc" else "↓" }}</span>{% endif %} + </a> + </th> +{% endmacro %} + {% block content %} <div class="d-flex justify-content-between align-items-center mb-3"> <h1 class="mb-0">Sources</h1> @@ -16,29 +29,72 @@ SPDX-License-Identifier: BSD-3-Clause class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#confirmation-modal" - data-action="{{ url_for('webmentions.rescan_sources') }}" + data-action="{{ url_for('webmentions.rescan_sources', page=sources.page, **filters) }}" data-title="Rescan sources?" data-message="All sources will be scanned again for changes and webmentions." data-confirm-label="Rescan" data-confirm-class="btn-primary">Rescan</button> </div> </div> + <form action="{{ url_for('webmentions.source_list') }}" + method="get" + class="row g-2 align-items-end mb-4" + role="search"> + {{ form.sort }} + {{ form.order }} + <div class="col-sm-6 col-lg"> + {{ form.status.label(class_="form-label") }} + {{ form.status(class_="form-select") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.q.label(class_="form-label") }} + {{ form.q(class_="form-control", placeholder="Search paths and URLs") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.date_from.label(class_="form-label") }} + {{ form.date_from(class_="form-control") }} + </div> + <div class="col-sm-6 col-lg"> + {{ form.date_to.label(class_="form-label") }} + {{ form.date_to(class_="form-control") }} + </div> + <div class="col-sm-6 col-lg-auto"> + {{ form.per_page.label(class_="form-label") }} + {{ form.per_page(class_="form-control", min=1, max=max_source_page_size, step=1) }} + </div> + <div class="col-auto d-flex gap-2 ms-auto"> + <button type="submit" class="btn btn-primary">Filter</button> + <a href="{{ url_for('webmentions.source_list') }}" + class="btn btn-outline-secondary">Clear</a> + </div> + </form> {% if sources.items %} <div class="table-responsive"> <table class="table table-bordered table-striped table-hover align-middle"> <thead> <tr> - <th scope="col">Path</th> - <th scope="col" class="d-none d-md-table-cell">URL</th> - <th scope="col">Webmentions</th> - <th scope="col" class="d-none d-md-table-cell">Revision</th> - <th scope="col" class="d-none d-md-table-cell">Created</th> - <th scope="col" class="d-none d-md-table-cell">Last seen</th> - <th scope="col">Status</th> + {{ sortable_header('Status', 'status') }} + {{ sortable_header('Path', 'path') }} + {{ sortable_header('URL', 'url', 'd-none d-md-table-cell') }} + {{ sortable_header('Webmentions', 'webmentions') }} + {{ sortable_header('Revision', 'revision', 'd-none d-md-table-cell') }} + {{ sortable_header('Created', 'created', 'd-none d-md-table-cell') }} + {{ sortable_header('Last seen', 'last_seen', 'd-none d-md-table-cell') }} </tr> </thead> <tbody> {% for source in sources %} + {% set statistics = namespace(sent=0, failed=0, retrying=0) %} + {% for webmention in source.sent_webmentions %} + {% if webmention.status == "sent" %} + {% set statistics.sent = statistics.sent + 1 %} + {% elif webmention.status == "failed" %} + {% set statistics.failed = statistics.failed + 1 %} + {% endif %} + {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %} + {% set statistics.retrying = statistics.retrying + 1 %} + {% endif %} + {% endfor %} {% if source.deleted_at is none %} {% set row_class = "table-success" %} {% set badge_class = "text-bg-success" %} @@ -49,13 +105,27 @@ SPDX-License-Identifier: BSD-3-Clause {% set status_text = "Deleted" %} {% endif %} <tr class="{{ row_class }}"> + <td> + <span class="badge {{ badge_class }}">{{ status_text }}</span> + </td> <td class="text-break"> - <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid) }}">{{ source.path }}</a> + <a href="{{ url_for('webmentions.source_detail', source_uuid=source.uuid, source_page=sources.page, **filters) }}">{{ source.path }}</a> </td> <td class="text-break d-none d-md-table-cell"> <a href="{{ source.url }}">{{ source.url }}</a> </td> - <td>{{ source.sent_webmentions | length }}</td> + <td> + <span class="badge text-bg-info">{{ source.sent_webmentions | length }} Total</span> + {% if statistics.sent %} + <span class="badge text-bg-success">{{ statistics.sent }} sent</span> + {% endif %} + {% if statistics.failed %} + <span class="badge text-bg-danger">{{ statistics.failed }} failed</span> + {% endif %} + {% if statistics.retrying %} + <span class="badge text-bg-warning">{{ statistics.retrying }} awaiting retry</span> + {% endif %} + </td> <td class="d-none d-md-table-cell">{{ source.revision }}</td> <td class="text-nowrap d-none d-md-table-cell"> <time datetime="{{ source.created_at | utc_isoformat }}" @@ -69,21 +139,22 @@ SPDX-License-Identifier: BSD-3-Clause {{ source.last_seen_at | relative_time }} </time> </td> - <td> - <span class="badge {{ badge_class }}">{{ status_text }}</span> - </td> </tr> {% endfor %} </tbody> </table> </div> {% if sources.pages > 1 %} - <div class="mt-3">{{ render_pagination(sources) }}</div> + <div class="mt-3">{{ render_pagination(sources, args=filters) }}</div> {% endif %} - {% include "includes/confirmation_modal.html" %} {% else %} - <p class="text-body-secondary">No Webmention sources have been discovered yet.</p> + {% if filters %} + <p class="text-body-secondary">No sources match these filters.</p> + {% else %} + <p class="text-body-secondary">No Webmention sources have been discovered yet.</p> + {% endif %} {% endif %} + {% include "includes/confirmation_modal.html" %} {% endblock %} {% block scripts %} diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 33e26a2..04f9d07 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -26,7 +26,7 @@ SPDX-License-Identifier: BSD-3-Clause {% block content %} <div class="mb-3"> - <a href="{{ url_for('webmentions.source_list') }}">← Back to sources</a> + <a href="{{ url_for('webmentions.source_list', page=source_page, **filters) }}">← Back to sources</a> </div> <div class="d-flex justify-content-between align-items-start mb-3"> <h1 class="mb-0">Source Details</h1> @@ -96,8 +96,8 @@ SPDX-License-Identifier: BSD-3-Clause <table class="table table-bordered table-striped table-hover align-middle"> <thead> <tr> - <th scope="col">Target</th> <th scope="col">Status</th> + <th scope="col">Target</th> <th scope="col" class="d-none d-md-table-cell">Active</th> <th scope="col" class="d-none d-md-table-cell">Revisions</th> <th scope="col" class="d-none d-md-table-cell">Delivery</th> @@ -110,13 +110,22 @@ SPDX-License-Identifier: BSD-3-Clause {% set style = sent_status_presentation.get(webmention.status, {"row_class": "table-info", "badge_class": "text-bg-info", "label": "Pending" }) %} <tr class="{{ style.row_class }}"> <td class="text-break"> - <a href="{{ webmention.target }}">{{ webmention.target }}</a> - </td> - <td class="text-break"> <span class="badge {{ style.badge_class }}">{{ style.label }}</span> {% if webmention.failure_reason is not none %} <div class="small mt-2">{{ webmention.failure_reason }}</div> {% endif %} + {% if webmention.awaiting_processing and webmention.attempt_count > 0 and webmention.next_retry_at is not none %} + <div class="small mt-2"> + Retry scheduled after attempt {{ webmention.attempt_count }}: + <time datetime="{{ webmention.next_retry_at | utc_isoformat }}" + title="{{ webmention.next_retry_at | utc_timestamp }}"> + {{ webmention.next_retry_at | relative_time }} + </time> + </div> + {% endif %} + </td> + <td class="text-break"> + <a href="{{ webmention.target }}">{{ webmention.target }}</a> </td> <td class="d-none d-md-table-cell"> {% if webmention.linked %} @@ -199,7 +208,7 @@ SPDX-License-Identifier: BSD-3-Clause class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#confirmation-modal" - data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid) }}" + data-action="{{ url_for('webmentions.resend_webmention', source_uuid=source.uuid, webmention_uuid=webmention.uuid, source_page=source_page, page=webmentions.page, **filters) }}" data-title="Resend Webmention?" data-message="This Webmention will be queued again for delivery to {{ webmention.target }}." data-confirm-label="Resend" @@ -211,7 +220,7 @@ SPDX-License-Identifier: BSD-3-Clause </table> </div> {% if webmentions.pages > 1 %} - <div class="mt-3">{{ render_pagination(webmentions) }}</div> + <div class="mt-3">{{ render_pagination(webmentions, args=dict(filters, source_page=source_page)) }}</div> {% endif %} {% include "includes/confirmation_modal.html" %} {% else %} diff --git a/webmentions_ssg/views/sources.py b/webmentions_ssg/views/sources.py index b68d2fd..1c9cc58 100644 --- a/webmentions_ssg/views/sources.py +++ b/webmentions_ssg/views/sources.py @@ -5,39 +5,166 @@ """Authenticated source document and sent Webmention views.""" import uuid +from datetime import UTC, date, datetime, time from http import HTTPStatus +from typing import TypedDict, cast import sqlalchemy as sa -from flask import abort, flash, redirect, render_template, url_for +from flask import abort, current_app, flash, redirect, render_template, request, url_for from flask.typing import ResponseReturnValue from flask_login import login_required from sqlalchemy.orm import selectinload from ..extensions import db from ..forms.admin import ConfirmActionForm -from ..models.sources import SentWebmention, SourceDocument +from ..forms.sources import ( + SOURCE_STATUS_CHOICES, + SourceFilterForm, + default_source_page_size, +) +from ..models.sources import SentWebmention, SentWebmentionStatus, SourceDocument from ..tasks.scanner import manual_scan_sources from ..tasks.sender import send_webmention from .blueprint import webmentions_bp +class SourceFilterArgs(TypedDict, total=False): + """Validated source-list query parameters; keys match GET form fields.""" + + status: str + q: str + date_from: str + date_to: str + sort: str + order: str + per_page: str + + +def source_filter_form() -> SourceFilterForm: + """Validate GET filters without requiring CSRF on the search form.""" + form = SourceFilterForm(formdata=request.args, meta={"csrf": False}) + if not form.validate(): + abort(HTTPStatus.BAD_REQUEST) + form.q.data = (form.q.data or "").strip() + return form + + +def source_filter_args(form: SourceFilterForm) -> SourceFilterArgs: + """Serialize non-default values for pagination, sorting, and navigation.""" + values: dict[str, str] = {} + for name in SourceFilterArgs.__annotations__: + field = form[name] + value = field.data + default = field.default() if callable(field.default) else field.default + if value is None or value == "" or value == default: + continue + values[name] = value.isoformat() if isinstance(value, date) else str(value) + return cast(SourceFilterArgs, values) + + @webmentions_bp.route("/sources") @login_required def source_list() -> ResponseReturnValue: - """Display sources with sent Webmentions. + """Display source documents with filters for outgoing delivery problems.""" + form = source_filter_form() + filters = source_filter_args(form) + query = sa.select(SourceDocument).options( + selectinload(SourceDocument.sent_webmentions) + ) - :returns: Rendered page containing the paginated source list. - """ + # EXISTS avoids duplicate source rows when several Webmentions match. + sent = sa.select(SentWebmention.uuid).where( + SentWebmention.source_uuid == SourceDocument.uuid + ) + if status := filters.get("status"): + if status == "failing": + query = query.where( + sent.where( + SentWebmention.status == SentWebmentionStatus.FAILED + ).exists() + ) + elif status == "retrying": + query = query.where( + sent.where( + SentWebmention.awaiting_processing, + SentWebmention.attempt_count > 0, + SentWebmention.next_retry_at.is_not(None), + ).exists() + ) + else: # deleted + query = query.where(SourceDocument.deleted_at.is_not(None)) + + if text := filters.get("q"): + query = query.where( + sa.or_( + SourceDocument.path.icontains(text, autoescape=True), + SourceDocument.url.icontains(text, autoescape=True), + sent.where( + SentWebmention.target.icontains(text, autoescape=True) + ).exists(), + ) + ) + + if start := filters.get("date_from"): + query = query.where( + SourceDocument.last_seen_at + >= datetime.combine(date.fromisoformat(start), time.min, UTC) + ) + if end := filters.get("date_to"): + query = query.where( + SourceDocument.last_seen_at + <= datetime.combine(date.fromisoformat(end), time.max, UTC) + ) + + count = ( + sa.select(sa.func.count(SentWebmention.uuid)) + .where(SentWebmention.source_uuid == SourceDocument.uuid) + .scalar_subquery() + ) + sort_columns = { + "path": SourceDocument.path, + "url": SourceDocument.url, + "webmentions": count, + "revision": SourceDocument.revision, + "created": SourceDocument.uuid, + "last_seen": SourceDocument.last_seen_at, + "status": sa.case((SourceDocument.deleted_at.is_(None), 0), else_=1), + } + sort_key = filters.get("sort", "last_seen") + sort_order = filters.get("order", "desc") + column = sort_columns[sort_key] + query = query.order_by(column.asc() if sort_order == "asc" else column.desc()) + if sort_key != "created": + query = query.order_by(SourceDocument.uuid.desc()) + + page_size = int(filters.get("per_page", default_source_page_size())) sources = db.paginate( - sa.select(SourceDocument) - .options(selectinload(SourceDocument.sent_webmentions)) - .order_by(SourceDocument.last_seen_at.desc()), - per_page=25, + query, + per_page=page_size, + max_per_page=current_app.config["SOURCES_MAX_PAGE_SIZE"], ) + + sort_urls = {} + for key in sort_columns: + if key == sort_key: + next_order = "asc" if sort_order == "desc" else "desc" + else: + next_order = "asc" if key in ("path", "url", "status") else "desc" + sort_filters: SourceFilterArgs = {**filters, "sort": key, "order": next_order} + sort_urls[key] = url_for("webmentions.source_list", **sort_filters) + return render_template( "sent.html", title="Sent Webmentions", sources=sources, + form=form, + filters=filters, + sort_key=sort_key, + sort_order=sort_order, + sort_urls=sort_urls, + status_choices=SOURCE_STATUS_CHOICES, + page_size=page_size, + max_source_page_size=current_app.config["SOURCES_MAX_PAGE_SIZE"], action_form=ConfirmActionForm(), ) @@ -52,10 +179,15 @@ def rescan_sources() -> ResponseReturnValue: if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) + filters = source_filter_args(source_filter_form()) manual_scan_sources() flash("Source rescan queued.", "success") - return redirect(url_for("webmentions.source_list")) + query_args: dict[str, str | int] = dict(filters) + page = request.args.get("page", 1, type=int) + if page > 1: + query_args["page"] = page + return redirect(url_for("webmentions.source_list", **query_args)) @webmentions_bp.route("/sources/<uuid:source_uuid>") @@ -66,6 +198,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: :param source_uuid: UUID of the source to display. :returns: Rendered source details page. """ + filters = source_filter_args(source_filter_form()) + source_page = request.args.get("source_page", 1, type=int) source = db.get_or_404(SourceDocument, source_uuid) webmentions = db.paginate( @@ -80,6 +214,8 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: title="Sent Webmentions", source=source, webmentions=webmentions, + filters=filters, + source_page=source_page, action_form=ConfirmActionForm(), ) @@ -103,6 +239,7 @@ def resend_webmention( if not ConfirmActionForm().validate_on_submit(): abort(HTTPStatus.BAD_REQUEST) + filters = source_filter_args(source_filter_form()) webmention = db.get_or_404(SentWebmention, webmention_uuid) if webmention.source_uuid != source_uuid: @@ -119,4 +256,11 @@ def resend_webmention( flash(f"Webmention {webmention.uuid} queued for resend.", "success") - return redirect(url_for("webmentions.source_detail", source_uuid=source_uuid)) + detail_args: dict[str, str | int] = dict(filters) + for name in ("page", "source_page"): + page = request.args.get(name, 1, type=int) + if page > 1: + detail_args[name] = page + return redirect( + url_for("webmentions.source_detail", source_uuid=source_uuid, **detail_args) + ) |
