From b908ee61770dc78f8d8fbbbfc4f3c00d7877d42a Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Wed, 18 Nov 2026 08:01:26 +0100 Subject: feat(core): add dashboard overview Replace the root redirect with an authenticated dashboard showing received and outgoing status counts, recent activity, pending sends, retries, and queue depth. Persist the last completed source scan so operational status survives restarts and only advances after a successful, conclusive scan. --- tests/tasks/test_scanner.py | 57 ++++++ tests/test_cli.py | 1 + tests/views/test_dashboard.py | 202 +++++++++++++++++++++ tests/views/test_session.py | 4 +- .../2b4c6e8a0d12_last_completed_source_scan.py | 38 ++++ webmentions_ssg/models/scan_state.py | 23 +++ webmentions_ssg/tasks/scanner.py | 6 + webmentions_ssg/templates/base.html | 2 + webmentions_ssg/templates/dashboard.html | 148 +++++++++++++++ webmentions_ssg/views/__init__.py | 3 +- webmentions_ssg/views/dashboard.py | 123 +++++++++++++ webmentions_ssg/views/session.py | 9 - 12 files changed, 604 insertions(+), 12 deletions(-) create mode 100644 tests/views/test_dashboard.py create mode 100644 webmentions_ssg/migrations/versions/2b4c6e8a0d12_last_completed_source_scan.py create mode 100644 webmentions_ssg/models/scan_state.py create mode 100644 webmentions_ssg/templates/dashboard.html create mode 100644 webmentions_ssg/views/dashboard.py diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index dbeeac9..af5a64e 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -17,6 +17,7 @@ from huey.exceptions import RetryTask from pytest import MonkeyPatch from webmentions_ssg import db +from webmentions_ssg.models.scan_state import ScanState from webmentions_ssg.models.sources import ( SentWebmention, SentWebmentionStatus, @@ -1464,3 +1465,59 @@ def test_scan_sources_ignores_root_index( assert len(sources) == 1 assert sources[0].path == "example/index.html" + + +def test_scan_records_only_completed_source_scans( + app: Flask, scanner_module: ModuleType, tmp_path: Path +) -> None: + configure_scanner(app, tmp_path) + with app.app_context(): + assert db.session.get(ScanState, 1) is None + + # Empty traversals are inconclusive and do not report completion. + run_scan(app, scanner_module) + with app.app_context(): + assert db.session.get(ScanState, 1) is None + + write_post(tmp_path, "completed", "Completed scan") + before = datetime.now(UTC) + run_scan(app, scanner_module) + after = datetime.now(UTC) + with app.app_context(): + state = db.session.get(ScanState, 1) + assert state is not None + recorded = state.last_completed_at.replace(tzinfo=UTC) + assert before <= recorded <= after + + # Another inconclusive pass must preserve the completion time. + (tmp_path / "completed" / "index.html").unlink() + run_scan(app, scanner_module) + with app.app_context(): + state = db.session.get(ScanState, 1) + assert state is not None + assert state.last_completed_at.replace(tzinfo=UTC) == recorded + + +def test_failed_scan_does_not_update_completion_time( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + write_post(tmp_path, "existing", "Scan succeeded") + run_scan(app, scanner_module) + + with app.app_context(): + previous = db.session.get(ScanState, 1) + assert previous is not None + previous_time = previous.last_completed_at + + def fail(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("parse failed") + + monkeypatch.setattr(scanner_module, "reconcile_source_tree", fail) + with pytest.raises(RuntimeError, match="parse failed"): + run_scan(app, scanner_module) + + with app.app_context(): + state = db.session.get(ScanState, 1) + assert state is not None + assert state.last_completed_at == previous_time diff --git a/tests/test_cli.py b/tests/test_cli.py index 5e71e2a..5699bb3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -148,6 +148,7 @@ def test_create_db_builds_schema_and_records_latest_migration( "users", "received_webmentions", "source_documents", + "scan_state", } <= table_names assert "sources" not in table_names assert { diff --git a/tests/views/test_dashboard.py b/tests/views/test_dashboard.py new file mode 100644 index 0000000..05730dd --- /dev/null +++ b/tests/views/test_dashboard.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +from datetime import UTC, datetime, timedelta +from http import HTTPStatus +from uuid import UUID + +import pytest +from bs4 import BeautifulSoup +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import db +from webmentions_ssg.models.received import ReceivedWebmention, ReceivedWebmentionStatus +from webmentions_ssg.models.scan_state import ScanState +from webmentions_ssg.models.sources import ( + SentWebmention, + SentWebmentionStatus, + SourceDocument, +) +from webmentions_ssg.views import dashboard + +from .helpers import log_in + + +def received_uuid(at: datetime, sequence: int) -> UUID: + """Construct a UUIDv7 at a controlled creation time for activity tests.""" + return UUID( + int=(int(at.timestamp() * 1000) << 80) | (7 << 76) | (2 << 62) | sequence + ) + + +def test_dashboard_is_private_and_empty_by_default( + app: Flask, client: FlaskClient +) -> None: + response = client.get("/") + assert response.status_code == HTTPStatus.FOUND + assert response.headers["Location"].startswith("/login?") + + log_in(app, client) + response = client.get("/") + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + received = soup.find("section", attrs={"aria-labelledby": "received-summary"}) + outgoing = soup.find("section", attrs={"aria-labelledby": "sent-summary"}) + assert received is not None + assert outgoing is not None + assert "0 total" in received.get_text(" ", strip=True) + assert "0 total" in outgoing.get_text(" ", strip=True) + assert "No completed scan yet" in response.get_data(as_text=True) + assert "Huey pending tasks" in response.get_data(as_text=True) + assert 'href="/received?status=declined"' in response.get_data(as_text=True) + assert 'href="/"' in response.get_data(as_text=True) + + +def test_dashboard_counts_statuses_windows_retries_and_queue( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + log_in(app, client) + now = datetime.now(UTC) + with app.app_context(): + for sequence, (status, days) in enumerate( + ( + (ReceivedWebmentionStatus.RECEIVED, 1), + (ReceivedWebmentionStatus.VERIFYING, 8), + (ReceivedWebmentionStatus.VERIFIED, 31), + (ReceivedWebmentionStatus.FAILED, 2), + (ReceivedWebmentionStatus.SOURCE_REMOVED, 3), + ), + start=1, + ): + db.session.add( + ReceivedWebmention( + uuid=received_uuid(now - timedelta(days=days), sequence), + source=f"https://source.example/{sequence}", + target="https://site.example/post", + status=status, + # Old updates must not change creation-based activity counts. + updated_at=now, + ) + ) + source = SourceDocument( + path="post/index.html", + url="https://site.example/post", + content_hash="0" * 32, + revision=2, + last_seen_at=now, + revised_at=now, + ) + source.sent_webmentions.extend( + [ + SentWebmention( + target="https://target.example/success", + desired_revision=2, + processed_revision=2, + status=SentWebmentionStatus.SENT, + last_sent_at=now - timedelta(days=3), + ), + SentWebmention( + target="https://target.example/unsupported", + desired_revision=2, + processed_revision=2, + status=SentWebmentionStatus.NO_ENDPOINT, + ), + SentWebmention( + target="https://target.example/failed", + desired_revision=2, + processed_revision=2, + status=SentWebmentionStatus.FAILED, + ), + SentWebmention( + target="https://target.example/retry", + desired_revision=2, + status=SentWebmentionStatus.FAILED, + attempt_count=1, + next_retry_at=now + timedelta(hours=1), + ), + SentWebmention( + target="https://target.example/pending", desired_revision=2 + ), + SentWebmention( + target="https://target.example/not-attempted", + desired_revision=2, + processed_revision=2, + linked=False, + ), + # A claimed in-flight send has a future deadline but no + # recorded failure and must not count as waiting to retry. + SentWebmention( + target="https://target.example/inflight", + desired_revision=2, + next_retry_at=now + timedelta(hours=1), + ), + ] + ) + db.session.add(source) + db.session.add(ScanState(id=1, last_completed_at=now)) + db.session.commit() + + monkeypatch.setattr(dashboard.huey.huey, "pending_count", lambda: 4) + response = client.get("/") + assert response.status_code == HTTPStatus.OK + soup = BeautifulSoup(response.get_data(as_text=True), "html.parser") + + received = soup.find("section", attrs={"aria-labelledby": "received-summary"}) + outgoing = soup.find("section", attrs={"aria-labelledby": "sent-summary"}) + assert received is not None + assert outgoing is not None + received_text = received.get_text(" ", strip=True) + outgoing_text = outgoing.get_text(" ", strip=True) + assert "5 total" in received_text + assert "Pending 1" in received_text + assert "Verifying 1" in received_text + assert "Verified 1" in received_text + assert "Declined 2" in received_text + assert "Failed: 1 · Source removed: 1" in received_text + assert "7 total" in outgoing_text + assert "Sent 1" in outgoing_text + assert "No endpoint 1" in outgoing_text + assert "Failed (final) 1" in outgoing_text + assert "Not attempted 1" in outgoing_text + assert "Pending 3" in outgoing_text + assert "Waiting to retry: 1" in outgoing_text + + recent = soup.find("section", attrs={"aria-labelledby": "recent-activity"}) + assert recent is not None + rows = recent.find_all("tr") + received_cells = rows[1].find_all(["td", "th"]) + assert [cell.get_text(" ", strip=True) for cell in received_cells] == [ + "New received Webmentions", + "3", + "4", + ] + sent_cells = rows[2].find_all(["td", "th"]) + assert [cell.get_text(" ", strip=True) for cell in sent_cells] == [ + "Webmentions last sent successfully", + "1", + "1", + ] + operations = soup.find("section", attrs={"aria-labelledby": "operations"}) + assert operations is not None + assert "Source documents 1" in operations.get_text(" ", strip=True) + assert "Huey pending tasks 4" in operations.get_text(" ", strip=True) + assert "Last completed source scan" in operations.get_text(" ", strip=True) + assert operations.find("time") is not None + + +def test_dashboard_handles_unavailable_queue( + app: Flask, client: FlaskClient, monkeypatch: pytest.MonkeyPatch +) -> None: + log_in(app, client) + + def unavailable() -> int: + raise RuntimeError("backend unavailable") + + monkeypatch.setattr(dashboard.huey.huey, "pending_count", unavailable) + response = client.get("/") + + assert response.status_code == HTTPStatus.OK + assert "Huey pending tasks" in response.get_data(as_text=True) + assert "Unavailable" in response.get_data(as_text=True) diff --git a/tests/views/test_session.py b/tests/views/test_session.py index 7edee71..f110a26 100644 --- a/tests/views/test_session.py +++ b/tests/views/test_session.py @@ -436,8 +436,8 @@ def test_logout_ends_session(app: Flask, client: FlaskClient) -> None: assert "/login" in protected_response.headers["Location"] -def test_index_redirects_to_received(client: FlaskClient) -> None: +def test_index_requires_login(client: FlaskClient) -> None: response = client.get("/") assert response.status_code == HTTPStatus.FOUND - assert response.headers["Location"].endswith("/received") + assert "/login" in response.headers["Location"] diff --git a/webmentions_ssg/migrations/versions/2b4c6e8a0d12_last_completed_source_scan.py b/webmentions_ssg/migrations/versions/2b4c6e8a0d12_last_completed_source_scan.py new file mode 100644 index 0000000..168647f --- /dev/null +++ b/webmentions_ssg/migrations/versions/2b4c6e8a0d12_last_completed_source_scan.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Track the last successful source scan. + +Revision ID: 2b4c6e8a0d12 +Revises: ef8a9b6c2d41 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "2b4c6e8a0d12" +down_revision = "ef8a9b6c2d41" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "scan_state", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=False), + ) + # Seed a best-effort historical completion time from previously scanned + # sources. The next completed scan will update the authoritative value. + op.execute( + sa.text( + "INSERT INTO scan_state (id, last_completed_at) " + "SELECT 1, MAX(last_seen_at) FROM source_documents " + "HAVING COUNT(*) > 0" + ) + ) + + +def downgrade(): + op.drop_table("scan_state") diff --git a/webmentions_ssg/models/scan_state.py b/webmentions_ssg/models/scan_state.py new file mode 100644 index 0000000..924c15d --- /dev/null +++ b/webmentions_ssg/models/scan_state.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Persist the last completed source scan for this single-site service.""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer +from sqlalchemy.orm import Mapped, mapped_column + +from ..extensions import Base + + +class ScanState(Base): + """The single row tracking the most recent successful source scan.""" + + __tablename__ = "scan_state" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + last_completed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 1f98702..9058b9b 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,6 +14,7 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey +from ..models.scan_state import ScanState from ..models.sources import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention @@ -304,6 +305,11 @@ def _scan_and_enqueue() -> None: db.session.commit() pending_count = enqueue_due_sends(scan_time) + # Mark a scan complete only after reconciliation and task enqueueing have + # succeeded. An inconclusive or interrupted scan keeps the previous value. + db.session.merge(ScanState(id=1, last_completed_at=datetime.now(UTC))) + db.session.commit() + current_app.logger.info( "Webmention source scan complete: %d sources scanned, " "%d missing sources, %d pending sends", diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index 4fffb38..115f193 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -43,6 +43,8 @@ SPDX-License-Identifier: BSD-3-Clause {% if current_user.is_authenticated %}