From e4a8194e6de25e8a48e2c61ce7f89a8159fd99d9 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Sun, 9 Aug 2026 14:15:29 +0200 Subject: Implement received Webmention handling Add the Flask application setup, database models and migrations, authentication, and configuration for development and testing. Implement asynchronous Webmention verification with Huey, including HTML and plain-text source validation, retries, status tracking, and size limits. Add status, login, and paginated received-Webmention views together with comprehensive tests for forms, views, and receiver tasks. --- .flaskenv | 3 + migrations/README | 1 + migrations/alembic.ini | 50 ++ migrations/env.py | 113 +++ migrations/script.py.mako | 24 + .../f63641044cc1_initial_database_schema.py | 50 ++ tests/conftest.py | 125 +++ tests/tasks/test_receiver.py | 886 +++++++++++++++++++++ tests/test_forms.py | 130 +++ tests/test_views.py | 183 +++++ webmentions_ssg/__init__.py | 141 ++++ webmentions_ssg/auth.py | 13 + webmentions_ssg/config.py | 50 ++ webmentions_ssg/forms/__init__.py | 42 + webmentions_ssg/forms/validators.py | 62 ++ webmentions_ssg/models.py | 89 +++ webmentions_ssg/tasks/__init__.py | 3 + webmentions_ssg/tasks/consumer.py | 8 + webmentions_ssg/tasks/extension.py | 159 ++++ webmentions_ssg/tasks/receiver.py | 225 ++++++ webmentions_ssg/templates/base.html | 44 + webmentions_ssg/templates/login.html | 8 + webmentions_ssg/templates/received.html | 69 ++ webmentions_ssg/templates/status.html | 64 ++ webmentions_ssg/views.py | 160 ++++ 25 files changed, 2702 insertions(+) create mode 100644 .flaskenv create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/f63641044cc1_initial_database_schema.py create mode 100644 tests/conftest.py create mode 100644 tests/tasks/test_receiver.py create mode 100644 tests/test_forms.py create mode 100644 tests/test_views.py create mode 100644 webmentions_ssg/auth.py create mode 100644 webmentions_ssg/config.py create mode 100644 webmentions_ssg/forms/__init__.py create mode 100644 webmentions_ssg/forms/validators.py create mode 100644 webmentions_ssg/models.py create mode 100644 webmentions_ssg/tasks/__init__.py create mode 100644 webmentions_ssg/tasks/consumer.py create mode 100644 webmentions_ssg/tasks/extension.py create mode 100644 webmentions_ssg/tasks/receiver.py create mode 100644 webmentions_ssg/templates/base.html create mode 100644 webmentions_ssg/templates/login.html create mode 100644 webmentions_ssg/templates/received.html create mode 100644 webmentions_ssg/templates/status.html create mode 100644 webmentions_ssg/views.py diff --git a/.flaskenv b/.flaskenv new file mode 100644 index 0000000..2737ea1 --- /dev/null +++ b/.flaskenv @@ -0,0 +1,3 @@ +WEBMENTIONS_SSG_DEV=1 +FLASK_APP=webmentions_ssg:create_app +FLASK_DEBUG=1 diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..0e04844 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..ec9d45c --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,50 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..4c97092 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,113 @@ +import logging +from logging.config import fileConfig + +from flask import current_app + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + + +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions['migrate'].db.get_engine() + except (TypeError, AttributeError): + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + **conf_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/f63641044cc1_initial_database_schema.py b/migrations/versions/f63641044cc1_initial_database_schema.py new file mode 100644 index 0000000..a3f0e40 --- /dev/null +++ b/migrations/versions/f63641044cc1_initial_database_schema.py @@ -0,0 +1,50 @@ +"""Initial database schema + +Revision ID: f63641044cc1 +Revises: +Create Date: 2026-08-09 11:18:10.051888 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f63641044cc1' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('received_webmentions', + sa.Column('uuid', sa.Uuid(), nullable=False), + sa.Column('source', sa.Text(), nullable=False), + sa.Column('target', sa.Text(), nullable=False), + sa.Column('status', sa.Text(), nullable=False), + sa.Column('failure_reason', sa.Text(), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('source', 'target', name='uq_webmention_source_target') + ) + op.create_table('users', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('username', sa.String(length=64), nullable=False), + sa.Column('password_hash', sa.String(length=256), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_users_username'), ['username'], unique=True) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_username')) + + op.drop_table('users') + op.drop_table('received_webmentions') + # ### end Alembic commands ### diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..94ba650 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,125 @@ +import uuid +from collections.abc import Callable, Iterator +from types import ModuleType + +import httpx +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import DATABASE as db +from webmentions_ssg import create_app +from webmentions_ssg.config import TestingConfig +from webmentions_ssg.models import ReceivedWebmention + +HTTPHandler = Callable[ + [httpx.Request], + httpx.Response, +] + + +@pytest.fixture +def app() -> Iterator[Flask]: + """ + Create a new Flask application and in-memory database + for every test. + """ + + application = create_app(TestingConfig) + + with application.app_context(): + db.create_all() + + yield application + + with application.app_context(): + db.session.remove() + db.drop_all() + db.engine.dispose() + + +@pytest.fixture +def client(app: Flask) -> FlaskClient: + return app.test_client() + + +@pytest.fixture +def receiver_module(app: Flask) -> ModuleType: + """ + Ensure the application and Huey extension are initialized + before retrieving the tasks module. + """ + + from webmentions_ssg.tasks import receiver + + return receiver + + +@pytest.fixture +def views_module(app: Flask) -> ModuleType: + from webmentions_ssg import views + + return views + + +@pytest.fixture +def make_webmention( + app: Flask, +) -> Callable[..., uuid.UUID]: + def create( + *, + source: str = "https://source.example/post", + target: str = ("https://dennisfink.me/blog/example/"), + status: str = "received", + failure_reason: str | None = None, + ) -> uuid.UUID: + identifier = uuid.uuid7() + + with app.app_context(): + webmention = ReceivedWebmention( + uuid=identifier, + source=source, + target=target, + status=status, + failure_reason=failure_reason, + ) + + db.session.add(webmention) + db.session.commit() + + return identifier + + return create + + +@pytest.fixture +def install_httpx_mock( + monkeypatch: pytest.MonkeyPatch, + receiver_module: ModuleType, +) -> Callable[[HTTPHandler], None]: + """ + Replace the HTTPX transport without replacing HTTPX itself. + """ + + real_client = httpx.Client + + def install(handler: HTTPHandler) -> None: + transport = httpx.MockTransport(handler) + + def create_client( + *args, + **kwargs, + ) -> httpx.Client: + return real_client( + *args, + transport=transport, + **kwargs, + ) + + monkeypatch.setattr( + receiver_module.httpx, + "Client", + create_client, + ) + + return install diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py new file mode 100644 index 0000000..4545121 --- /dev/null +++ b/tests/tasks/test_receiver.py @@ -0,0 +1,886 @@ +import logging +import uuid +from collections.abc import Callable +from types import ModuleType + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention + +SOURCE_URL = "https://source.example/article" +TARGET_URL = "https://dennisfink.me/blog/example/" + +ReceivedWebmentionFactory = Callable[..., uuid.UUID] +HTTPXMockInstaller = Callable[ + [Callable[[httpx.Request], httpx.Response]], + None, +] + + +def get_webmention_state( + app: Flask, + identifier: uuid.UUID, +) -> tuple[str, str | None]: + with app.app_context(): + webmention = db.session.get( + ReceivedWebmention, + identifier, + ) + + assert webmention is not None + + return ( + webmention.status, + webmention.failure_reason, + ) + + +@pytest.mark.parametrize( + ( + "body", + "source_url", + "target_url", + "expected", + ), + [ + pytest.param( + f'Reply', + SOURCE_URL, + TARGET_URL, + True, + id="a-href", + ), + pytest.param( + f'Target', + SOURCE_URL, + TARGET_URL, + True, + id="area-href", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="link-href", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="img-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="audio-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="video-src", + ), + pytest.param( + (f''), + SOURCE_URL, + TARGET_URL, + True, + id="audio-source-src", + ), + pytest.param( + (f''), + SOURCE_URL, + TARGET_URL, + True, + id="video-source-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="iframe-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="embed-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="script-src", + ), + pytest.param( + (f''), + SOURCE_URL, + TARGET_URL, + True, + id="track-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="image-input-src", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + True, + id="image-input-case-insensitive", + ), + pytest.param( + f'
Quotation
', + SOURCE_URL, + TARGET_URL, + True, + id="blockquote-cite", + ), + pytest.param( + f'Quotation', + SOURCE_URL, + TARGET_URL, + True, + id="q-cite", + ), + pytest.param( + f'Addition', + SOURCE_URL, + TARGET_URL, + True, + id="ins-cite", + ), + pytest.param( + f'Removal', + SOURCE_URL, + TARGET_URL, + True, + id="del-cite", + ), + pytest.param( + 'Reply', + "https://source.example/posts/article/", + "https://source.example/posts/target/", + True, + id="relative-href", + ), + pytest.param( + 'Reply', + SOURCE_URL, + TARGET_URL, + True, + id="base-url", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + False, + id="img-cite-invalid", + ), + pytest.param( + f'
Quote
', + SOURCE_URL, + TARGET_URL, + False, + id="blockquote-src-invalid", + ), + pytest.param( + f'Reply', + SOURCE_URL, + TARGET_URL, + False, + id="a-src-invalid", + ), + pytest.param( + f'
', + SOURCE_URL, + TARGET_URL, + False, + id="div-href-invalid", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + False, + id="link-src-invalid", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + False, + id="text-input-src-invalid", + ), + pytest.param( + (f''), + SOURCE_URL, + TARGET_URL, + False, + id="picture-source-src-invalid", + ), + pytest.param( + f'', + SOURCE_URL, + TARGET_URL, + False, + id="base-is-not-mention", + ), + pytest.param( + (f'Different page'), + SOURCE_URL, + TARGET_URL, + False, + id="longer-url", + ), + pytest.param( + (f''), + SOURCE_URL, + TARGET_URL, + False, + id="invalid-cite-does-not-override-valid-src", + ), + pytest.param( + f"

{TARGET_URL}

", + SOURCE_URL, + TARGET_URL, + False, + id="text-content", + ), + pytest.param( + 'Other site', + SOURCE_URL, + TARGET_URL, + False, + id="missing-target", + ), + ], +) +def test_html_mentions_target( + receiver_module: ModuleType, + body: str, + source_url: str, + target_url: str, + expected: bool, +) -> None: + assert ( + receiver_module.html_mentions_target( + body.encode(), + source_url, + target_url, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("body", "target_url", "expected"), + [ + (TARGET_URL, TARGET_URL, True), + (f"This post replies to {TARGET_URL}", TARGET_URL, True), + ( + f"https://example.com/first {TARGET_URL} https://example.com/last", + TARGET_URL, + True, + ), + (f"{TARGET_URL}more", TARGET_URL, False), + ("https://dennisfink.me/blog/other/", TARGET_URL, False), + ("/blog/example/", TARGET_URL, False), + ("There are no links here.", TARGET_URL, False), + ], +) +def test_text_mentions_target( + receiver_module: ModuleType, + body: str, + target_url: str, + expected: bool, +) -> None: + assert ( + receiver_module.text_mentions_target( + body, + target_url, + ) + is expected + ) + + +def test_fetch_source_returns_response_and_body( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, +) -> None: + captured_request: httpx.Request | None = None + + def handler( + request: httpx.Request, + ) -> httpx.Response: + nonlocal captured_request + captured_request = request + + return httpx.Response( + 200, + headers={ + "Content-Type": "text/html", + }, + content=b"

Document

", + ) + + install_httpx_mock(handler) + + with app.app_context(): + response, body = receiver_module.fetch_source(SOURCE_URL) + + assert response.status_code == 200 + assert body == b"

Document

" + + assert captured_request is not None + assert captured_request.url == SOURCE_URL + + accept = captured_request.headers["Accept"] + + assert "text/html" in accept + assert "application/xhtml+xml" in accept + assert "text/plain" in accept + + assert captured_request.headers["User-Agent"] == ( + f"{receiver_module.APP_NAME}/{receiver_module.VERSION} ReceivedWebmention" + ) + + +@pytest.mark.parametrize( + ("status_code", "exception_name"), + [ + (400, "VerificationError"), + (404, "VerificationError"), + (410, "SourceGoneError"), + (408, "TemporaryFetchError"), + (425, "TemporaryFetchError"), + (429, "TemporaryFetchError"), + (500, "TemporaryFetchError"), + (503, "TemporaryFetchError"), + ], +) +def test_fetch_source_maps_http_status_to_exception( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, + status_code: int, + exception_name: str, +) -> None: + install_httpx_mock(lambda request: httpx.Response(status_code)) + + exception_type = getattr( + receiver_module, + exception_name, + ) + + with ( + app.app_context(), + pytest.raises( + exception_type, + match=f"HTTP {status_code}", + ), + ): + receiver_module.fetch_source(SOURCE_URL) + + +def test_fetch_source_propagates_network_error( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, +) -> None: + def handler( + request: httpx.Request, + ) -> httpx.Response: + raise httpx.ConnectError( + "Connection refused", + request=request, + ) + + install_httpx_mock(handler) + + with ( + app.app_context(), + pytest.raises( + httpx.ConnectError, + match="Connection refused", + ), + ): + receiver_module.fetch_source(SOURCE_URL) + + +def test_fetch_source_follows_redirect( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, +) -> None: + requested_paths: list[str] = [] + + def handler( + request: httpx.Request, + ) -> httpx.Response: + requested_paths.append(request.url.path) + + match request.url.path: + case "/start": + return httpx.Response( + 302, + headers={ + "Location": "/final", + }, + ) + + case "/final": + return httpx.Response( + 200, + headers={ + "Content-Type": "text/html", + }, + content=b"Final document", + ) + + case _: + raise AssertionError(f"Unexpected URL: {request.url}") + + install_httpx_mock(handler) + + with app.app_context(): + response, body = receiver_module.fetch_source("https://source.example/start") + + assert requested_paths == [ + "/start", + "/final", + ] + assert response.url.path == "/final" + assert body == b"Final document" + + +def test_fetch_source_enforces_redirect_limit( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + app.config, + "WEBMENTIONS_SSG_MAX_REDIRECTS", + 1, + ) + + install_httpx_mock( + lambda request: httpx.Response( + 302, + headers={ + "Location": "/another", + }, + ) + ) + + with ( + app.app_context(), + pytest.raises(httpx.TooManyRedirects), + ): + receiver_module.fetch_source("https://source.example/start") + + +def test_fetch_source_rejects_declared_oversized_body( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + app.config, + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", + 10, + ) + + install_httpx_mock( + lambda request: httpx.Response( + 200, + headers={ + "Content-Type": "text/html", + "Content-Length": "11", + }, + content=b"x" * 11, + ) + ) + + with ( + app.app_context(), + pytest.raises( + receiver_module.VerificationError, + match="Source document is too large", + ), + ): + receiver_module.fetch_source(SOURCE_URL) + + +def test_fetch_source_rejects_streamed_oversized_body( + app: Flask, + receiver_module: ModuleType, + install_httpx_mock: HTTPXMockInstaller, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + app.config, + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", + 10, + ) + + class BodyStream(httpx.SyncByteStream): + def __iter__(self): + yield b"x" * 6 + yield b"x" * 6 + + install_httpx_mock( + lambda request: httpx.Response( + 200, + headers={ + "Content-Type": "text/html", + }, + stream=BodyStream(), + ) + ) + + with ( + app.app_context(), + pytest.raises( + receiver_module.VerificationError, + match="Source document is too large", + ), + ): + receiver_module.fetch_source(SOURCE_URL) + + +@pytest.mark.parametrize( + ("content_type", "body", "expected"), + [ + pytest.param( + "text/html; charset=utf-8", + f'Reply'.encode(), + True, + id="html", + ), + pytest.param( + "TEXT/HTML; CHARSET=UTF-8", + f'Reply'.encode(), + True, + id="case-insensitive-html", + ), + pytest.param( + "application/xhtml+xml", + b'Other', + False, + id="xhtml-without-target", + ), + pytest.param( + "text/plain; charset=utf-8", + f"Reply to {TARGET_URL}".encode(), + True, + id="plain-text-utf-8", + ), + pytest.param( + "text/plain; charset=iso-8859-1", + (f"Grüße. Reply to {TARGET_URL}").encode("iso-8859-1"), + True, + id="plain-text-declared-encoding", + ), + pytest.param( + "text/plain; charset=utf-8", + b"\xff Reply to " + TARGET_URL.encode(), + True, + id="plain-text-invalid-byte", + ), + pytest.param( + "text/plain", + b"No target here.", + False, + id="plain-text-without-target", + ), + ], +) +def test_source_mentions_target_by_media_type( + receiver_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + content_type: str, + body: bytes, + expected: bool, +) -> None: + response = httpx.Response( + 200, + headers={ + "Content-Type": content_type, + }, + content=body, + request=httpx.Request( + "GET", + SOURCE_URL, + ), + ) + + monkeypatch.setattr( + receiver_module, + "fetch_source", + lambda source_url: ( + response, + body, + ), + ) + + assert ( + receiver_module.source_mentions_target( + SOURCE_URL, + TARGET_URL, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("content_type", "expected_media_type"), + [ + ("application/json", "application/json"), + ("application/pdf", "application/pdf"), + ("", "missing"), + ], +) +def test_source_mentions_target_rejects_unsupported_media_type( + receiver_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + content_type: str, + expected_media_type: str, +) -> None: + headers = {} + + if content_type: + headers["Content-Type"] = content_type + + response = httpx.Response( + 200, + headers=headers, + content=b"Document", + request=httpx.Request( + "GET", + SOURCE_URL, + ), + ) + + monkeypatch.setattr( + receiver_module, + "fetch_source", + lambda source_url: ( + response, + b"Document", + ), + ) + + with pytest.raises( + receiver_module.VerificationError, + match=(f"Unsupported source content type: {expected_media_type}"), + ): + receiver_module.source_mentions_target( + SOURCE_URL, + TARGET_URL, + ) + + +def test_verify_webmention_marks_row_verifying_before_check( + app: Flask, + receiver_module: ModuleType, + make_webmention: ReceivedWebmentionFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identifier = make_webmention( + status="failed", + failure_reason="Earlier failure", + ) + + def verify_source( + source_url: str, + target_url: str, + ) -> bool: + webmention = db.session.get( + ReceivedWebmention, + identifier, + ) + + assert webmention is not None + assert webmention.status == "verifying" + assert webmention.failure_reason is None + assert source_url == webmention.source + assert target_url == webmention.target + + return True + + monkeypatch.setattr( + receiver_module, + "source_mentions_target", + verify_source, + ) + + receiver_module.verify_webmention.call_local(identifier) + + assert get_webmention_state( + app, + identifier, + ) == ( + "verified", + None, + ) + + +@pytest.mark.parametrize( + ("outcome", "expected_status", "expected_reason"), + [ + ("verified", "verified", None), + ("missing", "deleted", "Source does not mention target"), + ("gone", "deleted", "Source returned HTTP 410"), + ("permanent-failure", "failed", "Source returned HTTP 404"), + ], +) +def test_verify_webmention_persists_final_state( + app: Flask, + receiver_module: ModuleType, + make_webmention: ReceivedWebmentionFactory, + monkeypatch: pytest.MonkeyPatch, + outcome: str, + expected_status: str, + expected_reason: str | None, +) -> None: + identifier = make_webmention( + status="received", + ) + + def verify_source( + source_url: str, + target_url: str, + ) -> bool: + match outcome: + case "verified": + return True + + case "missing": + return False + + case "gone": + raise receiver_module.SourceGoneError("Source returned HTTP 410") + + case "permanent-failure": + raise receiver_module.VerificationError("Source returned HTTP 404") + + case _: + raise AssertionError(f"Unexpected outcome: {outcome}") + + monkeypatch.setattr( + receiver_module, + "source_mentions_target", + verify_source, + ) + + receiver_module.verify_webmention.call_local(identifier) + + assert get_webmention_state( + app, + identifier, + ) == ( + expected_status, + expected_reason, + ) + + +@pytest.mark.parametrize( + "failure", + [ + "temporary-http", + "network", + ], +) +def test_verify_webmention_persists_retryable_failure_and_reraises( + app: Flask, + receiver_module: ModuleType, + make_webmention: ReceivedWebmentionFactory, + monkeypatch: pytest.MonkeyPatch, + failure: str, +) -> None: + identifier = make_webmention() + + def verify_source( + source_url: str, + target_url: str, + ) -> bool: + match failure: + case "temporary-http": + raise receiver_module.TemporaryFetchError("Source returned HTTP 503") + + case "network": + raise httpx.ConnectError( + "Connection refused", + request=httpx.Request( + "GET", + source_url, + ), + ) + + case _: + raise AssertionError(f"Unexpected failure: {failure}") + + monkeypatch.setattr( + receiver_module, + "source_mentions_target", + verify_source, + ) + + match failure: + case "temporary-http": + expected_exception = receiver_module.TemporaryFetchError + expected_reason = "Source returned HTTP 503" + + case "network": + expected_exception = httpx.ConnectError + expected_reason = "Connection refused" + + case _: + raise AssertionError(f"Unexpected failure: {failure}") + + with pytest.raises( + expected_exception, + match=expected_reason, + ): + receiver_module.verify_webmention.call_local(identifier) + + assert get_webmention_state( + app, + identifier, + ) == ( + "failed", + expected_reason, + ) + + +def test_verify_webmention_ignores_unknown_identifier( + receiver_module: ModuleType, + caplog: pytest.LogCaptureFixture, +) -> None: + identifier = uuid.uuid7() + + with caplog.at_level(logging.WARNING): + receiver_module.verify_webmention.call_local(identifier) + + assert f"Cannot verify unknown ReceivedWebmention {identifier}" in caplog.text diff --git a/tests/test_forms.py b/tests/test_forms.py new file mode 100644 index 0000000..9be6ca8 --- /dev/null +++ b/tests/test_forms.py @@ -0,0 +1,130 @@ +import pytest +from flask import Flask +from werkzeug.datastructures import MultiDict + +from webmentions_ssg.forms import EndpointForm + +VALID_SOURCE = "https://source.example/post" +VALID_TARGET = "https://dennisfink.me/blog/example/" + + +@pytest.mark.parametrize( + ( + "form_data", + "invalid_field", + "expected_error", + ), + [ + pytest.param( + { + "target": VALID_TARGET, + }, + "source", + "This field is required.", + id="source-required", + ), + pytest.param( + { + "source": "not a URL", + "target": VALID_TARGET, + }, + "source", + "Invalid URL.", + id="source-url", + ), + pytest.param( + { + "source": "ftp://source.example/post", + "target": VALID_TARGET, + }, + "source", + "source must begin with http or https", + id="source-scheme", + ), + pytest.param( + { + "source": VALID_TARGET, + "target": VALID_TARGET, + }, + "source", + None, + id="source-not-equal-to-target", + ), + pytest.param( + { + "source": VALID_SOURCE, + }, + "target", + "This field is required.", + id="target-required", + ), + pytest.param( + { + "source": VALID_SOURCE, + "target": "not a URL", + }, + "target", + "Invalid URL.", + id="target-url", + ), + pytest.param( + { + "source": VALID_SOURCE, + "target": "ftp://dennisfink.me/blog/example/", + }, + "target", + "target must begin with http or https", + id="target-scheme", + ), + pytest.param( + { + "source": VALID_SOURCE, + "target": "https://example.com/post", + }, + "target", + None, + id="target-allowed-hostname", + ), + ], +) +def test_endpoint_form_rejects_invalid_data( + app: Flask, + form_data: dict[str, str], + invalid_field: str, + expected_error: str | None, +) -> None: + with app.test_request_context( + "/endpoint", + method="POST", + ): + form = EndpointForm( + formdata=MultiDict(form_data), + meta={"csrf": False}, + ) + + assert not form.validate() + assert invalid_field in form.errors + + if expected_error is not None: + assert expected_error in form.errors[invalid_field] + + +def test_endpoint_form_accepts_valid_data( + app: Flask, +) -> None: + with app.test_request_context( + "/endpoint", + method="POST", + ): + form = EndpointForm( + formdata=MultiDict( + { + "source": VALID_SOURCE, + "target": VALID_TARGET, + } + ), + meta={"csrf": False}, + ) + + assert form.validate() + assert form.errors == {} diff --git a/tests/test_views.py b/tests/test_views.py new file mode 100644 index 0000000..d7fd798 --- /dev/null +++ b/tests/test_views.py @@ -0,0 +1,183 @@ +import uuid +from types import ModuleType + +import pytest +import sqlalchemy as sa +from flask import Flask +from flask.testing import FlaskClient + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import ReceivedWebmention + + +def test_endpoint_only_accepts_post( + client: FlaskClient, +) -> None: + response = client.get("/endpoint") + + assert response.status_code == 405 + + +def test_endpoint_returns_form_errors( + client: FlaskClient, +) -> None: + response = client.post( + "/endpoint", + data={ + "source": "https://source.example/post", + "target": "https://example.com/post", + }, + ) + + assert response.status_code == 400 + + errors = response.get_json() + + assert errors is not None + assert "target" in errors + + +def test_endpoint_creates_webmention( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + queued: list[uuid.UUID] = [] + + monkeypatch.setattr( + views_module, + "verify_webmention", + queued.append, + ) + + source = "https://source.example/post" + target = "https://dennisfink.me/blog/example/" + + response = client.post( + "/endpoint", + data={ + "source": source, + "target": target, + }, + ) + + assert response.status_code == 201 + + with app.app_context(): + webmention = db.session.scalar(sa.select(ReceivedWebmention)) + + assert webmention is not None + assert webmention.source == source + assert webmention.target == target + assert webmention.status == "received" + assert webmention.failure_reason is None + + identifier = webmention.uuid + + assert response.headers["Location"].endswith(str(identifier)) + + assert queued == [identifier] + + +def test_endpoint_is_idempotent( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + queued: list[uuid.UUID] = [] + + monkeypatch.setattr( + views_module, + "verify_webmention", + queued.append, + ) + + data = { + "source": ("https://source.example/post"), + "target": ("https://dennisfink.me/blog/example/"), + } + + first_response = client.post( + "/endpoint", + data=data, + ) + second_response = client.post( + "/endpoint", + data=data, + ) + + assert first_response.status_code == 201 + assert second_response.status_code == 201 + + assert first_response.headers["Location"] == second_response.headers["Location"] + + with app.app_context(): + webmentions = db.session.scalars(sa.select(ReceivedWebmention)).all() + + assert len(webmentions) == 1 + + webmention = webmentions[0] + + assert webmention.source == data["source"] + assert webmention.target == data["target"] + assert webmention.status == "received" + assert webmention.failure_reason is None + + identifier = webmention.uuid + + assert queued == [ + identifier, + identifier, + ] + + +def test_resending_resets_failure_state( + app: Flask, + client: FlaskClient, + views_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + views_module, + "verify_webmention", + lambda identifier: None, + ) + + source = "https://source.example/post" + target = "https://dennisfink.me/blog/example/" + + with app.app_context(): + existing = ReceivedWebmention( + uuid=uuid.uuid7(), + source=source, + target=target, + status="failed", + failure_reason="Previous failure", + ) + + db.session.add(existing) + db.session.commit() + + identifier = existing.uuid + + response = client.post( + "/endpoint", + data={ + "source": source, + "target": target, + }, + ) + + assert response.status_code == 201 + + with app.app_context(): + webmention = db.session.get( + ReceivedWebmention, + identifier, + ) + + assert webmention is not None + assert webmention.status == "received" + assert webmention.failure_reason is None diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index 7f93695..f66e478 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -2,4 +2,145 @@ # # SPDX-License-Identifier: BSD-3-Clause +import logging +import logging.handlers +import os +from pathlib import Path + +import click +import coloredlogs +import sqlalchemy as sa +from flask import Flask +from flask_bootstrap import Bootstrap5 +from flask_migrate import Migrate, upgrade +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.orm import DeclarativeBase + +from webmentions_ssg.config import DefaultConfig, DevelopmentConfig, ProductionConfig + +from .tasks import Huey + +APP_NAME = "webmentions-ssg" VERSION = "0.0.1" + + +class Base(DeclarativeBase): + pass + + +BOOTSTRAP = Bootstrap5() +DATABASE = SQLAlchemy(model_class=Base) +MIGRATE = Migrate() +HUEY = Huey() + + +def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: + """Creates and configures the Flask application.""" + + development = os.environ.get( + "WEBMENTIONS_SSG_DEV", + "", + ).lower() in { + "1", + "true", + "yes", + "on", + } + + if config_class is None: + config_class = DevelopmentConfig if development else ProductionConfig + + app = Flask( + __name__, + instance_path=str(Path("testing").absolute()) if development else None, + ) + app.config.from_object(config_class) + + if not app.testing: + app.config.from_pyfile(app.config["CONFIG_FILE"], silent=True) + app.config.from_prefixed_env(prefix="FLASK") + app.config.from_prefixed_env(prefix="WEBMENTIONS_SSG") + + default_logging_formatter = logging.Formatter(app.config["LOG_FORMAT"]) + + log_file_handler = logging.handlers.RotatingFileHandler( + app.config["LOG_FILE"], + maxBytes=app.config["LOGFILE_MAX_BYTES"], + backupCount=app.config["LOGFILE_BACKUP_COUNT"], + ) + log_file_handler.setLevel(app.config["LOG_LEVEL"]) + log_file_handler.setFormatter(default_logging_formatter) + + logging.basicConfig(level=logging.DEBUG, handlers=[logging.NullHandler()]) + + app.logger.handlers.clear() + app.logger.setLevel(app.config["LOG_LEVEL"]) + app.logger.addHandler(log_file_handler) + + werkzeug_logger = logging.getLogger("werkzeug") + werkzeug_logger.setLevel(app.config["LOG_LEVEL"]) + werkzeug_logger.addHandler(log_file_handler) + + coloredlogs.install( + level=app.config["LOG_LEVEL"], + logger=app.logger, + fmt=app.config["LOG_FORMAT"], + ) + coloredlogs.install( + level=app.config["LOG_LEVEL"], + logger=werkzeug_logger, + fmt=app.config["LOG_FORMAT"], + ) + + DATABASE.init_app(app) + MIGRATE.init_app(app, db=DATABASE) + + from .auth import AUTH + + AUTH.init_app(app) + BOOTSTRAP.init_app(app) + HUEY.init_app(app) + + @app.context_processor + def inject_version(): + return dict(VERSION=VERSION) + + from .views import root_page + + app.register_blueprint(root_page) + + @app.cli.command("create-db") + def create_db() -> None: + """Create or update the database to the latest migration.""" + upgrade() + + @app.cli.command("create-user") + @click.argument("username") + @click.password_option( + confirmation_prompt=True, + ) + def create_user( + username: str, + password: str, + ) -> None: + """Create a user.""" + + from .models import User + + if ( + DATABASE.session.scalar(sa.select(User).where(User.username == username)) + is not None + ): + raise click.ClickException(f"User {username!r} already exists.") + + user = User( + username=username, + ) + user.set_password(password) + + DATABASE.session.add(user) + DATABASE.session.commit() + + click.echo(f"Created user {username!r}.") + + return app diff --git a/webmentions_ssg/auth.py b/webmentions_ssg/auth.py new file mode 100644 index 0000000..f8fd28c --- /dev/null +++ b/webmentions_ssg/auth.py @@ -0,0 +1,13 @@ +from flask_login import LoginManager + +from . import DATABASE as db +from .models import User + +AUTH = LoginManager() + +AUTH.login_view = "root.login" + + +@AUTH.user_loader +def load_user(id): + return db.session.get(User, int(id)) diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py new file mode 100644 index 0000000..7c448e9 --- /dev/null +++ b/webmentions_ssg/config.py @@ -0,0 +1,50 @@ +import logging +import secrets + + +class DefaultConfig: + TESTING = False + DEBUG = False + SECRET_KEY = secrets.token_urlsafe() + BOOTSTRAP_SERVE_LOCAL = True + # BOOTSTRAP_BOOTSWATCH_THEME = "Materia" + LOG_FILE = "/var/log/webmentions-ssg.log" + LOG_FORMAT = "[%(asctime)s]:%(levelname)s:%(name)s %(message)s" + LOG_LEVEL = logging.INFO + LOGFILE_MAX_BYTES = 20_000_000 + LOGFILE_BACKUP_COUNT = 10 + SQLALCHEMY_DATABASE_URI = "sqlite:///webmentions-ssg.db" + + +class DevelopmentConfig(DefaultConfig): + DEBUG = True + LOG_FILE = "testing/webmentions-ssg.log" + LOG_LEVEL = logging.DEBUG + CONFIG_FILE = "testing/webmentions-ssg.cfg" + + +class ProductionConfig(DefaultConfig): + CONFIG_FILE = "/etc/webmentions-ssg.cfg" + + +class TestingConfig(DefaultConfig): + TESTING = True + + SECRET_KEY = "testing-secret-key" + WTF_CSRF_ENABLED = False + + LOG_FILE = "testing/webmentions-ssg-tests.log" + LOG_LEVEL = logging.DEBUG + + SQLALCHEMY_DATABASE_URI = "sqlite:///:memory:" + SQLALCHEMY_TRACK_MODIFICATIONS = False + + HUEY_URL = "memory://" + + WEBMENTIONS_SSG_ALLOWED_HOSTNAMES = { + "dennisfink.me", + } + + WEBMENTIONS_SSG_MAX_REDIRECTS = 20 + WEBMENTIONS_SSG_MAX_SOURCE_BYTES = 1_000_000 + WEBMENTIONS_SSG_REQUEST_TIMEOUT = 5.0 diff --git a/webmentions_ssg/forms/__init__.py b/webmentions_ssg/forms/__init__.py new file mode 100644 index 0000000..98962ad --- /dev/null +++ b/webmentions_ssg/forms/__init__.py @@ -0,0 +1,42 @@ +import re + +from flask_wtf import FlaskForm +from wtforms import PasswordField, StringField, SubmitField +from wtforms.validators import URL, InputRequired, Regexp + +from .validators import AllowedHostname, NotEqualTo + + +class LoginForm(FlaskForm): + username = StringField("Username", validators=[InputRequired()]) + password = PasswordField("Password", validators=[InputRequired()]) + submit = SubmitField("Sign In") + + +class EndpointForm(FlaskForm): + source = StringField( + "source", + validators=[ + InputRequired(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="source must begin with http or https", + ), + NotEqualTo("target"), + ], + ) + target = StringField( + "target", + validators=[ + InputRequired(), + URL(), + Regexp( + "^https?://.*", + flags=re.IGNORECASE, + message="target must begin with http or https", + ), + AllowedHostname(), + ], + ) diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py new file mode 100644 index 0000000..20d9ebf --- /dev/null +++ b/webmentions_ssg/forms/validators.py @@ -0,0 +1,62 @@ +from urllib.parse import urlsplit + +from flask import current_app +from wtforms import ValidationError + + +class NotEqualTo: + """ + Compares the values of two fields. + + :param fieldname: + The name of the other field to compare to. + :param message: + Error message to raise in case of a validation error. Can be + interpolated with `%(other_label)s` and `%(other_name)s` to provide a + more helpful error. + """ + + def __init__(self, fieldname, message=None): + self.fieldname = fieldname + self.message = message + + def __call__(self, form, field): + try: + other = form[self.fieldname] + except KeyError as exc: + raise ValidationError( + field.gettext("Invalid field name '%s'.") % self.fieldname + ) from exc + + if field.data != other.data: + return + + d = { + "other_label": hasattr(other, "label") + and other.label.text + or self.fieldname, + "other_name": self.fieldname, + } + message = self.message + if message is None: + message = field.gettext("Field must not be equal to %(other_name)s.") + + raise ValidationError(message % d) + + +class AllowedHostname: + def __init__(self, message=None): + self.message = message + + def __call__(self, form, field): + if ( + urlsplit(field.data).hostname + in current_app.config["WEBMENTIONS_SSG_ALLOWED_HOSTNAMES"] + ): + return + + message = self.message + if self.message is None: + message = field.gettext("Invalid input.") + + raise ValidationError(message) diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py new file mode 100644 index 0000000..8c378aa --- /dev/null +++ b/webmentions_ssg/models.py @@ -0,0 +1,89 @@ +import uuid +from datetime import datetime, timezone +from typing import Optional + +from flask_login import UserMixin +from sqlalchemy import DateTime, String, Text, UniqueConstraint, Uuid +from sqlalchemy.orm import Mapped, mapped_column +from werkzeug.security import check_password_hash, generate_password_hash + +from . import Base + + +class User(UserMixin, Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column( + String(64), + index=True, + unique=True, + ) + password_hash: Mapped[Optional[str]] = mapped_column(String(256)) + + def __repr__(self): + return f"" + + def set_password(self, password): + self.password_hash = generate_password_hash(password) + + def check_password(self, password): + return check_password_hash( + self.password_hash, + password, + ) + + +class ReceivedWebmention(Base): + __tablename__ = "received_webmentions" + + uuid: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + primary_key=True, + ) + + source: Mapped[str] = mapped_column( + Text(), + nullable=False, + ) + target: Mapped[str] = mapped_column( + Text(), + nullable=False, + ) + + status: Mapped[str] = mapped_column( + Text(), + nullable=False, + default="received", + ) + failure_reason: Mapped[str | None] = mapped_column( + Text(), + nullable=True, + default=None, + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + __table_args__ = ( + UniqueConstraint( + "source", + "target", + name="uq_webmention_source_target", + ), + ) + + @property + def verified(self) -> bool: + return self.status == "verified" + + @property + def created_at(self) -> datetime: + return datetime.fromtimestamp( + self.uuid.time / 1000, + tz=timezone.utc, + ) diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py new file mode 100644 index 0000000..5af3987 --- /dev/null +++ b/webmentions_ssg/tasks/__init__.py @@ -0,0 +1,3 @@ +from .extension import Huey + +__all__ = ["Huey"] diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py new file mode 100644 index 0000000..5b85d0e --- /dev/null +++ b/webmentions_ssg/tasks/consumer.py @@ -0,0 +1,8 @@ +from .. import HUEY, create_app + +app = create_app() + +# Import the tasks so they are registered with the initialized Huey instance. +from . import receiver # noqa: E402, F401 + +huey = HUEY.huey diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py new file mode 100644 index 0000000..ebdaa76 --- /dev/null +++ b/webmentions_ssg/tasks/extension.py @@ -0,0 +1,159 @@ +from functools import wraps +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit + +from flask import Flask + + +class Huey: + def __init__(self, app: Flask | None = None): + self.app: Flask | None = None + self._huey = None + + if app is not None: + self.init_app(app) + + def init_app(self, app: Flask): + config: dict[str, Any] = { + "name": app.import_name, + "results": True, + "store_none": False, + "utc": True, + "immediate": app.config.get("TESTING", False), + **app.config.get("HUEY", {}), + } + + url = app.config.get("HUEY_URL", config.pop("url", "memory://")) + huey_class, storage_kwargs = self.backend_from_url(url) + + self.app = app + self._huey = huey_class( + **config, + **storage_kwargs, + ) + + app.extensions["huey"] = self + + @property + def huey(self): + if self._huey is None: + raise RuntimeError( + "Huey has not been initialized. " + "Call huey.init_app(app) before importing tasks." + ) + return self._huey + + def task(self, *task_args: Any, **task_kwargs: Any): + def decorator(func: Callable): + @wraps(func) + def wrapper(*args: Any, **kwargs: Any): + if self.app is None: + raise RuntimeError("Flask app is not available.") + + with self.app.app_context(): + return func(*args, **kwargs) + + return self.huey.task(*task_args, **task_kwargs)(wrapper) + + return decorator + + def periodic_task(self, *task_args: Any, **task_kwargs: Any): + def decorator(func: Callable): + @wraps(func) + def wrapper(*args: Any, **kwargs: Any): + if self.app is None: + raise RuntimeError("Flask app is not available.") + + with self.app.app_context(): + return func(*args, **kwargs) + + return self.huey.periodic_task(*task_args, **task_kwargs)(wrapper) + + return decorator + + def __getattr__(self, name: str): + """ + Forward unknown attributes to the real Huey instance. + + This lets you still use things like: + huey.enqueue(...) + huey.scheduled() + huey.pending() + """ + return getattr(self.huey, name) + + @staticmethod + def backend_from_url(url: str) -> tuple[Any, dict[str, str]]: + parsed = urlsplit(url) + scheme = parsed.scheme.lower() + + if scheme.startswith("redis") or scheme.startswith("rediss"): + fixed_url = urlunsplit(parsed._replace(scheme=scheme.split("+", 1)[0])) + + if scheme.endswith("priority+expire"): + from huey import PriorityRedisExpireHuey + + return PriorityRedisExpireHuey, {"url": fixed_url} + elif scheme.endswith("priority"): + from huey import PriorityRedisHuey + + return PriorityRedisHuey, {"url": fixed_url} + elif scheme.endswith("expire"): + from huey import RedisExpireHuey + + return RedisExpireHuey, {"url": fixed_url} + else: + from huey import RedisHuey + + return RedisHuey, {"url": url} + + elif scheme == "sqlite": + from huey import SqliteHuey + + prefix = "sqlite:///" + + if not url.startswith(prefix): + raise RuntimeError( + "SQLite Huey URLs must look like sqlite:///var/huey.db" + ) + + filename = url.removeprefix(prefix) + + if not filename: + raise RuntimeError("SQLite Huey URL must include a database path.") + + return SqliteHuey, {"filename": filename} + + elif scheme == "file": + from huey import FileHuey + + prefix = "file:///" + + if not url.startswith(prefix): + raise RuntimeError( + "File Huey URLs must look like file:///var/huey-queue" + ) + + path = url.removeprefix(prefix) + + if not path: + raise RuntimeError("File Huey URL must include a directory path.") + + return FileHuey, {"path": path} + + elif scheme in {"postgres", "postgresql"}: + from huey import PostgresHuey + + return PostgresHuey, {"dsn": url} + + elif scheme == "memory": + from huey import MemoryHuey + + return MemoryHuey, {} + + elif scheme == "blackhole": + from huey import BlackHoleHuey + + return BlackHoleHuey, {} + + raise RuntimeError(f"Unsupported HUEY_URL scheme: {scheme!r}") diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py new file mode 100644 index 0000000..ea48299 --- /dev/null +++ b/webmentions_ssg/tasks/receiver.py @@ -0,0 +1,225 @@ +import uuid +from urllib.parse import urljoin + +import httpx +import rfc3987 +from bs4 import BeautifulSoup +from flask import current_app + +from .. import APP_NAME, VERSION +from .. import DATABASE as db +from .. import HUEY as huey +from ..models import ReceivedWebmention + + +class VerificationError(Exception): + """The source permanently failed ReceivedWebmention verification.""" + + +class SourceGoneError(VerificationError): + """The source explicitly reports that it has been removed.""" + + +class TemporaryFetchError(Exception): + """Fetching the source may succeed when retried later.""" + + +IRI_PATTERN = rfc3987.get_compiled_pattern("IRI") + + +HTML_URL_ATTRIBUTES = { + "href": {"a", "area", "link"}, + "src": { + "audio", + "embed", + "iframe", + "img", + 'input[type="image" i]', + "script", + "audio source", + "video source", + "track", + "video", + }, + "cite": { + "blockquote", + "del", + "ins", + "q", + }, +} + + +def html_mentions_target( + body: bytes, + source_url: str, + target_url: str, +) -> bool: + """Check valid HTML URL attributes for the exact target URL.""" + + document = BeautifulSoup(body, "html.parser") + base_url = source_url + + if (base_element := document.select_one("base[href]")) is not None and isinstance( + base_href := base_element.get("href"), + str, + ): + base_url = urljoin( + source_url, + base_href.strip(), + ) + + for attribute, selectors in HTML_URL_ATTRIBUTES.items(): + selector = ", ".join( + [ + "{selector}[{attribute}]".format( + selector=selector_string, attribute=attribute + ) + for selector_string in selectors + ] + ) + + for element in document.select(selector): + if not isinstance( + reference := element.get(attribute), + str, + ): + continue + + if ( + urljoin( + base_url, + reference.strip(), + ) + == target_url + ): + return True + + return False + + +def text_mentions_target(body: str, target_url: str) -> bool: + """Check whether plain text contains the exact target IRI.""" + return any(match.group() == target_url for match in IRI_PATTERN.finditer(body)) + + +def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: + """Fetch a source with limits on redirects, time, and response size.""" + + with httpx.Client( + headers={ + "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention", + }, + timeout=httpx.Timeout( + current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) + ), + follow_redirects=True, + max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), + trust_env=False, + ) as client: + with client.stream("GET", source_url) as response: + match response.status_code: + case 200: + pass + case 410: + raise SourceGoneError("Source returned HTTP 410") + case status: + if status in {408, 425, 429} or 500 <= status <= 599: + raise TemporaryFetchError(f"Source returned HTTP {status}") + else: + raise VerificationError(f"Source returned HTTP {status}") + + max_source_bytes = current_app.config.get( + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", + 1_000_000, + ) + + if (content_length := response.headers.get("Content-Length")) is not None: + try: + if int(content_length) > max_source_bytes: + raise VerificationError("Source document is too large") + except ValueError: + pass + + body = bytearray() + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) + + if len(body) > max_source_bytes: + raise VerificationError("Source document is too large") + + return response, bytes(body) + + +def source_mentions_target(source_url: str, target_url: str) -> bool: + """Fetch the source and verify it according to its media type.""" + + response, body = fetch_source(source_url) + + media_type = ( + response.headers.get("Content-Type", "").partition(";")[0].strip().lower() + ) + + match media_type: + case "text/html" | "application/xhtml+xml": + return html_mentions_target(body, str(response.url), target_url) + case "text/plain": + try: + decoded_body = body.decode( + response.encoding or "utf-8", + errors="replace", + ) + except LookupError: + decoded_body = body.decode( + "utf-8", + errors="replace", + ) + return text_mentions_target(decoded_body, target_url) + case _: + raise VerificationError( + f"Unsupported source content type: {media_type or 'missing'}" + ) + + +@huey.task(retries=2, retry_delay=50) +def verify_webmention(webmention_uuid: uuid.UUID) -> None: + """Verify a ReceivedWebmention and store the result.""" + + webmention = db.session.get(ReceivedWebmention, webmention_uuid) + + if webmention is None: + current_app.logger.warning( + "Cannot verify unknown ReceivedWebmention %s", + webmention_uuid, + ) + return + + webmention.status = "verifying" + webmention.failure_reason = None + db.session.commit() + + try: + mentions_target = source_mentions_target(webmention.source, webmention.target) + except SourceGoneError as exc: + webmention.status = "deleted" + webmention.failure_reason = str(exc) + except VerificationError as exc: + webmention.status = "failed" + webmention.failure_reason = str(exc) + except (TemporaryFetchError, httpx.RequestError) as exc: + webmention.status = "failed" + webmention.failure_reason = str(exc) or "Source could not be fetched" + db.session.commit() + + # Huey retries the task because the exception escapes. + raise + else: + if mentions_target: + webmention.status = "verified" + webmention.failure_reason = None + else: + webmention.status = "deleted" + webmention.failure_reason = "Source does not mention target" + + db.session.commit() diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html new file mode 100644 index 0000000..237a437 --- /dev/null +++ b/webmentions_ssg/templates/base.html @@ -0,0 +1,44 @@ +{% from "bootstrap5/utils.html" import render_messages %} + + + + {% block head %} + + + + {% block styles %} + + {{ bootstrap.load_css() }} + {% endblock %} + {{ title }} + {% endblock %} + + +
+ +
+
{{ render_messages(container=True) }}
+
+ {% block content %} + {% endblock %} +
+ {% block scripts %} + + {{ bootstrap.load_js() }} + {% endblock %} + + diff --git a/webmentions_ssg/templates/login.html b/webmentions_ssg/templates/login.html new file mode 100644 index 0000000..e7d858c --- /dev/null +++ b/webmentions_ssg/templates/login.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} +{% set title = "Login" %} +{% from "bootstrap5/form.html" import render_form %} +{% block content %} +
+
{{ render_form(form) }}
+
+{% endblock %} diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html new file mode 100644 index 0000000..ea74557 --- /dev/null +++ b/webmentions_ssg/templates/received.html @@ -0,0 +1,69 @@ +{% extends "base.html" %} +{% from "bootstrap5/pagination.html" import render_pagination %} +{% block content %} +
+

Received Webmentions

+ {{ webmentions.total }} total +
+ {% if webmentions.items %} +
+ + + + + + + + + + + + + {% for webmention in webmentions %} + {% if webmention.status == "verified" %} + {% set row_class = "table-success" %} + {% set display_status = "Verified" %} + {% elif webmention.status in ("received", "verifying") %} + {% set row_class = "table-warning" %} + {% set display_status = "Pending" %} + {% else %} + {% set row_class = "table-danger" %} + {% set display_status = "Declined" %} + {% endif %} + + + + + + + + + {% endfor %} + +
StatusSourceTargetReceivedUpdatedReason
+ {{ display_status }} + + {{ webmention.source }} + + {{ webmention.target }} + + + + + {{ webmention.failure_reason or "—" }}
+
+ {% if webmentions.pages > 1 %} +
+ {{ render_pagination(webmentions, + align="center", + size="sm") }} +
+ {% endif %} + {% else %} +

No Webmentions have been received yet.

+ {% endif %} +{% endblock %} diff --git a/webmentions_ssg/templates/status.html b/webmentions_ssg/templates/status.html new file mode 100644 index 0000000..451f26f --- /dev/null +++ b/webmentions_ssg/templates/status.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} +{% from "bootstrap5/utils.html" import render_icon %} +{% block content %} +
+
+ {% if webmention.status == "verified" %} +
{{ render_icon("check-circle-fill", size="5rem", title="Verified") }}
+

Webmention verified

+ {% elif webmention.status in ("received", "verifying") %} +
{{ render_icon("hourglass-split", size="5rem", title="Verification pending") }}
+

Webmention verification pending

+ {% elif webmention.status == "deleted" %} +
{{ render_icon("x-circle-fill", size="5rem", title="Not verified") }}
+

Webmention not verified

+ {% elif webmention.status == "failed" %} +
{{ render_icon("x-circle-fill", size="5rem", title="Verification failed") }}
+

Webmention verification failed

+ {% else %} +
{{ render_icon("question-circle-fill", size="5rem", title="Unknown status") }}
+

Unknown Webmention status

+ {% endif %} +
+
+
+
+
Status
+
+ {{ webmention.status }} +
+
Source
+
+ {{ webmention.source }} +
+
Target
+
+ {{ webmention.target }} +
+
Received
+
+ +
+
Last updated
+
+ +
+
ID
+
+ {{ webmention.uuid }} +
+ {% if webmention.failure_reason %} +
Reason
+
+ {{ webmention.failure_reason }} +
+ {% endif %} +
+
+
+
+{% endblock %} diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py new file mode 100644 index 0000000..5408fa4 --- /dev/null +++ b/webmentions_ssg/views.py @@ -0,0 +1,160 @@ +import uuid +from typing import cast +from urllib.parse import urlsplit + +import sqlalchemy as sa +import sqlalchemy.exc as sa_exc +from flask import ( + Blueprint, + Response, + abort, + flash, + redirect, + render_template, + request, + url_for, +) +from flask.typing import ResponseReturnValue +from flask_login import current_user, login_required, login_user, logout_user + +from . import DATABASE as db +from . import forms +from .models import ReceivedWebmention, User +from .tasks.receiver import verify_webmention + +root_page = Blueprint("root", __name__) + + +@root_page.route("/") +def index() -> ResponseReturnValue: + return redirect(url_for("root.received")) + + +@root_page.route("/login", methods=["GET", "POST"]) +def login() -> ResponseReturnValue: + if current_user.is_authenticated: + return redirect(url_for("root.index")) + + form = forms.LoginForm() + + if form.validate_on_submit(): + user = db.session.scalar( + sa.select(User).where(User.username == form.username.data) + ) + + if user is None or not user.check_password(form.password.data): + flash("Invalid username or password", "danger") + return redirect(url_for("root.login")) + + login_user(user) + + next_page = request.args.get("next") + if not next_page or urlsplit(next_page).netloc != "": + next_page = url_for("root.index") + return redirect(next_page) + + return render_template("login.html", form=form) + + +@root_page.route("/logout") +def logout() -> ResponseReturnValue: + logout_user() + return redirect(url_for("root.index")) + + +@root_page.route("/received") +@login_required +def received() -> ResponseReturnValue: + webmentions = db.paginate( + sa.select(ReceivedWebmention).order_by(ReceivedWebmention.uuid.desc()), + per_page=25, + ) + + return render_template( + "received.html", + title="Received Webmentions", + webmentions=webmentions, + ) + + +@root_page.route("/endpoint", methods=["POST"]) +def endpoint() -> ResponseReturnValue: + form = forms.EndpointForm(meta={"csrf": False}) + + if not form.validate_on_submit(): + return form.errors, 400 + + source = cast(str, form.source.data) + target = cast(str, form.target.data) + + webmention = db.session.execute( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.source == source, + ReceivedWebmention.target == target, + ) + ).scalar_one_or_none() + + if webmention is None: + identifier = uuid.uuid7() + + webmention = ReceivedWebmention( + uuid=identifier, + source=source, + target=target, + ) + + db.session.add(webmention) + + try: + db.session.commit() + except sa_exc.IntegrityError: + db.session.rollback() + + # Another request may have inserted the same source/target pair + # after our SELECT but before our COMMIT. + webmention = db.session.execute( + sa.select(ReceivedWebmention).where( + ReceivedWebmention.source == source, + ReceivedWebmention.target == target, + ) + ).scalar_one() + + identifier = webmention.uuid + webmention.status = "received" + webmention.failure_reason = None + + db.session.commit() + + else: + identifier = webmention.uuid + + # Re-sent webmention: re-verify the existing row. + webmention.status = "received" + webmention.failure_reason = None + + db.session.commit() + + verify_webmention(webmention.uuid) + + status_url = url_for( + "root.status", + identifier=str(identifier), + _external=True, + ) + + return Response( + status=201, + headers={"Location": status_url}, + ) + + +@root_page.route("/status/") +def status(identifier: uuid.UUID) -> ResponseReturnValue: + webmention = db.session.get(ReceivedWebmention, identifier) + + if webmention is None: + return abort(404) + + return render_template( + "status.html", title="Webmention status", webmention=webmention + ) -- cgit v1.3.1