diff options
Diffstat (limited to '')
| -rw-r--r-- | migrations/versions/681a2a9a1bc0_.py | 66 | ||||
| -rw-r--r-- | pyproject.toml | 9 | ||||
| -rw-r--r-- | tests/tasks/test_receiver.py | 396 | ||||
| -rw-r--r-- | tests/tasks/test_scanner.py | 1558 | ||||
| -rw-r--r-- | tests/tasks/test_sender.py | 788 | ||||
| -rw-r--r-- | tests/test_forms.py | 80 | ||||
| -rw-r--r-- | tests/test_url_security.py | 54 | ||||
| -rw-r--r-- | tests/test_views.py | 202 | ||||
| -rw-r--r-- | uv.lock | 183 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/forms/validators.py | 22 | ||||
| -rw-r--r-- | webmentions_ssg/models.py | 182 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 19 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 5 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 65 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 397 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 315 | ||||
| -rw-r--r-- | webmentions_ssg/templates/base.html | 5 | ||||
| -rw-r--r-- | webmentions_ssg/templates/received.html | 2 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent.html | 61 | ||||
| -rw-r--r-- | webmentions_ssg/templates/sent_source.html | 155 | ||||
| -rw-r--r-- | webmentions_ssg/url_security.py | 28 | ||||
| -rw-r--r-- | webmentions_ssg/views.py | 85 |
23 files changed, 4089 insertions, 595 deletions
diff --git a/migrations/versions/681a2a9a1bc0_.py b/migrations/versions/681a2a9a1bc0_.py new file mode 100644 index 0000000..2420b79 --- /dev/null +++ b/migrations/versions/681a2a9a1bc0_.py @@ -0,0 +1,66 @@ +"""empty message + +Revision ID: 681a2a9a1bc0 +Revises: f63641044cc1 +Create Date: 2026-08-14 13:40:55.149604 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '681a2a9a1bc0' +down_revision = 'f63641044cc1' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('sources', + sa.Column('uuid', sa.Uuid(), nullable=False), + sa.Column('path', sa.Text(), nullable=False), + sa.Column('url', sa.Text(), nullable=False), + sa.Column('content_hash', sa.String(length=32), nullable=False), + sa.Column('revision', sa.Integer(), nullable=False), + sa.Column('last_seen_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('revised_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('path'), + sa.UniqueConstraint('url') + ) + op.create_table('sent_webmentions', + sa.Column('uuid', sa.Uuid(), nullable=False), + sa.Column('source_id', sa.Uuid(), nullable=False), + sa.Column('target', sa.Text(), nullable=False), + sa.Column('active', sa.Boolean(), nullable=False), + sa.Column('desired_revision', sa.Integer(), nullable=False), + sa.Column('processed_revision', sa.Integer(), nullable=True), + sa.Column('sent_revision', sa.Integer(), nullable=True), + sa.Column('status', sa.Enum('sent', 'unsupported', 'failed', name='sent_webmention_status', native_enum=False), nullable=True), + sa.Column('failure_reason', sa.Text(), nullable=True), + sa.Column('endpoint', sa.Text(), nullable=True), + sa.Column('response_status', sa.Integer(), nullable=True), + sa.Column('status_url', sa.Text(), nullable=True), + sa.Column('last_attempted_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_sent_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['source_id'], ['sources.uuid'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('source_id', 'target', name='uq_sent_webmention_source_target') + ) + with op.batch_alter_table('sent_webmentions', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_sent_webmentions_source_id'), ['source_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('sent_webmentions', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_sent_webmentions_source_id')) + + op.drop_table('sent_webmentions') + op.drop_table('sources') + # ### end Alembic commands ### diff --git a/pyproject.toml b/pyproject.toml index 9af850a..81dcd1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,9 @@ dependencies = [ "flask-wtf>=1.3.0", "httpx>=0.28.1", "huey>=3.1.0", + "mf2py>=2.0.1", "rfc3987>=1.3.8", + "xxhash>=3.8.1", ] [project.urls] @@ -34,6 +36,7 @@ Issues = "https://codeberg.org/metalgamer/webmentions-ssg/issues" [dependency-groups] dev = [ + "pyright>=1.1.411", "python-dotenv>=1.2.2", "types-wtforms>=3.2.1.20260518", ] @@ -70,6 +73,12 @@ addopts = [ "--cov=webmentions_ssg", "--cov-report=html", ] +filterwarnings = [ + "ignore:codecs\\.open\\(\\) is deprecated\\. Use open\\(\\) instead\\.:DeprecationWarning:mf2py\\.backcompat", +] [tool.uv] default-groups = ["dev", "test"] + +[tool.ruff.format] +skip-magic-trailing-comma = true diff --git a/tests/tasks/test_receiver.py b/tests/tasks/test_receiver.py index 19630c3..811ab63 100644 --- a/tests/tasks/test_receiver.py +++ b/tests/tasks/test_receiver.py @@ -9,10 +9,7 @@ from flask import Flask from webmentions_ssg import DATABASE as db from webmentions_ssg.models import ReceivedWebmention -from webmentions_ssg.url_security import ( - AddressResolutionError, - NonPublicAddressError, -) +from webmentions_ssg.url_security import AddressResolutionError SOURCE_URL = "https://source.example/article" TARGET_URL = "https://dennisfink.me/blog/example/" @@ -50,28 +47,16 @@ def create_webmention( return identifier -def get_webmention_state( - app: Flask, - identifier: uuid.UUID, -) -> tuple[str, str | None]: +def get_webmention_state(app: Flask, identifier: uuid.UUID) -> tuple[str, str | None]: with app.app_context(): - webmention = db.session.get( - ReceivedWebmention, - identifier, - ) + webmention = db.session.get(ReceivedWebmention, identifier) assert webmention is not None - return ( - webmention.status, - webmention.failure_reason, - ) + return (webmention.status, webmention.failure_reason) -def set_stream_response( - httpx_client: Mock, - response: httpx.Response, -) -> Mock: +def set_stream_response(httpx_client: Mock, response: httpx.Response) -> Mock: client = httpx_client.return_value.__enter__.return_value client.stream.return_value.__enter__.return_value = response return client @@ -80,65 +65,25 @@ def set_stream_response( @pytest.mark.parametrize( ("body", "expected"), [ + pytest.param(f'<a href="{TARGET_URL}">Reply</a>', True, id="a-href"), + pytest.param(f'<area href="{TARGET_URL}" alt="Target">', True, id="area-href"), pytest.param( - f'<a href="{TARGET_URL}">Reply</a>', - True, - id="a-href", - ), - pytest.param( - f'<area href="{TARGET_URL}" alt="Target">', - True, - id="area-href", - ), - pytest.param( - f'<link href="{TARGET_URL}" rel="alternate">', - True, - id="link-href", - ), - pytest.param( - f'<img src="{TARGET_URL}" alt="">', - True, - id="img-src", - ), - pytest.param( - f'<audio src="{TARGET_URL}"></audio>', - True, - id="audio-src", - ), - pytest.param( - f'<video src="{TARGET_URL}"></video>', - True, - id="video-src", - ), - pytest.param( - f'<audio><source src="{TARGET_URL}"></audio>', - True, - id="audio-source-src", + f'<link href="{TARGET_URL}" rel="alternate">', True, id="link-href" ), + pytest.param(f'<img src="{TARGET_URL}" alt="">', True, id="img-src"), + pytest.param(f'<audio src="{TARGET_URL}"></audio>', True, id="audio-src"), + pytest.param(f'<video src="{TARGET_URL}"></video>', True, id="video-src"), pytest.param( - f'<video><source src="{TARGET_URL}"></video>', - True, - id="video-source-src", + f'<audio><source src="{TARGET_URL}"></audio>', True, id="audio-source-src" ), pytest.param( - f'<iframe src="{TARGET_URL}"></iframe>', - True, - id="iframe-src", + f'<video><source src="{TARGET_URL}"></video>', True, id="video-source-src" ), + pytest.param(f'<iframe src="{TARGET_URL}"></iframe>', True, id="iframe-src"), + pytest.param(f'<embed src="{TARGET_URL}">', True, id="embed-src"), + pytest.param(f'<script src="{TARGET_URL}"></script>', True, id="script-src"), pytest.param( - f'<embed src="{TARGET_URL}">', - True, - id="embed-src", - ), - pytest.param( - f'<script src="{TARGET_URL}"></script>', - True, - id="script-src", - ), - pytest.param( - f'<video><track src="{TARGET_URL}"></video>', - True, - id="track-src", + f'<video><track src="{TARGET_URL}"></video>', True, id="track-src" ), pytest.param( f'<input type="image" src="{TARGET_URL}" alt="">', @@ -155,51 +100,23 @@ def set_stream_response( True, id="blockquote-cite", ), - pytest.param( - f'<q cite="{TARGET_URL}">Quotation</q>', - True, - id="q-cite", - ), - pytest.param( - f'<ins cite="{TARGET_URL}">Addition</ins>', - True, - id="ins-cite", - ), - pytest.param( - f'<del cite="{TARGET_URL}">Removal</del>', - True, - id="del-cite", - ), + pytest.param(f'<q cite="{TARGET_URL}">Quotation</q>', True, id="q-cite"), + pytest.param(f'<ins cite="{TARGET_URL}">Addition</ins>', True, id="ins-cite"), + pytest.param(f'<del cite="{TARGET_URL}">Removal</del>', True, id="del-cite"), pytest.param( '<base href="https://dennisfink.me/blog/"><a href="example/">Reply</a>', True, id="base-url", ), - pytest.param( - f'<img cite="{TARGET_URL}" alt="">', - False, - id="img-cite-invalid", - ), + pytest.param(f'<img cite="{TARGET_URL}" alt="">', False, id="img-cite-invalid"), pytest.param( f'<blockquote src="{TARGET_URL}">Quote</blockquote>', False, id="blockquote-src-invalid", ), - pytest.param( - f'<a src="{TARGET_URL}">Reply</a>', - False, - id="a-src-invalid", - ), - pytest.param( - f'<div href="{TARGET_URL}"></div>', - False, - id="div-href-invalid", - ), - pytest.param( - f'<link src="{TARGET_URL}">', - False, - id="link-src-invalid", - ), + pytest.param(f'<a src="{TARGET_URL}">Reply</a>', False, id="a-src-invalid"), + pytest.param(f'<div href="{TARGET_URL}"></div>', False, id="div-href-invalid"), + pytest.param(f'<link src="{TARGET_URL}">', False, id="link-src-invalid"), pytest.param( f'<input type="text" src="{TARGET_URL}">', False, @@ -210,51 +127,28 @@ def set_stream_response( False, id="picture-source-src-invalid", ), + pytest.param(f'<base href="{TARGET_URL}">', False, id="base-is-not-mention"), pytest.param( - f'<base href="{TARGET_URL}">', - False, - id="base-is-not-mention", - ), - pytest.param( - f'<a href="{TARGET_URL}more">Different page</a>', - False, - id="longer-url", + f'<a href="{TARGET_URL}more">Different page</a>', False, id="longer-url" ), pytest.param( f'<img cite="{TARGET_URL}" src="https://example.com/image.jpg" alt="">', False, id="invalid-cite-does-not-override-valid-src", ), + pytest.param(f"<p>{TARGET_URL}</p>", False, id="text-content"), pytest.param( - f"<p>{TARGET_URL}</p>", - False, - id="text-content", - ), - pytest.param( - '<a href="https://example.com/">Other site</a>', - False, - id="missing-target", + '<a href="https://example.com/">Other site</a>', False, id="missing-target" ), ], ) -def test_html_mentions_target( - receiver: ModuleType, - body: str, - expected: bool, -) -> None: +def test_html_mentions_target(receiver: ModuleType, body: str, expected: bool) -> None: assert ( - receiver.html_mentions_target( - body.encode(), - SOURCE_URL, - TARGET_URL, - ) - is expected + receiver.html_mentions_target(body.encode(), SOURCE_URL, TARGET_URL) is expected ) -def test_html_mentions_relative_target( - receiver: ModuleType, -) -> None: +def test_html_mentions_relative_target(receiver: ModuleType) -> None: assert receiver.html_mentions_target( b'<a href="../target/">Reply</a>', "https://source.example/posts/article/", @@ -279,63 +173,14 @@ def test_html_mentions_relative_target( ], ) def test_text_mentions_target( - receiver: ModuleType, - body: str, - target_url: str, - expected: bool, + receiver: ModuleType, body: str, target_url: str, expected: bool ) -> None: assert receiver.text_mentions_target(body, target_url) is expected -@patch("webmentions_ssg.tasks.receiver.ensure_public_url") -def test_ensure_public_request_accepts_public_url( - ensure_public_url: Mock, - receiver: ModuleType, -) -> None: - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - ensure_public_url.assert_called_once_with(SOURCE_URL) - - -@patch( - "webmentions_ssg.tasks.receiver.ensure_public_url", - side_effect=NonPublicAddressError("Non-public address"), -) -def test_ensure_public_request_rejects_non_public_address( - ensure_public_url: Mock, - receiver: ModuleType, -) -> None: - with pytest.raises( - receiver.VerificationError, - match="Source resolves to a non-public address", - ): - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - ensure_public_url.assert_called_once_with(SOURCE_URL) - - -@patch( - "webmentions_ssg.tasks.receiver.ensure_public_url", - side_effect=AddressResolutionError("Could not resolve hostname"), -) -def test_ensure_public_request_maps_dns_failure_to_temporary_error( - ensure_public_url: Mock, - receiver: ModuleType, -) -> None: - with pytest.raises( - receiver.TemporaryFetchError, - match="Source hostname could not be resolved", - ): - receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) - - ensure_public_url.assert_called_once_with(SOURCE_URL) - - @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_returns_response_and_body( - httpx_client: Mock, - app: Flask, - receiver: ModuleType, + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: response = httpx.Response( 200, @@ -360,13 +205,10 @@ def test_fetch_source_returns_response_and_body( } assert options["follow_redirects"] is True assert options["max_redirects"] == app.config.get( - "WEBMENTIONS_SSG_MAX_REDIRECTS", - 20, + "WEBMENTIONS_SSG_MAX_REDIRECTS", 20 ) assert options["trust_env"] is False - assert options["event_hooks"] == { - "request": [receiver.ensure_public_request], - } + assert options["event_hooks"] == {"request": [receiver.ensure_public_request]} @patch("webmentions_ssg.tasks.receiver.httpx.Client") @@ -392,51 +234,34 @@ def test_fetch_source_maps_http_status_to_exception( ) -> None: set_stream_response( httpx_client, - httpx.Response( - status_code, - request=httpx.Request("GET", SOURCE_URL), - ), + httpx.Response(status_code, request=httpx.Request("GET", SOURCE_URL)), ) exception_type = getattr(receiver, exception_name) - with ( - app.app_context(), - pytest.raises( - exception_type, - match=f"HTTP {status_code}", - ), - ): + with app.app_context(), pytest.raises(exception_type, match=f"HTTP {status_code}"): receiver.fetch_source(SOURCE_URL) @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_propagates_network_error( - httpx_client: Mock, - app: Flask, - receiver: ModuleType, + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: client = httpx_client.return_value.__enter__.return_value client.stream.side_effect = httpx.ConnectError( - "Connection refused", - request=httpx.Request("GET", SOURCE_URL), + "Connection refused", request=httpx.Request("GET", SOURCE_URL) ) with ( app.app_context(), - pytest.raises( - httpx.ConnectError, - match="Connection refused", - ), + pytest.raises(httpx.ConnectError, match="Connection refused"), ): receiver.fetch_source(SOURCE_URL) @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_rejects_declared_oversized_body( - httpx_client: Mock, - app: Flask, - receiver: ModuleType, + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: app.config["WEBMENTIONS_SSG_MAX_SOURCE_BYTES"] = 10 @@ -444,10 +269,7 @@ def test_fetch_source_rejects_declared_oversized_body( httpx_client, httpx.Response( 200, - headers={ - "Content-Type": "text/html", - "Content-Length": "11", - }, + headers={"Content-Type": "text/html", "Content-Length": "11"}, content=b"x" * 11, request=httpx.Request("GET", SOURCE_URL), ), @@ -455,19 +277,14 @@ def test_fetch_source_rejects_declared_oversized_body( with ( app.app_context(), - pytest.raises( - receiver.VerificationError, - match="Source document is too large", - ), + pytest.raises(receiver.VerificationError, match="Source document is too large"), ): receiver.fetch_source(SOURCE_URL) @patch("webmentions_ssg.tasks.receiver.httpx.Client") def test_fetch_source_rejects_streamed_oversized_body( - httpx_client: Mock, - app: Flask, - receiver: ModuleType, + httpx_client: Mock, app: Flask, receiver: ModuleType ) -> None: app.config["WEBMENTIONS_SSG_MAX_SOURCE_BYTES"] = 10 @@ -488,10 +305,7 @@ def test_fetch_source_rejects_streamed_oversized_body( with ( app.app_context(), - pytest.raises( - receiver.VerificationError, - match="Source document is too large", - ), + pytest.raises(receiver.VerificationError, match="Source document is too large"), ): receiver.fetch_source(SOURCE_URL) @@ -537,10 +351,7 @@ def test_fetch_source_rejects_streamed_oversized_body( id="plain-text-invalid-byte", ), pytest.param( - "text/plain", - b"No target here.", - False, - id="plain-text-without-target", + "text/plain", b"No target here.", False, id="plain-text-without-target" ), ], ) @@ -600,24 +411,14 @@ def test_source_mentions_target_rejects_unsupported_media_type( @patch("webmentions_ssg.tasks.receiver.source_mentions_target") def test_verify_webmention_marks_row_verifying_before_check( - source_mentions_target: Mock, - app: Flask, - receiver: ModuleType, + source_mentions_target: Mock, app: Flask, receiver: ModuleType ) -> None: identifier = create_webmention( - app, - status="failed", - failure_reason="Earlier failure", + app, status="failed", failure_reason="Earlier failure" ) - def verify_source( - source_url: str, - target_url: str, - ) -> bool: - webmention = db.session.get( - ReceivedWebmention, - identifier, - ) + def verify_source(source_url: str, target_url: str) -> bool: + webmention = db.session.get(ReceivedWebmention, identifier) assert webmention is not None assert webmention.status == "verifying" @@ -631,13 +432,7 @@ def test_verify_webmention_marks_row_verifying_before_check( receiver.verify_webmention.call_local(identifier) - assert get_webmention_state( - app, - identifier, - ) == ( - "verified", - None, - ) + assert get_webmention_state(app, identifier) == ("verified", None) @patch("webmentions_ssg.tasks.receiver.source_mentions_target") @@ -678,28 +473,13 @@ def test_verify_webmention_persists_final_state( receiver.verify_webmention.call_local(identifier) - assert get_webmention_state( - app, - identifier, - ) == ( - expected_status, - expected_reason, - ) + assert get_webmention_state(app, identifier) == (expected_status, expected_reason) @patch("webmentions_ssg.tasks.receiver.source_mentions_target") -@pytest.mark.parametrize( - "failure", - [ - "temporary-http", - "network", - ], -) +@pytest.mark.parametrize("failure", ["temporary-http", "network"]) def test_verify_webmention_persists_retryable_failure_and_reraises( - source_mentions_target: Mock, - app: Flask, - receiver: ModuleType, - failure: str, + source_mentions_target: Mock, app: Flask, receiver: ModuleType, failure: str ) -> None: identifier = create_webmention(app) @@ -708,32 +488,21 @@ def test_verify_webmention_persists_retryable_failure_and_reraises( exception = receiver.TemporaryFetchError("Source returned HTTP 503") case "network": exception = httpx.ConnectError( - "Connection refused", - request=httpx.Request("GET", SOURCE_URL), + "Connection refused", request=httpx.Request("GET", SOURCE_URL) ) case _: raise AssertionError(f"Unexpected failure: {failure}") source_mentions_target.side_effect = exception - with pytest.raises( - type(exception), - match=str(exception), - ): + with pytest.raises(type(exception), match=str(exception)): receiver.verify_webmention.call_local(identifier) - assert get_webmention_state( - app, - identifier, - ) == ( - "failed", - str(exception), - ) + assert get_webmention_state(app, identifier) == ("failed", str(exception)) def test_verify_webmention_ignores_unknown_identifier( - receiver: ModuleType, - caplog: pytest.LogCaptureFixture, + receiver: ModuleType, caplog: pytest.LogCaptureFixture ) -> None: identifier = uuid.uuid7() @@ -741,3 +510,52 @@ def test_verify_webmention_ignores_unknown_identifier( receiver.verify_webmention.call_local(identifier) assert f"Cannot verify unknown ReceivedWebmention {identifier}" in caplog.text + + +@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=True) +def test_ensure_public_request_accepts_public_url( + is_public_url: Mock, receiver: ModuleType +) -> None: + receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) + + is_public_url.assert_called_once_with(SOURCE_URL) + + +@patch("webmentions_ssg.tasks.receiver.is_public_url", return_value=False) +def test_ensure_public_request_rejects_non_public_address( + is_public_url: Mock, receiver: ModuleType +) -> None: + with pytest.raises( + receiver.VerificationError, match="Source resolves to a non-public address" + ): + receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) + + is_public_url.assert_called_once_with(SOURCE_URL) + + +@patch( + "webmentions_ssg.tasks.receiver.is_public_url", + side_effect=AddressResolutionError("Could not resolve hostname"), +) +def test_ensure_public_request_maps_dns_failure_to_temporary_error( + is_public_url: Mock, receiver: ModuleType +) -> None: + with pytest.raises( + receiver.TemporaryFetchError, match="Source hostname could not be resolved" + ): + receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) + + is_public_url.assert_called_once_with(SOURCE_URL) + + +@patch( + "webmentions_ssg.tasks.receiver.is_public_url", + side_effect=ValueError("No hostname was specified"), +) +def test_ensure_public_request_rejects_url_without_hostname( + is_public_url: Mock, receiver: ModuleType +) -> None: + with pytest.raises(receiver.VerificationError, match="Source URL has no hostname"): + receiver.ensure_public_request(httpx.Request("GET", SOURCE_URL)) + + is_public_url.assert_called_once_with(SOURCE_URL) diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py new file mode 100644 index 0000000..65031b4 --- /dev/null +++ b/tests/tasks/test_scanner.py @@ -0,0 +1,1558 @@ +from pathlib import Path +from types import ModuleType +from uuid import UUID + +import pytest +import sqlalchemy as sa +from bs4 import BeautifulSoup +from flask import Flask +from pytest import MonkeyPatch + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source + +BASE_URL = "https://dennisfink.me/blog/" +SOURCE_URL = f"{BASE_URL}example/" + + +@pytest.fixture +def scanner_module(app: Flask) -> ModuleType: + # Importing scanner registers Huey tasks. The dependency on the + # app fixture guarantees that Huey has been initialized first. + _ = app + + from webmentions_ssg.tasks import scanner + + return scanner + + +def run_scan(app: Flask, scanner_module: ModuleType) -> None: + with app.app_context(): + scanner_module.scan_sources() + + +def parse_html(html: str) -> BeautifulSoup: + return BeautifulSoup(html, "html5lib") + + +def write_post( + root: Path, + slug: str, + content: str, + *, + canonical: str | None = None, + entry_url: str | None = None, +) -> Path: + directory = root / slug + directory.mkdir(parents=True, exist_ok=True) + + source_url = f"{BASE_URL}{slug}/" + + if canonical is None: + canonical = source_url + + if entry_url is None: + entry_url = source_url + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{canonical}" + > + </head> + <body> + <article class="h-entry"> + <a + class="u-url" + href="{entry_url}" + > + Permalink + </a> + + <div class="e-content"> + {content} + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + return path + + +def configure_scanner(app: Flask, root: Path, *, base_url: str | None = None) -> None: + app.config["WEBMENTIONS_SSG_SOURCE_DIRECTORY"] = str(root) + app.config["WEBMENTIONS_SSG_SOURCE_BASE_URL"] = base_url + + +# --------------------------------------------------------------------------- +# Microformats parsing +# --------------------------------------------------------------------------- + + +def test_parse_entry_returns_microformats_entry(scanner_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element = document.find(class_="h-entry") + + assert element is not None + + entry = scanner_module.parse_entry(element, SOURCE_URL) + + assert "h-entry" in entry["type"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_parse_entry_rejects_non_entry(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article> + <p>Hello world</p> + </article> + """ + ) + + element = document.find("article") + + assert element is not None + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one parsed h-entry" + ): + scanner_module.parse_entry(element, SOURCE_URL) + + +def test_primary_entry_returns_source_entry(scanner_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry" id="source"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <p class="p-name"> + Example post + </p> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + element, entry = scanner_module.primary_entry(document, SOURCE_URL) + + assert element["id"] == "source" + assert entry["properties"]["name"] == ["Example post"] + assert entry["properties"]["url"] == [SOURCE_URL] + + +def test_primary_entry_rejects_missing_h_entry(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <main> + <p>No microformats here.</p> + </main> + """ + ) + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one h-entry, found 0" + ): + scanner_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_multiple_h_entries(scanner_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + First + </a> + </article> + + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Second + </a> + </article> + """ + ) + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" + ): + scanner_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nested_h_entry(scanner_module: ModuleType) -> None: + document = parse_html( + f""" + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/nested/" + > + Nested + </a> + </article> + </div> + </article> + """ + ) + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one h-entry, found 2" + ): + scanner_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_missing_u_url(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(scanner_module.SourceScanError, match="does not match"): + scanner_module.primary_entry(document, SOURCE_URL) + + +def test_primary_entry_rejects_nonmatching_u_url(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <a + class="u-url" + href="https://example.com/other/" + > + Other + </a> + + <div class="e-content"> + Hello world + </div> + </article> + """ + ) + + with pytest.raises(scanner_module.SourceScanError, match="does not match"): + scanner_module.primary_entry(document, SOURCE_URL) + + +# --------------------------------------------------------------------------- +# e-content +# --------------------------------------------------------------------------- + + +def test_content_element_returns_e_content(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content" id="content"> + Hello world + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + content = scanner_module.content_element(entry) + + assert content["id"] == "content" + + +def test_content_element_rejects_missing_e_content(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <p>Hello world</p> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one e-content, found 0" + ): + scanner_module.content_element(entry) + + +def test_content_element_rejects_multiple_e_content(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <article class="h-entry"> + <div class="e-content"> + First + </div> + + <div class="e-content"> + Second + </div> + </article> + """ + ) + + entry = document.find(class_="h-entry") + + assert entry is not None + + with pytest.raises( + scanner_module.SourceScanError, match="Expected exactly one e-content, found 2" + ): + scanner_module.content_element(entry) + + +# --------------------------------------------------------------------------- +# Canonical URLs +# --------------------------------------------------------------------------- + + +def test_canonical_url_returns_absolute_url(scanner_module: ModuleType) -> None: + document = parse_html( + f""" + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + </html> + """ + ) + + result = scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result == SOURCE_URL + + +def test_canonical_url_returns_none_when_missing(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <html> + <head> + <title>Example</title> + </head> + </html> + """ + ) + + result = scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + assert result is None + + +def test_canonical_url_resolves_relative_url_with_base_url( + scanner_module: ModuleType, +) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + result = scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + assert result == ("https://dennisfink.me/blog/example/canonical/") + + +def test_relative_canonical_requires_base_url(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href="./canonical/" + > + </head> + </html> + """ + ) + + with pytest.raises(scanner_module.SourceScanError, match="relative canonical"): + scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=None + ) + + +def test_empty_canonical_is_rejected(scanner_module: ModuleType) -> None: + document = parse_html( + """ + <html> + <head> + <link + rel="canonical" + href=" " + > + </head> + </html> + """ + ) + + with pytest.raises(scanner_module.SourceScanError, match="empty canonical URL"): + scanner_module.canonical_url( + document, relative_path=Path("example/index.html"), base_url=BASE_URL + ) + + +# --------------------------------------------------------------------------- +# Target extraction +# --------------------------------------------------------------------------- + + +def test_scan_source_uses_canonical_without_base_url( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + assert len(scanned.content_hash) == 32 + + +def test_scan_source_falls_back_to_base_url( + scanner_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <a href="https://example.com/target"> + Target + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=BASE_URL) + + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_source_requires_canonical_or_base_url( + scanner_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <p>Hello world.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + with pytest.raises(scanner_module.SourceScanError, match="no canonical URL"): + scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + +def test_scan_source_extracts_only_content_links( + scanner_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a href="https://example.com/metadata"> + Metadata link + </a> + + <div class="e-content"> + <a href="https://example.com/content"> + Content link + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/content"}) + + +def test_scan_source_extracts_relative_content_link( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="../other/"> + Other post + </a> + """, + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://dennisfink.me/blog/other/"}) + + +@pytest.mark.parametrize( + "property_name", ("in-reply-to", "like-of", "repost-of", "bookmark-of") +) +def test_scan_source_extracts_reaction_properties( + scanner_module: ModuleType, tmp_path: Path, property_name: str +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + target = f"https://example.com/{property_name}" + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <a + class="u-{property_name}" + href="{target}" + > + Reaction target + </a> + + <div class="e-content"> + <p>Post content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({target}) + + +def test_scan_source_extracts_anchor_from_e_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize("tag", ("link", "area")) +def test_scan_source_ignores_non_anchor_href_elements( + scanner_module: ModuleType, tmp_path: Path, tag: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <{tag} href="https://example.com/target"> + """, + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset() + + +def test_scan_source_ignores_self_fragment( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + """ + <a href="#section"> + Section + </a> + + <a href="https://example.com/target"> + Target + </a> + """, + ) + + scanned = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +# --------------------------------------------------------------------------- +# Database reconciliation +# --------------------------------------------------------------------------- + + +def test_scan_creates_source_and_webmentions( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + + <a href="https://example.com/b"> + B + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + + assert source is not None + assert source.path == "example/index.html" + assert source.url == SOURCE_URL + assert source.revision == 1 + assert source.deleted_at is None + assert len(source.content_hash) == 32 + + webmentions = list( + db.session.scalars( + sa.select(SentWebmention).order_by(SentWebmention.target) + ) + ) + + assert [webmention.target for webmention in webmentions] == [ + "https://example.com/a", + "https://example.com/b", + ] + + assert all(webmention.active for webmention in webmentions) + assert all(webmention.desired_revision == 1 for webmention in webmentions) + assert all(webmention.processed_revision is None for webmention in webmentions) + + identifiers = {webmention.uuid for webmention in webmentions} + + assert set(queued) == identifiers + + +def test_unchanged_source_does_not_increment_revision( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert webmention is not None + + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + + db.session.commit() + + queued.clear() + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + + assert source is not None + assert source.revision == 1 + + assert queued == [] + + +def test_updated_source_increments_revision( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <p>Original content</p> + + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert webmention is not None + + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + + db.session.commit() + + queued.clear() + + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <p>Updated content</p> + + <a href="https://example.com/a"> + A + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + + assert webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision == 1 + assert webmention.sent_revision == 1 + assert webmention.pending + + identifier = webmention.uuid + + assert queued == [identifier] + + +def test_new_target_is_added_on_update( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <p>No links yet.</p> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + assert queued == [] + + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <a href="https://example.com/new"> + New + </a> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + + assert webmention.target == ("https://example.com/new") + assert webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision is None + assert webmention.sent_revision is None + assert webmention.pending + + identifier = webmention.uuid + + assert queued == [identifier] + + +def test_removed_sent_target_is_queued_again( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/sent"> + Sent + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert webmention is not None + + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + + identifier = webmention.uuid + db.session.commit() + + queued.clear() + + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <p>The link is gone.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + + assert not webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision == 1 + assert webmention.sent_revision == 1 + assert webmention.pending + + assert queued == [identifier] + + +def test_removed_unsent_target_is_not_queued( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/unsent"> + Unsent + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + queued.clear() + + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a class="u-url" href="{SOURCE_URL}"> + Permalink + </a> + + <div class="e-content"> + <p>The link is gone.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + + assert not webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision == 2 + assert webmention.sent_revision is None + assert not webmention.pending + + assert queued == [] + + +# --------------------------------------------------------------------------- +# Source deletion/restoration +# --------------------------------------------------------------------------- + + +def test_deleted_source_queues_previously_sent_webmention( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert webmention is not None + + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + + identifier = webmention.uuid + db.session.commit() + + queued.clear() + path.unlink() + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + assert source.deleted_at is not None + + assert not webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision == 1 + assert webmention.sent_revision == 1 + assert webmention.pending + + assert queued == [identifier] + + +def test_deleted_source_does_not_queue_unsent_webmention( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + queued.clear() + path.unlink() + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 2 + assert source.deleted_at is not None + + assert not webmention.active + assert webmention.desired_revision == 2 + assert webmention.processed_revision == 2 + assert webmention.sent_revision is None + assert not webmention.pending + + assert queued == [] + + +def test_restored_source_creates_new_revision( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + with app.app_context(): + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert webmention is not None + + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + + db.session.commit() + + path.unlink() + run_scan(app, scanner_module) + + queued.clear() + + write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + + assert source is not None + assert webmention is not None + + assert source.revision == 3 + assert source.deleted_at is None + + assert webmention.active + assert webmention.desired_revision == 3 + assert webmention.sent_revision == 1 + assert webmention.pending + + identifier = webmention.uuid + + assert queued == [identifier] + + +# --------------------------------------------------------------------------- +# Queue recovery and invalid sources +# --------------------------------------------------------------------------- + + +def test_pending_webmention_is_requeued_on_next_scan( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + + write_post( + tmp_path, + "example", + """ + <a href="https://example.com/a"> + A + </a> + """, + ) + + queued: list[UUID] = [] + + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + + assert len(queued) == 1 + + identifier = queued[0] + queued.clear() + + run_scan(app, scanner_module) + + assert queued == [identifier] + + +def test_invalid_known_source_is_not_deleted( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + configure_scanner(app, tmp_path) + + path = write_post( + tmp_path, + "example", + """ + <p>Hello world.</p> + """, + ) + + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + + run_scan(app, scanner_module) + + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <p>The h-entry disappeared.</p> + </body> + </html> + """, + encoding="utf-8", + ) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + + assert source is not None + assert source.revision == 1 + assert source.deleted_at is None + + assert "Could not scan source" in caplog.text + assert "Expected exactly one h-entry, found 0" in caplog.text + + +def test_scan_sources_accepts_valid_base_url( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path, base_url=BASE_URL) + + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + """ + <!doctype html> + <html> + <body> + <article class="h-entry"> + <a class="u-url" href="./"> + Permalink + </a> + + <div class="e-content"> + <p>Hello world.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + + assert source is not None + assert source.url == SOURCE_URL + + +def test_scan_sources_rejects_invalid_base_url( + app: Flask, scanner_module: ModuleType, tmp_path: Path +) -> None: + configure_scanner(app, tmp_path, base_url="not-a-url") + + with pytest.raises(RuntimeError, match="absolute HTTP or HTTPS URL"): + run_scan(app, scanner_module) + + +@pytest.mark.parametrize( + "href", + ( + "https://dennisfink.me/blog/other/", + "https://www.dennisfink.me/blog/other/", + "https://webmentions.dennisfink.me/endpoint", + "https://foo.bar.dennisfink.me/example", + ), +) +def test_scan_source_ignores_matching_hostname( + scanner_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Ignored + </a> + + <a href="https://example.com/target"> + External + </a> + """, + ) + + scanned = scanner_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({"https://example.com/target"}) + + +@pytest.mark.parametrize( + "href", + ( + "https://notdennisfink.me/example", + "https://dennisfink.me.example.com/example", + "https://example.com/example", + ), +) +def test_scan_source_keeps_nonmatching_hostname( + scanner_module: ModuleType, tmp_path: Path, href: str +) -> None: + path = write_post( + tmp_path, + "example", + f""" + <a href="{href}"> + Target + </a> + """, + ) + + scanned = scanner_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset({href}) + + +def test_scan_source_ignores_reaction_to_matching_hostname( + scanner_module: ModuleType, tmp_path: Path +) -> None: + directory = tmp_path / "example" + directory.mkdir() + + path = directory / "index.html" + path.write_text( + f""" + <!doctype html> + <html> + <head> + <link + rel="canonical" + href="{SOURCE_URL}" + > + </head> + <body> + <article class="h-entry"> + <a + class="u-url" + href="{SOURCE_URL}" + > + Permalink + </a> + + <a + class="u-in-reply-to" + href="https://www.dennisfink.me/blog/other/" + > + Reply target + </a> + + <div class="e-content"> + <p>Reply content.</p> + </div> + </article> + </body> + </html> + """, + encoding="utf-8", + ) + + scanned = scanner_module.scan_source_file( + path, + root=tmp_path, + base_url=None, + ignored_hostnames=("dennisfink.me", "*.dennisfink.me"), + ) + + assert scanned.targets == frozenset() diff --git a/tests/tasks/test_sender.py b/tests/tasks/test_sender.py new file mode 100644 index 0000000..86799f7 --- /dev/null +++ b/tests/tasks/test_sender.py @@ -0,0 +1,788 @@ +import logging +import uuid +from datetime import datetime, timezone +from types import ModuleType +from unittest.mock import MagicMock, Mock + +import httpx +import pytest +from flask import Flask + +from webmentions_ssg import DATABASE as db +from webmentions_ssg.models import SentWebmention, SentWebmentionStatus, Source + +SOURCE_URL = "https://dennisfink.me/blog/example/" +TARGET_URL = "https://example.com/post" +ENDPOINT_URL = "https://example.com/webmention" +STATUS_URL = "https://example.com/webmention/status/123" + + +def create_sent_webmention( + app: Flask, + *, + active: bool = True, + desired_revision: int = 1, + processed_revision: int | None = None, + sent_revision: int | None = None, + status: SentWebmentionStatus | None = None, + failure_reason: str | None = None, + endpoint: str | None = None, + response_status: int | None = None, + status_url: str | None = None, +) -> uuid.UUID: + now = datetime.now(timezone.utc) + + with app.app_context(): + source = Source( + path="example/index.html", + url=SOURCE_URL, + content_hash="0" * 32, + revision=desired_revision, + last_seen_at=now, + revised_at=now, + ) + + webmention = SentWebmention( + target=TARGET_URL, + active=active, + desired_revision=desired_revision, + processed_revision=processed_revision, + sent_revision=sent_revision, + status=status, + failure_reason=failure_reason, + endpoint=endpoint, + response_status=response_status, + status_url=status_url, + ) + + source.sent_webmentions.append(webmention) + + db.session.add(source) + db.session.commit() + + return webmention.uuid + + +def mock_sender_requests( + sender_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + endpoint: str | None = ENDPOINT_URL, + result: tuple[int, str | None] = (202, None), +) -> tuple[Mock, Mock, Mock, Mock]: + httpx_client = MagicMock() + client = httpx_client.return_value.__enter__.return_value + + discover = Mock(return_value=endpoint) + post = Mock(return_value=result) + + monkeypatch.setattr(sender_module.httpx, "Client", httpx_client) + monkeypatch.setattr(sender_module, "discover_webmention_endpoint", discover) + monkeypatch.setattr(sender_module, "post_webmention", post) + + return httpx_client, client, discover, post + + +@pytest.fixture +def sender_module(app: Flask) -> ModuleType: + # Importing sender registers Huey tasks. The dependency on the + # app fixture guarantees that Huey has been initialized first. + _ = app + + from webmentions_ssg.tasks import sender + + return sender + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ( + '<https://example.com/webmention>; rel="webmention"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + "<https://example.com/webmention>; rel=webmention", + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/webmention>; rel="webmention alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + '<https://example.com/webmention>; REL="WebMention Alternate"', + ("https://example.com/webmention", {"webmention", "alternate"}), + ), + ( + "<https://example.com/webmention>; " + 'type="text/html"; rel="webmention"; title="Endpoint"', + ("https://example.com/webmention", {"webmention"}), + ), + ( + '<https://example.com/stylesheet>; type="text/css"', + ("https://example.com/stylesheet", set()), + ), + ), +) +def test_parse_link_value( + sender_module: ModuleType, value: str, expected: tuple[str, set[str]] +) -> None: + assert sender_module.parse_link_value(value) == expected + + +@pytest.mark.parametrize( + "value", ("", "https://example.com/webmention", "<https://example.com/webmention") +) +def test_parse_link_value_rejects_malformed_value( + sender_module: ModuleType, value: str +) -> None: + assert sender_module.parse_link_value(value) is None + + +@pytest.mark.parametrize( + ("target", "headers", "html", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/1", + (("Link", "</test/1/webmention?head=true>; rel=webmention"),), + "", + "https://example.com/test/1/webmention?head=true", + id="1-http-link-unquoted-rel-relative-url", + ), + pytest.param( + "https://example.com/test/2", + ( + ( + "Link", + "<https://example.com/test/2/webmention?head=true>; rel=webmention", + ), + ), + "", + "https://example.com/test/2/webmention?head=true", + id="2-http-link-unquoted-rel-absolute-url", + ), + pytest.param( + "https://example.com/test/3", + (), + """ + <link + rel="webmention" + href="/test/3/webmention" + > + """, + "https://example.com/test/3/webmention", + id="3-html-link-relative-url", + ), + pytest.param( + "https://example.com/test/4", + (), + """ + <link + rel="webmention" + href="https://example.com/test/4/webmention" + > + """, + "https://example.com/test/4/webmention", + id="4-html-link-absolute-url", + ), + pytest.param( + "https://example.com/test/5", + (), + """ + <a + rel="webmention" + href="/test/5/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/5/webmention", + id="5-html-a-relative-url", + ), + pytest.param( + "https://example.com/test/6", + (), + """ + <a + rel="webmention" + href="https://example.com/test/6/webmention" + > + Endpoint + </a> + """, + "https://example.com/test/6/webmention", + id="6-html-a-absolute-url", + ), + pytest.param( + "https://example.com/test/7", + ( + ( + "LinK", + "<https://example.com/test/7/webmention?head=true>; rel=webmention", + ), + ), + "", + "https://example.com/test/7/webmention?head=true", + id="7-http-link-strange-casing", + ), + pytest.param( + "https://example.com/test/8", + ( + ( + "Link", + '<https://example.com/test/8/webmention?head=true>; rel="webmention"', + ), + ), + "", + "https://example.com/test/8/webmention?head=true", + id="8-http-link-quoted-rel", + ), + pytest.param( + "https://example.com/test/9", + (), + """ + <link + rel="webmention somethingelse" + href="https://example.com/test/9/webmention" + > + """, + "https://example.com/test/9/webmention", + id="9-multiple-html-rel-values", + ), + pytest.param( + "https://example.com/test/10", + ( + ( + "Link", + "<https://example.com/test/10/webmention?head=true>; " + 'rel="webmention somethingelse"', + ), + ), + "", + "https://example.com/test/10/webmention?head=true", + id="10-multiple-link-header-rel-values", + ), + pytest.param( + "https://example.com/test/11", + (("Link", '</test/11/webmention>; rel="webmention"'),), + """ + <link + rel="webmention" + href="/test/11/webmention/error" + > + + <a + rel="webmention" + href="/test/11/webmention/error" + > + Wrong endpoint + </a> + """, + "https://example.com/test/11/webmention", + id="11-http-link-precedence", + ), + pytest.param( + "https://example.com/test/12", + (), + """ + <link + rel="not-webmention" + href="/test/12/webmention/error" + > + + <a + rel="webmention" + href="/test/12/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/12/webmention", + id="12-exact-rel-match", + ), + pytest.param( + "https://example.com/test/13", + (), + """ + <!-- + <a + rel="webmention" + href="/test/13/webmention/error" + > + False endpoint + </a> + --> + + <a + rel="webmention" + href="/test/13/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/13/webmention", + id="13-endpoint-inside-html-comment", + ), + pytest.param( + "https://example.com/test/14", + (), + """ + <code> + <a + rel="webmention" + href="/test/14/webmention/error" + > + False endpoint + </a> + </code> + + <a + rel="webmention" + href="/test/14/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/14/webmention", + id="14-endpoint-in-escaped-html", + ), + pytest.param( + "https://example.com/test/15", + (), + """ + <link + rel="webmention" + href="" + > + """, + "https://example.com/test/15", + id="15-empty-href", + ), + pytest.param( + "https://example.com/test/16", + (), + """ + <a + rel="webmention" + href="/test/16/webmention" + > + First endpoint + </a> + + <link + rel="webmention" + href="/test/16/webmention/error" + > + """, + "https://example.com/test/16/webmention", + id="16-a-before-link", + ), + pytest.param( + "https://example.com/test/17", + (), + """ + <link + rel="webmention" + href="/test/17/webmention" + > + + <a + rel="webmention" + href="/test/17/webmention/error" + > + Second endpoint + </a> + """, + "https://example.com/test/17/webmention", + id="17-link-before-a", + ), + pytest.param( + "https://example.com/test/18", + ( + ("Link", '<https://example.com/test/18/webmention/error>; rel="other"'), + ( + "Link", + "<https://example.com/test/18/webmention?head=true>; " + 'rel="webmention"', + ), + ), + "", + "https://example.com/test/18/webmention?head=true", + id="18-multiple-http-link-headers", + ), + pytest.param( + "https://example.com/test/19", + ( + ( + "Link", + "<https://example.com/test/19/webmention/error>; " + 'rel="other", ' + "<https://example.com/test/19/webmention?head=true>; " + 'rel="webmention"', + ), + ), + "", + "https://example.com/test/19/webmention?head=true", + id="19-single-header-multiple-values", + ), + pytest.param( + "https://example.com/test/20", + (), + """ + <link rel="webmention"> + + <a + rel="webmention" + href="/test/20/webmention" + > + Correct endpoint + </a> + """, + "https://example.com/test/20/webmention", + id="20-link-without-href", + ), + pytest.param( + "https://example.com/test/21", + (), + """ + <link + rel="webmention" + href="/test/21/webmention?query=yes" + > + """, + "https://example.com/test/21/webmention?query=yes", + id="21-endpoint-query-string", + ), + pytest.param( + "https://example.com/test/22", + (), + """ + <link + rel="webmention" + href="22/webmention" + > + """, + "https://example.com/test/22/webmention", + id="22-endpoint-relative-to-path", + ), + ), +) +def test_discovery( + app: Flask, + sender_module: ModuleType, + target: str, + headers: tuple[tuple[str, str], ...], + html: str, + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "HEAD": + return httpx.Response(200, headers=headers) + + return httpx.Response(200, headers={"Content-Type": "text/html"}, text=html) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +@pytest.mark.parametrize( + ("target", "responses", "expected_endpoint"), + ( + pytest.param( + "https://example.com/test/23/page", + { + "/test/23/page": (302, {"Location": "page/redirect-key"}), + "/test/23/page/redirect-key": ( + 200, + {"Link": ("<webmention-endpoint/endpoint-key>; rel=webmention")}, + ), + }, + ("https://example.com/test/23/page/webmention-endpoint/endpoint-key"), + id="relative-endpoint-after-redirect", + ), + ), +) +def test_discovery_redirect( + app: Flask, + sender_module: ModuleType, + target: str, + responses: dict[str, tuple[int, dict[str, str]]], + expected_endpoint: str, +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + try: + status, headers = responses[request.url.path] + except KeyError: + raise AssertionError(f"Unexpected request: {request.url}") from None + + return httpx.Response(status, headers=headers) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint(client, target) + + assert endpoint == expected_endpoint + + +def test_discovery_stops_after_head(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + return httpx.Response(200, headers={"Link": "</webmention>; rel=webmention"}) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD"] + + +def test_discovery_falls_back_to_get(app: Flask, sender_module: ModuleType) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + + if request.method == "HEAD": + return httpx.Response(200) + + return httpx.Response( + 200, + headers={"Content-Type": "text/html"}, + text=""" + <link + rel="webmention" + href="/webmention" + > + """, + ) + + with ( + app.app_context(), + httpx.Client( + transport=httpx.MockTransport(handler), follow_redirects=True + ) as client, + ): + endpoint = sender_module.discover_webmention_endpoint( + client, "https://example.com/post" + ) + + assert endpoint == "https://example.com/webmention" + assert [request.method for request in requests] == ["HEAD", "GET"] + + +def test_send_webmention_persists_success( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention(app) + + _, client, discover, post = mock_sender_requests( + sender_module, monkeypatch, result=(201, STATUS_URL) + ) + + sender_module.send_webmention.call_local(identifier) + + discover.assert_called_once_with(client, TARGET_URL) + post.assert_called_once_with( + client, endpoint=ENDPOINT_URL, source=SOURCE_URL, target=TARGET_URL + ) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.processed_revision == 1 + assert webmention.sent_revision == 1 + assert webmention.status == SentWebmentionStatus.SENT + assert webmention.failure_reason is None + assert webmention.endpoint == ENDPOINT_URL + assert webmention.response_status == 201 + assert webmention.status_url == STATUS_URL + assert webmention.last_attempted_at is not None + assert webmention.last_sent_at is not None + assert not webmention.pending + + +def test_send_webmention_marks_unsupported_target_processed( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + desired_revision=2, + processed_revision=1, + sent_revision=1, + status=SentWebmentionStatus.SENT, + endpoint="https://old.example/webmention", + response_status=201, + status_url="https://old.example/status/123", + ) + + _, client, discover, post = mock_sender_requests( + sender_module, monkeypatch, endpoint=None + ) + + sender_module.send_webmention.call_local(identifier) + + discover.assert_called_once_with(client, TARGET_URL) + post.assert_not_called() + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.processed_revision == 2 + assert webmention.sent_revision == 1 + assert webmention.status == SentWebmentionStatus.UNSUPPORTED + assert webmention.failure_reason == "No Webmention endpoint discovered" + assert webmention.endpoint is None + assert webmention.response_status is None + assert webmention.status_url is None + assert webmention.last_attempted_at is not None + assert not webmention.pending + + +def test_send_webmention_persists_permanent_failure( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + desired_revision=2, + processed_revision=1, + sent_revision=1, + status=SentWebmentionStatus.SENT, + status_url="https://old.example/status/123", + ) + + mock_sender_requests(sender_module, monkeypatch, result=(400, None)) + + sender_module.send_webmention.call_local(identifier) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.processed_revision == 2 + assert webmention.sent_revision == 1 + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.failure_reason == "Webmention endpoint returned HTTP 400" + assert webmention.endpoint == ENDPOINT_URL + assert webmention.response_status == 400 + assert webmention.status_url is None + assert not webmention.pending + + +def test_send_webmention_persists_temporary_failure_and_reraises( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + desired_revision=2, + processed_revision=1, + sent_revision=1, + status=SentWebmentionStatus.SENT, + status_url="https://old.example/status/123", + ) + + mock_sender_requests(sender_module, monkeypatch, result=(503, None)) + + with pytest.raises( + sender_module.TemporarySenderError, + match="Webmention endpoint returned HTTP 503", + ): + sender_module.send_webmention.call_local(identifier) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.processed_revision == 1 + assert webmention.sent_revision == 1 + assert webmention.status == SentWebmentionStatus.FAILED + assert webmention.failure_reason == "Webmention endpoint returned HTTP 503" + assert webmention.endpoint == ENDPOINT_URL + assert webmention.response_status == 503 + assert webmention.status_url is None + assert webmention.pending + + +def test_send_webmention_ignores_processed_revision( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention( + app, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + ) + + httpx_client, _, discover, post = mock_sender_requests(sender_module, monkeypatch) + + sender_module.send_webmention.call_local(identifier) + + httpx_client.assert_not_called() + discover.assert_not_called() + post.assert_not_called() + + +def test_send_webmention_ignores_result_for_superseded_revision( + app: Flask, sender_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + identifier = create_sent_webmention(app) + + _, _, _, post = mock_sender_requests(sender_module, monkeypatch) + + def supersede_revision(*args, **kwargs) -> tuple[int, None]: + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + + webmention.desired_revision = 2 + db.session.commit() + + return (202, None) + + post.side_effect = supersede_revision + + sender_module.send_webmention.call_local(identifier) + + with app.app_context(): + webmention = db.session.get(SentWebmention, identifier) + + assert webmention is not None + assert webmention.desired_revision == 2 + assert webmention.processed_revision is None + assert webmention.sent_revision is None + assert webmention.status is None + assert webmention.pending + + +def test_send_webmention_ignores_unknown_identifier( + sender_module: ModuleType, caplog: pytest.LogCaptureFixture +) -> None: + identifier = uuid.uuid7() + + with caplog.at_level(logging.WARNING): + sender_module.send_webmention.call_local(identifier) + + assert f"Cannot send unknown SentWebmention {identifier}" in caplog.text diff --git a/tests/test_forms.py b/tests/test_forms.py index 417c655..73c6ffc 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -10,83 +10,54 @@ VALID_SOURCE = "https://source.example/post" VALID_TARGET = "https://dennisfink.me/blog/example/" -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) @pytest.mark.parametrize( - ( - "form_data", - "invalid_field", - "expected_error", - ), + ("form_data", "invalid_field", "expected_error"), [ pytest.param( - { - "target": VALID_TARGET, - }, + {"target": VALID_TARGET}, "source", "This field is required.", id="source-required", ), pytest.param( - { - "source": "not a URL", - "target": VALID_TARGET, - }, + {"source": "not a URL", "target": VALID_TARGET}, "source", "Invalid URL.", id="source-url", ), pytest.param( - { - "source": "ftp://source.example/post", - "target": VALID_TARGET, - }, + {"source": "ftp://source.example/post", "target": VALID_TARGET}, "source", "source must begin with http or https", id="source-scheme", ), pytest.param( - { - "source": VALID_TARGET, - "target": VALID_TARGET, - }, + {"source": VALID_TARGET, "target": VALID_TARGET}, "source", None, id="source-not-equal-to-target", ), pytest.param( - { - "source": VALID_SOURCE, - }, + {"source": VALID_SOURCE}, "target", "This field is required.", id="target-required", ), pytest.param( - { - "source": VALID_SOURCE, - "target": "not a URL", - }, + {"source": VALID_SOURCE, "target": "not a URL"}, "target", "Invalid URL.", id="target-url", ), pytest.param( - { - "source": VALID_SOURCE, - "target": "ftp://dennisfink.me/blog/example/", - }, + {"source": VALID_SOURCE, "target": "ftp://dennisfink.me/blog/example/"}, "target", "target must begin with http or https", id="target-scheme", ), pytest.param( - { - "source": VALID_SOURCE, - "target": "https://example.com/post", - }, + {"source": VALID_SOURCE, "target": "https://example.com/post"}, "target", None, id="target-allowed-hostname", @@ -99,14 +70,8 @@ def test_endpoint_form_rejects_invalid_data( invalid_field: str, expected_error: str | None, ) -> None: - with app.test_request_context( - "/endpoint", - method="POST", - ): - form = EndpointForm( - formdata=MultiDict(form_data), - meta={"csrf": False}, - ) + with app.test_request_context("/endpoint", method="POST"): + form = EndpointForm(formdata=MultiDict(form_data), meta={"csrf": False}) assert not form.validate() assert invalid_field in form.errors @@ -115,24 +80,11 @@ def test_endpoint_form_rejects_invalid_data( assert expected_error in form.errors[invalid_field] -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) -def test_endpoint_form_accepts_valid_data( - app: Flask, -) -> None: - with app.test_request_context( - "/endpoint", - method="POST", - ): +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) +def test_endpoint_form_accepts_valid_data(app: Flask) -> None: + with app.test_request_context("/endpoint", method="POST"): form = EndpointForm( - formdata=MultiDict( - { - "source": VALID_SOURCE, - "target": VALID_TARGET, - } - ), + formdata=MultiDict({"source": VALID_SOURCE, "target": VALID_TARGET}), meta={"csrf": False}, ) diff --git a/tests/test_url_security.py b/tests/test_url_security.py index b10e6ff..ffa0378 100644 --- a/tests/test_url_security.py +++ b/tests/test_url_security.py @@ -1,11 +1,9 @@ from unittest.mock import Mock, patch +import dns.resolver import pytest -from webmentions_ssg.url_security import ( - NonPublicAddressError, - ensure_public_url, -) +from webmentions_ssg.url_security import AddressResolutionError, is_public_url @patch("webmentions_ssg.url_security.dns.resolver.resolve_name") @@ -21,14 +19,50 @@ from webmentions_ssg.url_security import ( ("http://169.254.169.254/", ["169.254.169.254"]), ("http://localhost/", ["127.0.0.1", "::1"]), ("http://internal.example/", ["192.168.1.10"]), + ("https://example.com/", ["93.184.216.34", "192.168.1.10"]), + ], +) +def test_is_public_url_returns_false_for_non_public_addresses( + resolve_name: Mock, url: str, resolved_addresses: list[str] +) -> None: + resolve_name.return_value.addresses.return_value = resolved_addresses + + assert not is_public_url(url) + + +@patch("webmentions_ssg.url_security.dns.resolver.resolve_name") +@pytest.mark.parametrize( + ("url", "resolved_addresses"), + [ + ("https://example.com/", ["93.184.216.34"]), + ( + "https://example.com/", + ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"], + ), ], ) -def test_ensure_public_url_rejects_non_public_addresses( - resolve_name: Mock, - url: str, - resolved_addresses: list[str], +def test_is_public_url_returns_true_for_public_addresses( + resolve_name: Mock, url: str, resolved_addresses: list[str] ) -> None: resolve_name.return_value.addresses.return_value = resolved_addresses - with pytest.raises(NonPublicAddressError): - ensure_public_url(url) + assert is_public_url(url) + + +@patch( + "webmentions_ssg.url_security.dns.resolver.resolve_name", + side_effect=dns.resolver.NXDOMAIN(), +) +def test_is_public_url_raises_for_resolution_failure(resolve_name: Mock) -> None: + with pytest.raises(AddressResolutionError, match="Could not resolve hostname"): + is_public_url("https://nonexistent.example/") + + resolve_name.assert_called_once_with("nonexistent.example") + + +@patch("webmentions_ssg.url_security.dns.resolver.resolve_name") +def test_is_public_url_raises_when_url_has_no_hostname(resolve_name: Mock) -> None: + with pytest.raises(ValueError, match="No hostname was specified"): + is_public_url("/relative/url") + + resolve_name.assert_not_called() diff --git a/tests/test_views.py b/tests/test_views.py index 06e546c..aac281e 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,29 +1,89 @@ import uuid +from datetime import datetime, timezone from unittest.mock import Mock, call, patch +import pytest import sqlalchemy as sa from flask import Flask from flask.testing import FlaskClient from webmentions_ssg import DATABASE as db -from webmentions_ssg.models import ReceivedWebmention +from webmentions_ssg.models import ( + ReceivedWebmention, + SentWebmention, + SentWebmentionStatus, + Source, + User, +) -def test_endpoint_only_accepts_post( - client: FlaskClient, -) -> None: +def log_in(app: Flask, client: FlaskClient) -> None: + with app.app_context(): + user = User(username="admin") + + db.session.add(user) + db.session.commit() + + user_id = user.id + + with client.session_transaction() as session: + session["_user_id"] = str(user_id) + session["_fresh"] = True + + +def create_sent_source(app: Flask) -> uuid.UUID: + now = datetime.now(timezone.utc) + + with app.app_context(): + source = Source( + path="blog/example/index.html", + url="https://dennisfink.me/blog/example/", + content_hash="0" * 32, + revision=2, + last_seen_at=now, + revised_at=now, + ) + + source.sent_webmentions.extend( + [ + SentWebmention( + target="https://example.com/b", + active=True, + desired_revision=2, + processed_revision=2, + sent_revision=2, + status=SentWebmentionStatus.SENT, + endpoint="https://example.com/webmention", + response_status=202, + ), + SentWebmention( + target="https://example.com/a", + active=True, + desired_revision=2, + processed_revision=2, + sent_revision=1, + status=SentWebmentionStatus.FAILED, + failure_reason="Webmention endpoint returned HTTP 400", + endpoint="https://example.com/webmention", + response_status=400, + ), + ] + ) + + db.session.add(source) + db.session.commit() + + return source.uuid + + +def test_endpoint_only_accepts_post(client: FlaskClient) -> None: response = client.get("/endpoint") assert response.status_code == 405 -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) -def test_endpoint_returns_form_errors( - client: FlaskClient, -) -> None: +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) +def test_endpoint_returns_form_errors(client: FlaskClient) -> None: response = client.post( "/endpoint", data={ @@ -40,26 +100,15 @@ def test_endpoint_returns_form_errors( assert "target" in errors -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) @patch("webmentions_ssg.views.verify_webmention") def test_endpoint_creates_webmention( - verify_webmention: Mock, - app: Flask, - client: FlaskClient, + verify_webmention: Mock, app: Flask, client: FlaskClient ) -> None: source = "https://source.example/post" target = "https://dennisfink.me/blog/example/" - response = client.post( - "/endpoint", - data={ - "source": source, - "target": target, - }, - ) + response = client.post("/endpoint", data={"source": source, "target": target}) assert response.status_code == 201 @@ -78,29 +127,18 @@ def test_endpoint_creates_webmention( verify_webmention.assert_called_once_with(identifier) -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) @patch("webmentions_ssg.views.verify_webmention") def test_endpoint_is_idempotent( - verify_webmention: Mock, - app: Flask, - client: FlaskClient, + verify_webmention: Mock, app: Flask, client: FlaskClient ) -> None: data = { "source": "https://source.example/post", "target": "https://dennisfink.me/blog/example/", } - first_response = client.post( - "/endpoint", - data=data, - ) - second_response = client.post( - "/endpoint", - data=data, - ) + first_response = client.post("/endpoint", data=data) + second_response = client.post("/endpoint", data=data) assert first_response.status_code == 201 assert second_response.status_code == 201 @@ -120,21 +158,13 @@ def test_endpoint_is_idempotent( identifier = webmention.uuid - assert verify_webmention.call_args_list == [ - call(identifier), - call(identifier), - ] + assert verify_webmention.call_args_list == [call(identifier), call(identifier)] -@patch( - "webmentions_ssg.forms.validators.ensure_public_url", - new=lambda url: None, -) +@patch("webmentions_ssg.forms.validators.is_public_url", new=lambda url: True) @patch("webmentions_ssg.views.verify_webmention") def test_resending_resets_failure_state( - verify_webmention: Mock, - app: Flask, - client: FlaskClient, + verify_webmention: Mock, app: Flask, client: FlaskClient ) -> None: source = "https://source.example/post" target = "https://dennisfink.me/blog/example/" @@ -153,24 +183,72 @@ def test_resending_resets_failure_state( identifier = existing.uuid - response = client.post( - "/endpoint", - data={ - "source": source, - "target": target, - }, - ) + response = client.post("/endpoint", data={"source": source, "target": target}) assert response.status_code == 201 with app.app_context(): - webmention = db.session.get( - ReceivedWebmention, - identifier, - ) + webmention = db.session.get(ReceivedWebmention, identifier) assert webmention is not None assert webmention.status == "received" assert webmention.failure_reason is None verify_webmention.assert_called_once_with(identifier) + + +@pytest.mark.parametrize("path", ["/sent", f"/sent/{uuid.uuid7()}"]) +def test_sent_views_require_login(client: FlaskClient, path: str) -> None: + response = client.get(path) + + assert response.status_code == 302 + assert "/login" in response.headers["Location"] + + +def test_sent_lists_sources(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get("/sent") + + assert response.status_code == 200 + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + assert f"/sent/{identifier}" in html + + +def test_sent_source_lists_webmentions(app: Flask, client: FlaskClient) -> None: + log_in(app, client) + identifier = create_sent_source(app) + + response = client.get(f"/sent/{identifier}") + + assert response.status_code == 200 + + html = response.get_data(as_text=True) + + assert "blog/example/index.html" in html + assert "https://dennisfink.me/blog/example/" in html + + first_target = "https://example.com/a" + second_target = "https://example.com/b" + + assert first_target in html + assert second_target in html + assert html.index(first_target) < html.index(second_target) + + assert "Webmention endpoint returned HTTP 400" in html + assert "https://example.com/webmention" in html + + +def test_sent_source_returns_404_for_unknown_source( + app: Flask, client: FlaskClient +) -> None: + log_in(app, client) + + response = client.get(f"/sent/{uuid.uuid7()}") + + assert response.status_code == 404 @@ -73,6 +73,41 @@ wheels = [ ] [[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] name = "click" version = "8.4.2" source = { registry = "https://pypi.org/simple" } @@ -273,6 +308,19 @@ wheels = [ ] [[package]] +name = "html5lib" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, + { name = "webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" }, +] + +[[package]] name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } @@ -403,6 +451,29 @@ wheels = [ ] [[package]] +name = "mf2py" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "html5lib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/7d/bccfc42706cb24053e7897c33c14e79a8c9c69379d21edfca13ec93ed0ac/mf2py-2.0.1.tar.gz", hash = "sha256:1380924633413b8d72e704b5c86b4382c4b1371699edecc907b01cd21138d7cd", size = 21843, upload-time = "2023-12-08T03:41:59.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/88/b1d83c9e71cbdaefcec38ea350d2bd6360a9d1e030b090ad4b0fcc421ca1/mf2py-2.0.1-py3-none-any.whl", hash = "sha256:092806e17f1a93db4aafa5e8d3c4124b5e42cd89027e2db48a5248ef4eabde03", size = 25767, upload-time = "2023-12-08T03:41:58.443Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] name = "packaging" version = "26.2" source = { registry = "https://pypi.org/simple" } @@ -439,6 +510,19 @@ wheels = [ ] [[package]] +name = "pyright" +version = "1.1.411" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/ab/265f7dc69d28113ebba19092e57b075f41543b2ed048429c5f56e2b88eac/pyright-1.1.411.tar.gz", hash = "sha256:d885a0551f2e763b089a02702174e7f4ba77548cddabc972ab86d1f7f1b0f998", size = 4112861, upload-time = "2026-06-25T02:14:06.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/49/385be530a6a5b78d1cbcd5c2e38debc8959a2fc6bdb716f4e581002979fc/pyright-1.1.411-py3-none-any.whl", hash = "sha256:dc7c72a8e2700c55baa127554040e067041ea53ccfd50bf96308cc4291c7d5d9", size = 6181526, upload-time = "2026-06-25T02:14:04.691Z" }, +] + +[[package]] name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } @@ -478,6 +562,21 @@ wheels = [ ] [[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] name = "rfc3987" version = "1.3.8" source = { registry = "https://pypi.org/simple" } @@ -487,6 +586,15 @@ wheels = [ ] [[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] name = "soupsieve" version = "2.9.1" source = { registry = "https://pypi.org/simple" } @@ -544,6 +652,24 @@ wheels = [ ] [[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "webencodings" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, +] + +[[package]] name = "webmentions-ssg" source = { editable = "." } dependencies = [ @@ -558,11 +684,14 @@ dependencies = [ { name = "flask-wtf" }, { name = "httpx" }, { name = "huey" }, + { name = "mf2py" }, { name = "rfc3987" }, + { name = "xxhash" }, ] [package.dev-dependencies] dev = [ + { name = "pyright" }, { name = "python-dotenv" }, { name = "types-wtforms" }, ] @@ -584,11 +713,14 @@ requires-dist = [ { name = "flask-wtf", specifier = ">=1.3.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "huey", specifier = ">=3.1.0" }, + { name = "mf2py", specifier = ">=2.0.1" }, { name = "rfc3987", specifier = ">=1.3.8" }, + { name = "xxhash", specifier = ">=3.8.1" }, ] [package.metadata.requires-dev] dev = [ + { name = "pyright", specifier = ">=1.1.411" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "types-wtforms", specifier = ">=3.2.1.20260518" }, ] @@ -620,3 +752,54 @@ sdist = { url = "https://files.pythonhosted.org/packages/e9/91/ed9b517da898e3fb7 wheels = [ { url = "https://files.pythonhosted.org/packages/18/76/bb225c8300f3a0ba28e01df51419c6c9574a297c43d71b29048e03b65deb/wtforms-3.2.2-py3-none-any.whl", hash = "sha256:72b90d5d921bd3119252069cf0301e9c13915f9e52792652bc91c5dda4b79e56", size = 158656, upload-time = "2026-05-03T05:53:46.072Z" }, ] + +[[package]] +name = "xxhash" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, +] diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 7c448e9..6996b57 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -14,6 +14,9 @@ class DefaultConfig: LOGFILE_MAX_BYTES = 20_000_000 LOGFILE_BACKUP_COUNT = 10 SQLALCHEMY_DATABASE_URI = "sqlite:///webmentions-ssg.db" + WEBMENTIONS_SSG_SOURCE_DIRECTORY = None + WEBMENTIONS_SSG_SOURCE_BASE_URL = None + WEBMENTIONS_SSG_SCANNER_SCHEDULE = "0 * * * *" class DevelopmentConfig(DefaultConfig): @@ -41,9 +44,7 @@ class TestingConfig(DefaultConfig): HUEY_URL = "memory://" - WEBMENTIONS_SSG_ALLOWED_HOSTNAMES = { - "dennisfink.me", - } + WEBMENTIONS_SSG_ALLOWED_HOSTNAMES = {"dennisfink.me"} WEBMENTIONS_SSG_MAX_REDIRECTS = 20 WEBMENTIONS_SSG_MAX_SOURCE_BYTES = 1_000_000 diff --git a/webmentions_ssg/forms/validators.py b/webmentions_ssg/forms/validators.py index e979644..c4b5eda 100644 --- a/webmentions_ssg/forms/validators.py +++ b/webmentions_ssg/forms/validators.py @@ -3,11 +3,7 @@ from urllib.parse import urlsplit from flask import current_app from wtforms import ValidationError -from ..url_security import ( - AddressResolutionError, - NonPublicAddressError, - ensure_public_url, -) +from ..url_security import AddressResolutionError, is_public_url class NotEqualTo: @@ -73,15 +69,13 @@ class PublicURL: self.message = message def __call__(self, form, field): - try: - ensure_public_url(field.data) - except ( - AddressResolutionError, - NonPublicAddressError, - ) as exc: - message = self.message + message = self.message - if message is None: - message = field.gettext("URL must resolve to a public address.") + if message is None: + message = field.gettext("URL must resolve to a public address.") + try: + if not is_public_url(field.data): + raise ValidationError(message) + except (AddressResolutionError, ValueError) as exc: raise ValidationError(message) from exc diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 8c378aa..02c42ef 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -1,89 +1,189 @@ +from __future__ import annotations + import uuid from datetime import datetime, timezone +from enum import StrEnum from typing import Optional from flask_login import UserMixin -from sqlalchemy import DateTime, String, Text, UniqueConstraint, Uuid -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship from werkzeug.security import check_password_hash, generate_password_hash from . import Base +class SentWebmentionStatus(StrEnum): + SENT = "sent" + UNSUPPORTED = "unsupported" + FAILED = "failed" + + class User(UserMixin, Base): __tablename__ = "users" id: Mapped[int] = mapped_column(primary_key=True) - username: Mapped[str] = mapped_column( - String(64), - index=True, - unique=True, - ) + username: Mapped[str] = mapped_column(String(64), index=True, unique=True) password_hash: Mapped[Optional[str]] = mapped_column(String(256)) - def __repr__(self): + def __repr__(self) -> str: return f"<User {self.username}>" - def set_password(self, password): + def set_password(self, password: str) -> None: self.password_hash = generate_password_hash(password) - def check_password(self, password): - return check_password_hash( - self.password_hash, - password, - ) + def check_password(self, password: str) -> bool: + return check_password_hash(self.password_hash, password) class ReceivedWebmention(Base): __tablename__ = "received_webmentions" - uuid: Mapped[uuid.UUID] = mapped_column( - Uuid(as_uuid=True), - primary_key=True, + uuid: Mapped[uuid.UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True) + + source: Mapped[str] = mapped_column(Text(), nullable=False) + target: Mapped[str] = mapped_column(Text(), nullable=False) + + status: Mapped[str] = mapped_column(Text(), nullable=False, default="received") + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None ) - source: Mapped[str] = mapped_column( - Text(), + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), nullable=False, ) - target: Mapped[str] = mapped_column( - Text(), - nullable=False, + + __table_args__ = ( + UniqueConstraint("source", "target", name="uq_webmention_source_target"), + ) + + @property + def verified(self) -> bool: + return self.status == "verified" + + @property + def created_at(self) -> datetime: + return uuid7_to_datetime(self.uuid) + + +class Source(Base): + __tablename__ = "sources" + + uuid: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid.uuid7 + ) + + path: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + url: Mapped[str] = mapped_column(Text(), nullable=False, unique=True) + + content_hash: Mapped[str] = mapped_column(String(32), nullable=False) + revision: Mapped[int] = mapped_column(Integer(), nullable=False, default=1) + + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + revised_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + deleted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + + sent_webmentions: Mapped[list[SentWebmention]] = relationship( + back_populates="source", cascade="all, delete-orphan", lazy="selectin" + ) + + @property + def created_at(self) -> datetime: + return uuid7_to_datetime(self.uuid) + + +class SentWebmention(Base): + __tablename__ = "sent_webmentions" + + uuid: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid.uuid7 ) - status: Mapped[str] = mapped_column( - Text(), + source_id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("sources.uuid", ondelete="CASCADE"), nullable=False, - default="received", + index=True, ) - failure_reason: Mapped[str | None] = mapped_column( - Text(), + + target: Mapped[str] = mapped_column(Text(), nullable=False) + + active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + + desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) + processed_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + sent_revision: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + + status: Mapped[SentWebmentionStatus | None] = mapped_column( + Enum( + SentWebmentionStatus, + name="sent_webmention_status", + values_callable=lambda enum_type: [member.value for member in enum_type], + native_enum=False, + ), nullable=True, default=None, ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - onupdate=lambda: datetime.now(timezone.utc), - nullable=False, + failure_reason: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + + endpoint: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + response_status: Mapped[int | None] = mapped_column( + Integer(), nullable=True, default=None + ) + status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) + + last_attempted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None + ) + last_sent_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, default=None ) + source: Mapped[Source] = relationship(back_populates="sent_webmentions") + __table_args__ = ( UniqueConstraint( - "source", - "target", - name="uq_webmention_source_target", + "source_id", "target", name="uq_sent_webmention_source_target" ), ) @property - def verified(self) -> bool: - return self.status == "verified" + def pending(self) -> bool: + return ( + self.processed_revision is None + or self.processed_revision < self.desired_revision + ) @property def created_at(self) -> datetime: - return datetime.fromtimestamp( - self.uuid.time / 1000, - tz=timezone.utc, - ) + return uuid7_to_datetime(self.uuid) + + +def uuid7_to_datetime(identifier: uuid.UUID) -> datetime: + return datetime.fromtimestamp(identifier.time / 1000, tz=timezone.utc) diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 5b85d0e..e53fce5 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -1,8 +1,19 @@ +from huey import Huey, crontab + from .. import HUEY, create_app -app = create_app() -# Import the tasks so they are registered with the initialized Huey instance. -from . import receiver # noqa: E402, F401 +def create_consumer() -> Huey: + app = create_app() + + from . import receiver, scanner, sender # noqa: E402, F401 + + schedule = crontab( + *app.config["WEBMENTIONS_SSG_SCANNER_SCHEDULE"].split(), strict=True + ) + HUEY.periodic_task(schedule)(scanner.scan_sources) + + return HUEY.huey + -huey = HUEY.huey +huey = create_consumer() diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index ebdaa76..15bc669 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -27,10 +27,7 @@ class Huey: huey_class, storage_kwargs = self.backend_from_url(url) self.app = app - self._huey = huey_class( - **config, - **storage_kwargs, - ) + self._huey = huey_class(**config, **storage_kwargs) app.extensions["huey"] = self diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index 9475866..39def73 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -10,11 +10,7 @@ from .. import APP_NAME, VERSION from .. import DATABASE as db from .. import HUEY as huey from ..models import ReceivedWebmention -from ..url_security import ( - AddressResolutionError, - NonPublicAddressError, - ensure_public_url, -) +from ..url_security import AddressResolutionError, is_public_url class VerificationError(Exception): @@ -46,33 +42,20 @@ HTML_URL_ATTRIBUTES = { "track", "video", }, - "cite": { - "blockquote", - "del", - "ins", - "q", - }, + "cite": {"blockquote", "del", "ins", "q"}, } -def html_mentions_target( - body: bytes, - source_url: str, - target_url: str, -) -> bool: +def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: """Check valid HTML URL attributes for the exact target URL.""" document = BeautifulSoup(body, "html.parser") base_url = source_url if (base_element := document.select_one("base[href]")) is not None and isinstance( - base_href := base_element.get("href"), - str, + base_href := base_element.get("href"), str ): - base_url = urljoin( - source_url, - base_href.strip(), - ) + base_url = urljoin(source_url, base_href.strip()) for attribute, selectors in HTML_URL_ATTRIBUTES.items(): selector = ", ".join( @@ -85,19 +68,10 @@ def html_mentions_target( ) for element in document.select(selector): - if not isinstance( - reference := element.get(attribute), - str, - ): + if not isinstance(reference := element.get(attribute), str): continue - if ( - urljoin( - base_url, - reference.strip(), - ) - == target_url - ): + if urljoin(base_url, reference.strip()) == target_url: return True return False @@ -111,9 +85,10 @@ def text_mentions_target(body: str, target_url: str) -> bool: def ensure_public_request(request: httpx.Request) -> None: """Prevent requests to non-public network addresses.""" try: - ensure_public_url(str(request.url)) - except NonPublicAddressError as exc: - raise VerificationError("Source resolves to a non-public address") from exc + if not is_public_url(str(request.url)): + raise VerificationError("Source resolves to a non-public address") + except ValueError as exc: + raise VerificationError("Source URL has no hostname") from exc except AddressResolutionError as exc: raise TemporaryFetchError("Source hostname could not be resolved") from exc @@ -132,9 +107,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: follow_redirects=True, max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), trust_env=False, - event_hooks={ - "request": [ensure_public_request], - }, + event_hooks={"request": [ensure_public_request]}, ) as client: with client.stream("GET", source_url) as response: match response.status_code: @@ -149,8 +122,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: raise VerificationError(f"Source returned HTTP {status}") max_source_bytes = current_app.config.get( - "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", - 1_000_000, + "WEBMENTIONS_SSG_MAX_SOURCE_BYTES", 1_000_000 ) if (content_length := response.headers.get("Content-Length")) is not None: @@ -185,14 +157,10 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: case "text/plain": try: decoded_body = body.decode( - response.encoding or "utf-8", - errors="replace", + response.encoding or "utf-8", errors="replace" ) except LookupError: - decoded_body = body.decode( - "utf-8", - errors="replace", - ) + decoded_body = body.decode("utf-8", errors="replace") return text_mentions_target(decoded_body, target_url) case _: raise VerificationError( @@ -208,8 +176,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: if webmention is None: current_app.logger.warning( - "Cannot verify unknown ReceivedWebmention %s", - webmention_uuid, + "Cannot verify unknown ReceivedWebmention %s", webmention_uuid ) return diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py new file mode 100644 index 0000000..f80e5f4 --- /dev/null +++ b/webmentions_ssg/tasks/scanner.py @@ -0,0 +1,397 @@ +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import datetime, timezone +from fnmatch import fnmatchcase +from itertools import chain +from pathlib import Path +from typing import Any +from urllib.parse import quote, urldefrag, urljoin, urlsplit + +import mf2py +import sqlalchemy as sa +import xxhash +from bs4 import BeautifulSoup, Tag +from flask import current_app +from sqlalchemy.orm import selectinload + +from .. import DATABASE as db +from .. import HUEY as huey +from ..models import SentWebmention, Source +from ..url_security import is_http_url +from .sender import send_webmention + +REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") + + +class SourceScanError(Exception): + """A source document cannot safely be processed.""" + + +@dataclass(frozen=True) +class ScannedSource: + path: str + url: str + content_hash: str + targets: frozenset[str] + + +def source_url_for_path(relative_path: Path, base_url: str) -> str: + """Derive the public source URL from its relative filesystem path.""" + directory = relative_path.parent.as_posix() + + return urljoin(base_url, f"{quote(directory, safe='/')}/") + + +def canonical_url( + document: BeautifulSoup, *, relative_path: Path, base_url: str | None +) -> str | None: + """Return the canonical URL declared by the document.""" + if (link := document.select_one('link[rel~="canonical"][href]')) is not None: + href = link.get("href") + + if not isinstance(href, str) or not (href := href.strip()): + raise SourceScanError("Document contains an empty canonical URL") + + if is_http_url(href): + return href + + if base_url is None: + raise SourceScanError( + "Document contains a relative canonical URL, but " + "WEBMENTIONS_SSG_SOURCE_BASE_URL is not configured" + ) + + resolved = urljoin(source_url_for_path(relative_path, base_url), href) + + if not is_http_url(resolved): + raise SourceScanError( + f"Canonical URL is not a valid HTTP or HTTPS URL: {href!r}" + ) + + return resolved + + +def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: + """Yield URL values from a microformats property.""" + properties = entry.get("properties") + + if not isinstance(properties, dict): + return + + values = properties.get(property_name) + + if not isinstance(values, list): + return + + yield from (value for value in values if isinstance(value, str)) + + +def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: + """Parse the source h-entry with mf2py.""" + parsed = mf2py.parse(doc=str(element), url=base_url) + + if not isinstance(parsed, dict): + raise SourceScanError("Microformats parser did not return a document object") + + items = parsed.get("items") + + if not isinstance(items, list) or len(items) != 1: + raise SourceScanError("Expected exactly one parsed h-entry") + + entry = items[0] + + if not isinstance(entry, dict): + raise SourceScanError("Parsed h-entry is not an object") + + types = entry.get("type") + + if not isinstance(types, list) or "h-entry" not in types: + raise SourceScanError("Parsed microformats item is not an h-entry") + + return entry + + +def primary_entry( + document: BeautifulSoup, source_url: str +) -> tuple[Tag, dict[str, Any]]: + """Return and validate the source h-entry.""" + entries = document.find_all(class_="h-entry") + + if len(entries) != 1: + raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") + + entry_element = entries[0] + mf2_entry = parse_entry(entry_element, source_url) + + urls = tuple(property_urls(mf2_entry, "url")) + + if source_url not in urls: + raise SourceScanError( + f"The h-entry u-url does not match the source URL {source_url!r}: {urls!r}" + ) + + return entry_element, mf2_entry + + +def content_element(entry: Tag) -> Tag: + """Return the source h-entry's e-content element.""" + contents = entry.find_all(class_="e-content") + + if len(contents) != 1: + raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") + + return contents[0] + + +def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: + """Resolve and validate a possible Webmention target URL.""" + value = value.strip() + + if not value: + return None + + target = urljoin(base_url, value) + + if not is_http_url(target): + return None + + if urldefrag(target)[0] == urldefrag(source_url)[0]: + return None + + return target + + +def iter_targets( + content: Tag, + mf2_entry: dict[str, Any], + *, + base_url: str, + source_url: str, + ignored_hostnames: tuple[str, ...] = (), +) -> Iterator[str]: + """Yield outgoing Webmention targets from an h-entry.""" + hrefs = ( + href + for element in content.find_all("a", href=True) + if isinstance(href := element.get("href"), str) + ) + + reactions = chain.from_iterable( + property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + ) + + for value in chain(hrefs, reactions): + target = normalize_target(value, base_url=base_url, source_url=source_url) + if target is not None: + hostname = urlsplit(target).hostname or "" + if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): + yield target + + +def scan_source_file( + path: Path, + *, + root: Path, + base_url: str | None, + ignored_hostnames: tuple[str, ...] = (), +) -> ScannedSource: + """Parse one generated source document.""" + document = BeautifulSoup(path.read_bytes(), "html5lib") + + relative_path = path.relative_to(root) + + source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + + if source_url is None: + if base_url is None: + raise SourceScanError( + "Document has no canonical URL and " + "WEBMENTIONS_SSG_SOURCE_BASE_URL is not configured" + ) + + source_url = source_url_for_path(relative_path, base_url) + + entry_element, mf2_entry = primary_entry(document, source_url) + + content = content_element(entry_element) + + return ScannedSource( + path=relative_path.as_posix(), + url=source_url, + content_hash=xxhash.xxh3_128_hexdigest(str(entry_element).encode("utf-8")), + targets=frozenset( + iter_targets( + content, + mf2_entry, + base_url=source_url, + source_url=source_url, + ignored_hostnames=ignored_hostnames, + ) + ), + ) + + +def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: + """Create a source from a newly discovered document.""" + source = Source( + path=scanned.path, + url=scanned.url, + content_hash=scanned.content_hash, + revision=1, + last_seen_at=scan_time, + revised_at=scan_time, + deleted_at=None, + ) + + for target in scanned.targets: + source.sent_webmentions.append( + SentWebmention(target=target, active=True, desired_revision=1) + ) + + return source + + +def update_source( + source: Source, scanned: ScannedSource, scan_time: datetime +) -> Source: + """Update a source from a newly scanned revision.""" + if source.url != scanned.url: + raise SourceScanError( + f"Source path {source.path!r} changed public URL " + f"from {source.url!r} to {scanned.url!r}" + ) + + source.last_seen_at = scan_time + + if source.deleted_at is None and source.content_hash == scanned.content_hash: + return source + + source.revision += 1 + source.content_hash = scanned.content_hash + source.revised_at = scan_time + source.deleted_at = None + + existing = {webmention.target: webmention for webmention in source.sent_webmentions} + + for webmention in existing.values(): + webmention.active = webmention.target in scanned.targets + webmention.desired_revision = source.revision + + if not webmention.active and webmention.sent_revision is None: + webmention.processed_revision = source.revision + + for target in scanned.targets: + if target not in existing: + source.sent_webmentions.append( + SentWebmention( + target=target, active=True, desired_revision=source.revision + ) + ) + + return source + + +@huey.lock_task("scan-webmention-sources") +def scan_sources() -> None: + """Scan generated source documents and queue pending Webmentions.""" + directory = current_app.config.get("WEBMENTIONS_SSG_SOURCE_DIRECTORY") + + if directory is None: + return + + root = Path(directory).expanduser().resolve(strict=True) + + if not root.is_dir(): + raise RuntimeError(f"Source directory is not a directory: {root}") + + base_url = current_app.config.get("WEBMENTIONS_SSG_SOURCE_BASE_URL") + + if base_url is not None: + if not is_http_url(base_url): + raise RuntimeError( + "WEBMENTIONS_SSG_SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL" + ) + base_url = f"{base_url.rstrip('/')}/" + + ignored_hostnames = tuple( + pattern.lower().rstrip(".") + for pattern in current_app.config.get("WEBMENTIONS_SSG_IGNORED_HOSTNAMES", ()) + ) + + sources_by_path = { + source.path: source + for source in db.session.scalars( + sa.select(Source).options(selectinload(Source.sent_webmentions)) + ) + } + + seen_paths: set[str] = set() + scanned_count = 0 + scan_time = datetime.now(timezone.utc) + + for index_path in root.rglob("index.html"): + relative_path = index_path.relative_to(root) + + if relative_path.parent == Path("."): + continue + + seen_paths.add(relative_path.as_posix()) + + try: + scanned = scan_source_file( + index_path, + root=root, + base_url=base_url, + ignored_hostnames=ignored_hostnames, + ) + + if (source := sources_by_path.get(scanned.path)) is None: + source = create_source(scanned, scan_time) + db.session.add(source) + else: + source = update_source(source, scanned, scan_time) + + sources_by_path[source.path] = source + scanned_count += 1 + + except (OSError, SourceScanError) as exc: + current_app.logger.warning("Could not scan source %s: %s", index_path, exc) + + for path, source in sources_by_path.items(): + if source.deleted_at is not None or path in seen_paths: + continue + + source.revision += 1 + source.revised_at = scan_time + source.deleted_at = scan_time + + for webmention in source.sent_webmentions: + webmention.active = False + webmention.desired_revision = source.revision + + if webmention.sent_revision is None: + webmention.processed_revision = source.revision + + # Persist the desired state before queueing any work. + db.session.commit() + + pending_count = 0 + + for identifier in db.session.scalars( + sa.select(SentWebmention.uuid) + .where( + sa.or_( + SentWebmention.processed_revision.is_(None), + SentWebmention.processed_revision < SentWebmention.desired_revision, + ) + ) + .order_by(SentWebmention.uuid) + ): + send_webmention(identifier) + pending_count += 1 + + current_app.logger.info( + "Webmention source scan complete: %d sources scanned, %d pending sends", + scanned_count, + pending_count, + ) diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py new file mode 100644 index 0000000..fa9b624 --- /dev/null +++ b/webmentions_ssg/tasks/sender.py @@ -0,0 +1,315 @@ +import re +import uuid +from datetime import datetime, timezone +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup +from flask import current_app + +from .. import APP_NAME, VERSION +from .. import DATABASE as db +from .. import HUEY as huey +from ..models import SentWebmention, SentWebmentionStatus +from ..url_security import AddressResolutionError, is_http_url, is_public_url + +LINK_SPLIT = re.compile(r",\s*(?=<)") + +DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} + + +class SenderError(Exception): + """A Webmention cannot be sent.""" + + +class TemporarySenderError(SenderError): + """A Webmention send failed for a potentially temporary reason.""" + + +class PermanentSenderError(SenderError): + """A Webmention send failed permanently for this source revision.""" + + +def temporary_http_status(status: int) -> bool: + """Return whether an HTTP status should be retried.""" + return status in {408, 425, 429} or 500 <= status <= 599 + + +def ensure_public_request(request: httpx.Request) -> None: + """Prevent requests to non-public network addresses.""" + try: + if not is_public_url(str(request.url)): + raise PermanentSenderError("Request resolves to a non-public address") + except AddressResolutionError as exc: + raise TemporarySenderError("Request hostname could not be resolved") from exc + + +def resolve_endpoint(response: httpx.Response, href: str) -> str: + """Resolve and validate a discovered Webmention endpoint.""" + endpoint = urljoin(str(response.url), href.strip()) + + if not is_http_url(endpoint): + raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") + + return endpoint + + +def parse_link_value(value: str) -> tuple[str, set[str]] | None: + """Parse a Link header value into its target and relations.""" + value = value.strip() + + if not value.startswith("<"): + return None + + href, separator, parameters = value[1:].partition(">") + + if not separator: + return None + + relations: set[str] = set() + + for parameter in parameters.split(";"): + name, separator, value = parameter.partition("=") + + if separator and name.strip().lower() == "rel": + relations.update( + relation.lower() for relation in value.strip().strip("\"'").split() + ) + + return href, relations + + +def endpoint_from_headers(response: httpx.Response) -> str | None: + """Return the first Webmention endpoint advertised by HTTP Link.""" + for header in response.headers.get_list("Link"): + for value in LINK_SPLIT.split(header): + if (link := parse_link_value(value)) is None: + continue + + href, relations = link + + if "webmention" in relations: + return resolve_endpoint(response, href) + + return None + + +def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: + """Return the first HTML Webmention endpoint in document order.""" + document = BeautifulSoup(body, "html5lib") + + for element in document.find_all(["link", "a"], href=True): + relations = element.get("rel") + + if isinstance(relations, str): + relations = relations.split() + + if not relations: + continue + + if not any( + isinstance(relation, str) and relation.lower() == "webmention" + for relation in relations + ): + continue + + href = element.get("href") + + if not isinstance(href, str): + continue + + return resolve_endpoint(response, href) + + return None + + +def read_target_body(response: httpx.Response) -> bytes: + """Read a target document up to the configured size limit.""" + max_bytes = current_app.config.get("WEBMENTIONS_SSG_MAX_TARGET_BYTES", 1_000_000) + + if (content_length := response.headers.get("Content-Length")) is not None: + try: + if int(content_length) > max_bytes: + raise PermanentSenderError("Target document is too large") + except ValueError: + pass + + body = bytearray() + + for chunk in response.iter_bytes(chunk_size=64 * 1024): + body.extend(chunk) + + if len(body) > max_bytes: + raise PermanentSenderError("Target document is too large") + + return bytes(body) + + +def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: + """Discover the Webmention endpoint advertised by a target.""" + head_response = client.head(target, headers=DISCOVERY_HEADERS) + + if endpoint := endpoint_from_headers(head_response): + return endpoint + + with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: + if endpoint := endpoint_from_headers(response): + return endpoint + + if temporary_http_status(response.status_code): + raise TemporarySenderError(f"Target returned HTTP {response.status_code}") + + if not response.is_success: + raise PermanentSenderError(f"Target returned HTTP {response.status_code}") + + media_type = ( + response.headers.get("Content-Type", "").partition(";")[0].strip().lower() + ) + + if media_type not in {"text/html", "application/xhtml+xml"}: + return None + + return endpoint_from_html(response, read_target_body(response)) + + +def post_webmention( + client: httpx.Client, *, endpoint: str, source: str, target: str +) -> tuple[int, str | None]: + """POST a Webmention and return its status code and status URL.""" + with client.stream( + "POST", endpoint, data={"source": source, "target": target} + ) as response: + status_url = None + + if response.status_code == 201 and ( + location := response.headers.get("Location") + ): + status_url = urljoin(str(response.url), location) + + return (response.status_code, status_url) + + +def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: + """Return whether an attempt still represents the desired revision.""" + db.session.refresh(webmention) + + return webmention.desired_revision == revision and ( + webmention.processed_revision is None + or webmention.processed_revision < revision + ) + + +@huey.task(retries=2, retry_delay=50) +def send_webmention(webmention_uuid: uuid.UUID) -> None: + """Discover a receiver endpoint and send one Webmention.""" + webmention = db.session.get(SentWebmention, webmention_uuid) + + if webmention is None: + current_app.logger.warning( + "Cannot send unknown SentWebmention %s", webmention_uuid + ) + return + + if ( + webmention.processed_revision is not None + and webmention.processed_revision >= webmention.desired_revision + ): + return + + revision = webmention.desired_revision + source = webmention.source.url + target = webmention.target + + webmention.last_attempted_at = datetime.now(timezone.utc) + webmention.endpoint = None + webmention.response_status = None + webmention.status_url = None + + db.session.commit() + + endpoint: str | None = None + response_status: int | None = None + + try: + with httpx.Client( + headers={"User-Agent": (f"{APP_NAME}/{VERSION} SentWebmention")}, + timeout=httpx.Timeout( + current_app.config.get("WEBMENTIONS_SSG_REQUEST_TIMEOUT", 5.0) + ), + follow_redirects=True, + max_redirects=current_app.config.get("WEBMENTIONS_SSG_MAX_REDIRECTS", 20), + trust_env=False, + event_hooks={"request": [ensure_public_request]}, + ) as client: + endpoint = discover_webmention_endpoint(client, target) + + if endpoint is None: + if not attempt_is_current(webmention, revision): + return + + webmention.processed_revision = revision + webmention.status = SentWebmentionStatus.UNSUPPORTED + webmention.failure_reason = "No Webmention endpoint discovered" + webmention.endpoint = None + webmention.response_status = None + + db.session.commit() + return + + (response_status, status_url) = post_webmention( + client, endpoint=endpoint, source=source, target=target + ) + + if not (200 <= response_status <= 299): + message = f"Webmention endpoint returned HTTP {response_status}" + + if temporary_http_status(response_status): + raise TemporarySenderError(message) + + raise PermanentSenderError(message) + + except (TemporarySenderError, httpx.RequestError) as exc: + if not attempt_is_current(webmention, revision): + return + + webmention.status = SentWebmentionStatus.FAILED + webmention.failure_reason = str(exc) or "Webmention request failed" + webmention.endpoint = endpoint + webmention.response_status = response_status + + db.session.commit() + + # Leave processed_revision unchanged. Huey may retry the + # attempt, and the periodic scanner will also rediscover + # outstanding work. + raise + + except PermanentSenderError as exc: + if not attempt_is_current(webmention, revision): + return + + webmention.processed_revision = revision + webmention.status = SentWebmentionStatus.FAILED + webmention.failure_reason = str(exc) + webmention.endpoint = endpoint + webmention.response_status = response_status + + db.session.commit() + return + + if not attempt_is_current(webmention, revision): + return + + webmention.processed_revision = revision + webmention.sent_revision = revision + + webmention.status = SentWebmentionStatus.SENT + webmention.failure_reason = None + + webmention.endpoint = endpoint + webmention.response_status = response_status + webmention.status_url = status_url + + webmention.last_sent_at = datetime.now(timezone.utc) + + db.session.commit() diff --git a/webmentions_ssg/templates/base.html b/webmentions_ssg/templates/base.html index 237a437..eb11b8e 100644 --- a/webmentions_ssg/templates/base.html +++ b/webmentions_ssg/templates/base.html @@ -22,8 +22,9 @@ {% if current_user.is_authenticated %} <div class="navbar-nav flex-row gap-3"> <a class="nav-link{% if request.endpoint == 'root.received' %} active{% endif %}" - href="{{ url_for("root.received") }}">Received</a> - <span class="nav-link disabled">Sent</span> + href='{{ url_for("root.received") }}'>Received</a> + <a class="nav-link{% if request.endpoint == 'root.sent' %} active{% endif %}" + href='{{ url_for("root.sent") }}'>Sent</a> <a class="nav-link" href="{{ url_for("root.logout") }}">Logout</a> </div> {% endif %} diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index 64f225e..a44b706 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -3,7 +3,7 @@ {% from "bootstrap5/pagination.html" import render_pagination %} {% block content %} <div class="d-flex justify-content-between align-items-center mb-3"> - <h1 class="h3 mb-0">Received Webmentions</h1> + <h1 class="mb-0">Received Webmentions</h1> <span class="text-body-secondary">{{ webmentions.total }} total</span> </div> {% if webmentions.items %} diff --git a/webmentions_ssg/templates/sent.html b/webmentions_ssg/templates/sent.html new file mode 100644 index 0000000..69a91d8 --- /dev/null +++ b/webmentions_ssg/templates/sent.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{% from "bootstrap5/pagination.html" import render_pagination %} +{% block content %} + <div class="d-flex justify-content-between align-items-center mb-3"> + <h1 class="mb-0">Sources</h1> + <span class="text-body-secondary">{{ sources.total }} sources</span> + </div> + {% if sources.items %} + <div class="table-responsive"> + <table class="table table-bordered table-striped table-hover align-middle"> + <thead> + <tr> + <th scope="col">Path</th> + <th scope="col">URL</th> + <th scope="col">Webmentions</th> + <th scope="col">Revision</th> + <th scope="col">Created</th> + <th scope="col">Last seen</th> + <th scope="col">Status</th> + </tr> + </thead> + <tbody> + {% for source in sources %} + <tr> + <td class="text-nowrap"> + <a href="{{ url_for('root.sent_source', identifier=source.uuid) }}">{{ source.path }}</a> + </td> + <td class="text-break"> + <a href="{{ source.url }}">{{ source.url }}</a> + </td> + <td>{{ source.sent_webmentions | length }}</td> + <td>{{ source.revision }}</td> + <td class="text-nowrap"> + <time datetime="{{ source.created_at.isoformat() }}"> + {{ source.created_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + </td> + <td class="text-nowrap"> + <time datetime="{{ source.last_seen_at.isoformat() }}"> + {{ source.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + </td> + <td> + {% if source.deleted_at is none %} + <span class="badge text-bg-success">Active</span> + {% else %} + <span class="badge text-bg-danger">Deleted</span> + {% endif %} + </td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% if sources.pages > 1 %} + <div class="mt-3">{{ render_pagination(sources) }}</div> + {% endif %} + {% else %} + <p class="text-body-secondary">No Webmention sources have been discovered yet.</p> + {% endif %} +{% endblock %} diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html new file mode 100644 index 0000000..6d397f2 --- /dev/null +++ b/webmentions_ssg/templates/sent_source.html @@ -0,0 +1,155 @@ +{% extends "base.html" %} +{% block content %} + <div class="mb-3"> + <a href="{{ url_for("root.sent") }}">← Back to sources</a> + </div> + <div class="d-flex justify-content-between align-items-start mb-3"> + <h1 class="mb-0">Source Details</h1> + </div> + <div class="d-flex justify-content-between align-items-start mb-3"> + <dl class="row"> + <dt class="col-sm-2">UUID</dt> + <dd class="col-sm-4"> + {{ source.uuid }} + </dd> + <dt class="col-sm-2">Hash</dt> + <dd class="col-sm-4"> + {{ source.content_hash }} + </dd> + <dt class="col-sm-2">Path</dt> + <dd class="col-sm-4"> + {{ source.path }} + </dd> + <dt class="col-sm-2">URL</dt> + <dd class="col-sm-4"> + <a href="{{ source.url }}" class="text-break">{{ source.url }}</a> + </dd> + <dt class="col-sm-2">Created at</dt> + <dd class="col-sm-4"> + <time datetime="{{ source.created_at.isoformat() }}"> + {{ source.created_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + </dd> + <dt class="col-sm-2">Last seen at</dt> + <dd class="col-sm-4"> + <time datetime="{{ source.last_seen_at.isoformat() }}"> + {{ source.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + </dd> + <dt class="col-sm-2">Revised at</dt> + <dd class="col-sm-4"> + <time datetime="{{ source.revised_at.isoformat() }}"> + {{ source.revised_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + </dd> + <dt class="col-sm-2">Deleted at</dt> + <dd class="col-sm-4"> + {% if source.deleted_at is none %} + N/A + {% else %} + <time datetime="{{ source.deleted_at.isoformat() }}"> + {{ source.deleted_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + {% endif %} + </dd> + <dt class="col-sm-2">Revision</dt> + <dd class="col-sm-4"> + {{ source.revision }} + </dd> + </dl> + </div> + <div class="d-flex justify-content-between align-items-start mb-3"> + <h2 class="mb-0">Sent Webmentions</h2> + <span class="text-body-secondary">{{ webmentions | length }} webmentions</span> + </div> + {% if webmentions %} + <div class="table-responsive"> + <table class="table table-bordered table-striped table-hover align-middle"> + <thead> + <tr> + <th scope="col">Target</th> + <th scope="col">Status</th> + <th scope="col">Active</th> + <th scope="col">Desired revision</th> + <th scope="col">Processed revision</th> + <th scope="col">Sent revision</th> + <th scope="col">Failure Reason</th> + <th scope="col">Endpoint</th> + <th scope="col">Response Status</th> + <th scope="col">Status URL</th> + <th scope="col">Last Attempted at</th> + <th scope="col">Last Sent at</th> + </tr> + </thead> + <tbody> + {% for webmention in webmentions %} + <tr> + <td class="text-break"> + <a href="{{ webmention.target }}">{{ webmention.target }}</a> + </td> + <td>{{ webmention.status.value }}</td> + <td> + {% if webmention.active %} + <span class="badge text-bg-success">Active</span> + {% else %} + <span class="badge text-bg-secondary">Removed</span> + {% endif %} + </td> + <td>{{ webmention.desired_revision }}</td> + <td> + {{ webmention.processed_revision + if webmention.processed_revision is not none + else "—" }} + </td> + <td> + {{ webmention.sent_revision + if webmention.sent_revision is not none + else "—" }} + </td> + <td> + {{ webmention.failure_reason + if webmention.failure_reason is not none + else "—" }} + </td> + <td> + {{ webmention.endpoint + if webmention.endpoint is not none + else "—" }} + </td> + <td> + {{ webmention.response_status + if webmention.response_status is not none + else "—" }} + </td> + <td> + {{ webmention.status_url + if webmention.status_url is not none + else "—" }} + </td> + <td> + {% if webmention.last_attempted_at is not none %} + <time datetime="{{ webmention.last_attempted_at.isoformat() }}"> + {{ webmention.last_attempted_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + {% else %} + — + {% endif %} + </td> + <td> + {% if webmention.last_sent_at is not none %} + <time datetime="{{ webmention.last_sent_at.isoformat() }}"> + {{ webmention.last_sent_at.strftime("%Y-%m-%d %H:%M:%S") }} + </time> + {% else %} + — + {% endif %} + </td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% else %} + <p class="text-body-secondary">No Webmentions have been discovered for this source.</p> + {% endif %} +{% endblock %} diff --git a/webmentions_ssg/url_security.py b/webmentions_ssg/url_security.py index ce5d700..51c037a 100644 --- a/webmentions_ssg/url_security.py +++ b/webmentions_ssg/url_security.py @@ -9,15 +9,11 @@ class AddressResolutionError(Exception): pass -class NonPublicAddressError(Exception): - pass - - -def ensure_public_url(url: str) -> None: +def is_public_url(url: str) -> bool: hostname = urlsplit(url).hostname if hostname is None: - raise NonPublicAddressError("URL has no hostname") + raise ValueError("No hostname was specified") try: answers = dns.resolver.resolve_name(hostname) @@ -25,13 +21,19 @@ def ensure_public_url(url: str) -> None: raise AddressResolutionError( f"Could not resolve hostname {hostname!r}" ) from exc + else: + for address in answers.addresses(): + if not ipaddress.ip_address(address).is_global: + return False - addresses = {ipaddress.ip_address(address) for address in answers.addresses()} + return True - if not addresses: - raise AddressResolutionError(f"Hostname {hostname!r} did not resolve") - if any(not address.is_global for address in addresses): - raise NonPublicAddressError( - f"Hostname {hostname!r} resolves to a non-public address" - ) +def is_http_url(url: str) -> bool: + """Return whether a URL is an absolute HTTP or HTTPS URL.""" + try: + parsed = urlsplit(url) + except ValueError: + return False + + return parsed.scheme.lower() in {"http", "https"} and parsed.hostname is not None diff --git a/webmentions_ssg/views.py b/webmentions_ssg/views.py index b52ae07..eeea765 100644 --- a/webmentions_ssg/views.py +++ b/webmentions_ssg/views.py @@ -16,10 +16,11 @@ from flask import ( ) from flask.typing import ResponseReturnValue from flask_login import current_user, login_required, login_user, logout_user +from sqlalchemy.orm import selectinload from . import CSRF, forms from . import DATABASE as db -from .models import ReceivedWebmention, User +from .models import ReceivedWebmention, Source, User from .tasks.receiver import verify_webmention root_page = Blueprint("root", __name__) @@ -80,18 +81,13 @@ def received() -> ResponseReturnValue: @root_page.post("/received/<uuid:identifier>/delete") @login_required -def delete_received_webmention( - identifier: uuid.UUID, -) -> ResponseReturnValue: +def delete_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: form = forms.AdminActionForm() if not form.validate_on_submit(): return abort(400) - webmention = db.session.get( - ReceivedWebmention, - identifier, - ) + webmention = db.session.get(ReceivedWebmention, identifier) if webmention is None: return abort(404) @@ -102,27 +98,19 @@ def delete_received_webmention( flash(f"Webmention {webmention.uuid} deleted.", "success") return redirect( - url_for( - "root.received", - page=request.args.get("page", 1, type=int), - ) + url_for("root.received", page=request.args.get("page", 1, type=int)) ) @root_page.post("/received/<uuid:identifier>/reverify") @login_required -def reverify_received_webmention( - identifier: uuid.UUID, -) -> ResponseReturnValue: +def reverify_received_webmention(identifier: uuid.UUID) -> ResponseReturnValue: form = forms.AdminActionForm() if not form.validate_on_submit(): return abort(400) - webmention = db.session.get( - ReceivedWebmention, - identifier, - ) + webmention = db.session.get(ReceivedWebmention, identifier) if webmention is None: return abort(404) @@ -137,10 +125,41 @@ def reverify_received_webmention( flash(f"Webmention {webmention.uuid} queued for reverification.", "success") return redirect( - url_for( - "root.received", - page=request.args.get("page", 1, type=int), - ) + url_for("root.received", page=request.args.get("page", 1, type=int)) + ) + + +@root_page.route("/sent") +@login_required +def sent() -> ResponseReturnValue: + sources = db.paginate( + sa.select(Source) + .options(selectinload(Source.sent_webmentions)) + .order_by(Source.last_seen_at.desc()), + per_page=25, + ) + return render_template("sent.html", title="Sent Webmentions", sources=sources) + + +@root_page.route("/sent/<uuid:identifier>") +@login_required +def sent_source(identifier: uuid.UUID) -> ResponseReturnValue: + source = db.session.scalar( + sa.select(Source) + .options(selectinload(Source.sent_webmentions)) + .where(Source.uuid == identifier) + ) + if source is None: + return abort(404) + + webmentions = sorted( + source.sent_webmentions, key=lambda webmention: webmention.target + ) + return render_template( + "sent_source.html", + title="Sent Webmentions", + source=source, + webmentions=webmentions, ) @@ -157,19 +176,14 @@ def endpoint() -> ResponseReturnValue: webmention = db.session.execute( sa.select(ReceivedWebmention).where( - ReceivedWebmention.source == source, - ReceivedWebmention.target == target, + ReceivedWebmention.source == source, ReceivedWebmention.target == target ) ).scalar_one_or_none() if webmention is None: identifier = uuid.uuid7() - webmention = ReceivedWebmention( - uuid=identifier, - source=source, - target=target, - ) + webmention = ReceivedWebmention(uuid=identifier, source=source, target=target) db.session.add(webmention) @@ -204,16 +218,9 @@ def endpoint() -> ResponseReturnValue: verify_webmention(webmention.uuid) - status_url = url_for( - "root.status", - identifier=str(identifier), - _external=True, - ) + status_url = url_for("root.status", identifier=str(identifier), _external=True) - return Response( - status=201, - headers={"Location": status_url}, - ) + return Response(status=201, headers={"Location": status_url}) @root_page.route("/status/<uuid:identifier>") |