From 89bff71238d06178528515e45cc00bcc8d70d663 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 12:17:04 +0100 Subject: fix(sender): prevent duplicate canonical source URLs Keep persisted sources, including deleted ones, as owners of their canonical URLs so newly scanned aliases cannot claim the same source identity. Skip duplicate canonical paths without blocking unrelated sources from being saved or queued for delivery. --- tests/tasks/test_scanner.py | 149 +++++++++++++++++++++++++++++++++++++++ webmentions_ssg/tasks/scanner.py | 16 +++++ 2 files changed, 165 insertions(+) diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 8dee47d..4e6c818 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -735,6 +735,155 @@ def test_scan_creates_source_and_webmentions( assert set(queued) == identifiers +def test_duplicate_canonical_url_skips_alias_and_commits_other_sources( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + configure_scanner(app, tmp_path) + + for slug in ("first", "second"): + write_post( + tmp_path, + slug, + f'Link', + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + write_post( + tmp_path, "unrelated", 'Link' + ) + + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + + run_scan(app, scanner_module) + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + webmentions = list(db.session.scalars(sa.select(SentWebmention))) + + assert len(sources) == 2 + assert {source.url for source in sources} == { + SOURCE_URL, + f"{BASE_URL}unrelated/", + } + duplicate_paths = {source.path for source in sources} & { + "first/index.html", + "second/index.html", + } + assert len(duplicate_paths) == 1 + assert all(source.revision == 1 for source in sources) + assert len(webmentions) == 2 + assert {webmention.target for webmention in webmentions} in ( + {"https://example.com/first", "https://example.com/unrelated"}, + {"https://example.com/second", "https://example.com/unrelated"}, + ) + assert set(queued) == {webmention.uuid for webmention in webmentions} + + assert "duplicate canonical URL" in caplog.text + assert SOURCE_URL in caplog.text + + +def test_existing_source_owns_canonical_url_when_alias_is_added( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + configure_scanner(app, tmp_path) + write_post( + tmp_path, + "owner", + "

Original content.

", + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + + run_scan(app, scanner_module) + + # The later-added alias must not take the persisted owner's URL, regardless + # of filesystem traversal order. + write_post( + tmp_path, + "alias", + 'Link', + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + write_post( + tmp_path, + "owner", + 'Link', + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + targets = list(db.session.scalars(sa.select(SentWebmention.target))) + + assert len(sources) == 1 + assert sources[0].path == "owner/index.html" + assert sources[0].url == SOURCE_URL + assert sources[0].revision == 2 + assert targets == ["https://example.com/owner"] + + assert "duplicate canonical URL" in caplog.text + assert "alias/index.html" in caplog.text + assert "owner/index.html" in caplog.text + + +def test_deleted_source_still_reserves_its_canonical_url( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + monkeypatch: MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + configure_scanner(app, tmp_path) + path = write_post( + tmp_path, + "owner", + "

Original content.

", + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None) + + run_scan(app, scanner_module) + path.unlink() + run_scan(app, scanner_module) + + write_post( + tmp_path, + "alias", + "

Alias content.

", + canonical=SOURCE_URL, + entry_url=SOURCE_URL, + ) + write_post(tmp_path, "unrelated", "

Another source.

") + run_scan(app, scanner_module) + + with app.app_context(): + sources = { + source.path: source for source in db.session.scalars(sa.select(Source)) + } + assert set(sources) == {"owner/index.html", "unrelated/index.html"} + assert sources["owner/index.html"].deleted_at is not None + assert sources["unrelated/index.html"].deleted_at is None + + assert "duplicate canonical URL" in caplog.text + assert SOURCE_URL in caplog.text + + def test_unchanged_source_does_not_increment_revision( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 4bb6736..e8149ba 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -439,6 +439,10 @@ def scan_sources() -> None: sa.select(Source).options(selectinload(Source.sent_webmentions)) ) } + # Existing sources retain their canonical URLs even when marked deleted. + # Reserve these URLs before scanning so an alias cannot claim one just + # because it is discovered earlier than its persisted owner. + paths_by_url = {source.url: source.path for source in sources_by_path.values()} seen_paths: set[str] = set() scanned_count = 0 @@ -470,6 +474,17 @@ def scan_sources() -> None: ) continue + if (owner_path := paths_by_url.get(scanned.url)) is not None and ( + owner_path != scanned.path + ): + current_app.logger.warning( + "Skipping source %s: duplicate canonical URL %r (already used by %s)", + index_path, + scanned.url, + owner_path, + ) + continue + try: if (source := sources_by_path.get(scanned.path)) is None: source = create_source(scanned, scan_time) @@ -478,6 +493,7 @@ def scan_sources() -> None: source = update_source(source, scanned, scan_time) sources_by_path[source.path] = source + paths_by_url[source.url] = source.path scanned_count += 1 except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) -- cgit v1.3.1