aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tests/tasks/test_scanner.py149
-rw-r--r--webmentions_ssg/tasks/scanner.py16
2 files changed, 165 insertions, 0 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 8dee47d..4e6c818 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -735,6 +735,155 @@ def test_scan_creates_source_and_webmentions(
assert set(queued) == identifiers
+def test_duplicate_canonical_url_skips_alias_and_commits_other_sources(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ configure_scanner(app, tmp_path)
+
+ for slug in ("first", "second"):
+ write_post(
+ tmp_path,
+ slug,
+ f'<a href="https://example.com/{slug}">Link</a>',
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+ write_post(
+ tmp_path, "unrelated", '<a href="https://example.com/unrelated">Link</a>'
+ )
+
+ queued: list[UUID] = []
+ monkeypatch.setattr(scanner_module, "send_webmention", queued.append)
+
+ run_scan(app, scanner_module)
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+ webmentions = list(db.session.scalars(sa.select(SentWebmention)))
+
+ assert len(sources) == 2
+ assert {source.url for source in sources} == {
+ SOURCE_URL,
+ f"{BASE_URL}unrelated/",
+ }
+ duplicate_paths = {source.path for source in sources} & {
+ "first/index.html",
+ "second/index.html",
+ }
+ assert len(duplicate_paths) == 1
+ assert all(source.revision == 1 for source in sources)
+ assert len(webmentions) == 2
+ assert {webmention.target for webmention in webmentions} in (
+ {"https://example.com/first", "https://example.com/unrelated"},
+ {"https://example.com/second", "https://example.com/unrelated"},
+ )
+ assert set(queued) == {webmention.uuid for webmention in webmentions}
+
+ assert "duplicate canonical URL" in caplog.text
+ assert SOURCE_URL in caplog.text
+
+
+def test_existing_source_owns_canonical_url_when_alias_is_added(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ configure_scanner(app, tmp_path)
+ write_post(
+ tmp_path,
+ "owner",
+ "<p>Original content.</p>",
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+ monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None)
+
+ run_scan(app, scanner_module)
+
+ # The later-added alias must not take the persisted owner's URL, regardless
+ # of filesystem traversal order.
+ write_post(
+ tmp_path,
+ "alias",
+ '<a href="https://example.com/alias">Link</a>',
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+ write_post(
+ tmp_path,
+ "owner",
+ '<a href="https://example.com/owner">Link</a>',
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+ targets = list(db.session.scalars(sa.select(SentWebmention.target)))
+
+ assert len(sources) == 1
+ assert sources[0].path == "owner/index.html"
+ assert sources[0].url == SOURCE_URL
+ assert sources[0].revision == 2
+ assert targets == ["https://example.com/owner"]
+
+ assert "duplicate canonical URL" in caplog.text
+ assert "alias/index.html" in caplog.text
+ assert "owner/index.html" in caplog.text
+
+
+def test_deleted_source_still_reserves_its_canonical_url(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ monkeypatch: MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ configure_scanner(app, tmp_path)
+ path = write_post(
+ tmp_path,
+ "owner",
+ "<p>Original content.</p>",
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+ monkeypatch.setattr(scanner_module, "send_webmention", lambda identifier: None)
+
+ run_scan(app, scanner_module)
+ path.unlink()
+ run_scan(app, scanner_module)
+
+ write_post(
+ tmp_path,
+ "alias",
+ "<p>Alias content.</p>",
+ canonical=SOURCE_URL,
+ entry_url=SOURCE_URL,
+ )
+ write_post(tmp_path, "unrelated", "<p>Another source.</p>")
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = {
+ source.path: source for source in db.session.scalars(sa.select(Source))
+ }
+ assert set(sources) == {"owner/index.html", "unrelated/index.html"}
+ assert sources["owner/index.html"].deleted_at is not None
+ assert sources["unrelated/index.html"].deleted_at is None
+
+ assert "duplicate canonical URL" in caplog.text
+ assert SOURCE_URL in caplog.text
+
+
def test_unchanged_source_does_not_increment_revision(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index 4bb6736..e8149ba 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -439,6 +439,10 @@ def scan_sources() -> None:
sa.select(Source).options(selectinload(Source.sent_webmentions))
)
}
+ # Existing sources retain their canonical URLs even when marked deleted.
+ # Reserve these URLs before scanning so an alias cannot claim one just
+ # because it is discovered earlier than its persisted owner.
+ paths_by_url = {source.url: source.path for source in sources_by_path.values()}
seen_paths: set[str] = set()
scanned_count = 0
@@ -470,6 +474,17 @@ def scan_sources() -> None:
)
continue
+ if (owner_path := paths_by_url.get(scanned.url)) is not None and (
+ owner_path != scanned.path
+ ):
+ current_app.logger.warning(
+ "Skipping source %s: duplicate canonical URL %r (already used by %s)",
+ index_path,
+ scanned.url,
+ owner_path,
+ )
+ continue
+
try:
if (source := sources_by_path.get(scanned.path)) is None:
source = create_source(scanned, scan_time)
@@ -478,6 +493,7 @@ def scan_sources() -> None:
source = update_source(source, scanned, scan_time)
sources_by_path[source.path] = source
+ paths_by_url[source.url] = source.path
scanned_count += 1
except SourceScanError as exc:
current_app.logger.warning("Could not scan source %s: %s", index_path, exc)