From dcbbfdb38d4e18e377d4e326d82f35341840c6cc Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Fri, 30 Oct 2026 19:23:17 +0100 Subject: fix(sender): stabilize source content fingerprints Hash normalized reader-facing content and outgoing targets instead of raw h-entry markup so generated attributes and relative timestamps do not trigger spurious source revisions or repeat Webmentions. Preserve meaningful changes such as image updates and reaction target changes while ignoring target ordering and other markup noise. --- tests/tasks/test_scanner.py | 167 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) (limited to 'tests/tasks/test_scanner.py') diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 21a13c6..7aefff4 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -508,6 +508,119 @@ def test_scan_source_uses_canonical_without_base_url( assert len(scanned.content_hash) == 32 +def test_scan_source_ignores_markup_noise_in_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + "

Hello world!

" + 'Target', + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + write_post( + tmp_path, + "example", + '

Hello world!

' + ' Target ', + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + +def test_scan_source_uses_stable_time_value_inside_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post( + tmp_path, + "example", + '', + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash == second.content_hash + + +def test_scan_source_hash_changes_when_image_changes( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", 'A photo') + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace("/images/a.jpg", "/images/b.jpg"), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_includes_reaction_targets_outside_content( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '
', + 'Like' + '
', + ), + encoding="utf-8", + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + "https://example.com/first", "https://example.com/second" + ), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == frozenset({"https://example.com/first"}) + assert second.targets == frozenset({"https://example.com/second"}) + assert first.content_hash != second.content_hash + + +def test_scan_source_hash_ignores_reaction_target_order( + scanner_module: ModuleType, tmp_path: Path +) -> None: + path = write_post(tmp_path, "example", "

Same content.

") + html = path.read_text(encoding="utf-8") + first_link = 'A' + second_link = 'B' + path.write_text( + html.replace( + '
', + first_link + second_link + '
', + ), + encoding="utf-8", + ) + first = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + path.write_text( + path.read_text(encoding="utf-8").replace( + first_link + second_link, second_link + first_link + ), + encoding="utf-8", + ) + second = scanner_module.scan_source_file(path, root=tmp_path, base_url=None) + + assert first.targets == second.targets + assert first.content_hash == second.content_hash + + def test_scan_source_falls_back_to_base_url( scanner_module: ModuleType, tmp_path: Path ) -> None: @@ -976,6 +1089,56 @@ def test_unchanged_source_does_not_increment_revision( assert queued == [] +def test_changing_relative_timestamp_does_not_resend( + app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + configure_scanner(app, tmp_path) + path = write_post( + tmp_path, "example", 'Target' + ) + html = path.read_text(encoding="utf-8") + path.write_text( + html.replace( + '
', + '' + '
', + ), + encoding="utf-8", + ) + + queued: list[UUID] = [] + monkeypatch.setattr(scanner_module, "send_webmention", queued.append) + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + assert source is not None and webmention is not None + original_hash = source.content_hash + webmention.processed_revision = 1 + webmention.sent_revision = 1 + webmention.status = SentWebmentionStatus.SENT + db.session.commit() + + queued.clear() + path.write_text( + path.read_text(encoding="utf-8").replace("5 minutes ago", "6 minutes ago"), + encoding="utf-8", + ) + run_scan(app, scanner_module) + + with app.app_context(): + source = db.session.scalar(sa.select(Source)) + webmention = db.session.scalar(sa.select(SentWebmention)) + assert source is not None and webmention is not None + assert source.content_hash == original_hash + assert source.revision == 1 + assert webmention.desired_revision == 1 + assert webmention.processed_revision == 1 + + assert queued == [] + + def test_updated_source_increments_revision( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1597,7 +1760,6 @@ def test_pending_webmention_is_requeued_on_next_scan( assert queued == [identifier] - def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: @@ -1631,7 +1793,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget queued.clear() # Changing the h-entry creates a new desired revision with a fresh budget. - path.write_text(path.read_text().replace('>A', '>A updated')) + path.write_text(path.read_text().replace(">A", ">A updated")) with app.app_context(): webmention = db.session.get(SentWebmention, identifier) assert webmention is not None @@ -1647,6 +1809,7 @@ def test_temporary_failure_waits_until_retry_deadline_and_revision_resets_budget assert webmention.attempt_count == 0 assert webmention.next_retry_at is None + def test_invalid_known_source_is_not_deleted( app: Flask, scanner_module: ModuleType, -- cgit v1.3.1