diff options
Diffstat (limited to 'tests/tasks/test_source_parser.py')
| -rw-r--r-- | tests/tasks/test_source_parser.py | 41 |
1 files changed, 41 insertions, 0 deletions
diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py index 38752b0..38a5dc1 100644 --- a/tests/tasks/test_source_parser.py +++ b/tests/tasks/test_source_parser.py @@ -1014,3 +1014,44 @@ def test_normalize_target_url_ignores_url_rejected_by_validator( ) is None ) + + +def test_outgoing_targets_skip_urls_with_unparseable_hostname( + source_parser_module: ModuleType, monkeypatch: MonkeyPatch +) -> None: + document = parse_html( + '<div><a href="https://bad.example/post">bad</a>' + '<a href="https://good.example/post">good</a></div>' + ) + content = document.find("div") + assert content is not None + original = source_parser_module.normalize_hostname + + def invalid_hostname(url: str, *, from_url: bool = False) -> str: + if "bad.example" in url: + raise ValueError("Malformed URL hostname") + return original(url, from_url=from_url) + + monkeypatch.setattr(source_parser_module, "normalize_hostname", invalid_hostname) + assert list( + source_parser_module.iter_outgoing_targets( + content, + {}, + base_url=SOURCE_URL, + source_url=SOURCE_URL, + ) + ) == ["https://good.example/post"] + + +def test_stable_content_text_excludes_hidden_markup( + source_parser_module: ModuleType, +) -> None: + document = parse_html( + "<div>Hello<script>secret-script</script>" + "<style>secret-style</style><template>secret-template</template>world</div>" + ) + content = document.find("div") + assert content is not None + normalized = source_parser_module.stable_content_text(content) + assert "Hello" in normalized and "world" in normalized + assert "secret-" not in normalized |
