aboutsummaryrefslogtreecommitdiff
path: root/tests/tasks/test_source_parser.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--tests/tasks/test_source_parser.py41
1 files changed, 41 insertions, 0 deletions
diff --git a/tests/tasks/test_source_parser.py b/tests/tasks/test_source_parser.py
index 38752b0..38a5dc1 100644
--- a/tests/tasks/test_source_parser.py
+++ b/tests/tasks/test_source_parser.py
@@ -1014,3 +1014,44 @@ def test_normalize_target_url_ignores_url_rejected_by_validator(
)
is None
)
+
+
+def test_outgoing_targets_skip_urls_with_unparseable_hostname(
+ source_parser_module: ModuleType, monkeypatch: MonkeyPatch
+) -> None:
+ document = parse_html(
+ '<div><a href="https://bad.example/post">bad</a>'
+ '<a href="https://good.example/post">good</a></div>'
+ )
+ content = document.find("div")
+ assert content is not None
+ original = source_parser_module.normalize_hostname
+
+ def invalid_hostname(url: str, *, from_url: bool = False) -> str:
+ if "bad.example" in url:
+ raise ValueError("Malformed URL hostname")
+ return original(url, from_url=from_url)
+
+ monkeypatch.setattr(source_parser_module, "normalize_hostname", invalid_hostname)
+ assert list(
+ source_parser_module.iter_outgoing_targets(
+ content,
+ {},
+ base_url=SOURCE_URL,
+ source_url=SOURCE_URL,
+ )
+ ) == ["https://good.example/post"]
+
+
+def test_stable_content_text_excludes_hidden_markup(
+ source_parser_module: ModuleType,
+) -> None:
+ document = parse_html(
+ "<div>Hello<script>secret-script</script>"
+ "<style>secret-style</style><template>secret-template</template>world</div>"
+ )
+ content = document.find("div")
+ assert content is not None
+ normalized = source_parser_module.stable_content_text(content)
+ assert "Hello" in normalized and "world" in normalized
+ assert "secret-" not in normalized