aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/source_parser.py
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg/tasks/source_parser.py')
-rw-r--r--webmentions_ssg/tasks/source_parser.py53
1 files changed, 28 insertions, 25 deletions
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py
index 31d0c98..e063525 100644
--- a/webmentions_ssg/tasks/source_parser.py
+++ b/webmentions_ssg/tasks/source_parser.py
@@ -18,11 +18,11 @@ import xxhash
from bs4 import BeautifulSoup, Tag
from bs4.element import NavigableString, PageElement
-from ..net import hostname_of, is_http_url
+from ..net import is_http_url, normalized_hostname
REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of")
-CONTENT_BLOCK_TAGS = frozenset(
+BLOCK_LEVEL_TAGS = frozenset(
{
"address",
"article",
@@ -83,7 +83,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str:
return urljoin(base_url, f"{quote(directory, safe='/')}/")
-def canonical_url(
+def declared_canonical_url(
document: BeautifulSoup, *, relative_path: Path, base_url: str | None
) -> str | None:
"""Return the canonical URL declared by a source document.
@@ -111,7 +111,7 @@ def canonical_url(
if base_url is None:
raise SourceScanError(
"Document contains a relative canonical URL, but "
- "SOURCE_BASE_URL is not configured"
+ "SITE_BASE_URL is not configured"
)
resolved = urljoin(source_url_for_path(relative_path, base_url), href)
@@ -124,7 +124,7 @@ def canonical_url(
return resolved
-def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
+def mf2_string_values(entry: dict[str, Any], property_name: str) -> Iterator[str]:
"""Yield URL values from a microformats property.
:param entry: Parsed microformats entry.
@@ -144,7 +144,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]:
yield from (value for value in values if isinstance(value, str))
-def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
+def parse_h_entry(element: Tag, base_url: str) -> dict[str, Any]:
"""Parse and validate an h-entry with mf2py.
:param element: HTML element containing the h-entry.
@@ -175,7 +175,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]:
return entry
-def primary_entry(
+def find_primary_h_entry(
document: BeautifulSoup, source_url: str
) -> tuple[Tag, dict[str, Any]]:
"""Return and validate the primary h-entry of a source document.
@@ -195,9 +195,9 @@ def primary_entry(
raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}")
entry_element = entries[0]
- mf2_entry = parse_entry(entry_element, source_url)
+ mf2_entry = parse_h_entry(entry_element, source_url)
- urls = tuple(property_urls(mf2_entry, "url"))
+ urls = tuple(mf2_string_values(mf2_entry, "url"))
if source_url not in urls:
raise SourceScanError(
@@ -207,15 +207,15 @@ def primary_entry(
return entry_element, mf2_entry
-def content_element(entry: Tag) -> Tag:
+def find_e_content(entry_element: Tag) -> Tag:
"""Return the e-content element of an h-entry.
- :param entry: HTML element containing the h-entry.
+ :param entry_element: HTML element containing the h-entry.
:returns: The h-entry's e-content element.
:raises SourceScanError: If the h-entry does not contain exactly one e-content
element.
"""
- contents = entry.find_all(class_="e-content")
+ contents = entry_element.find_all(class_="e-content")
if len(contents) != 1:
raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}")
@@ -223,7 +223,7 @@ def content_element(entry: Tag) -> Tag:
return contents[0]
-def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None:
+def normalize_target_url(value: str, *, base_url: str, source_url: str) -> str | None:
"""Resolve and validate a possible Webmention target URL.
Empty values, non-HTTP URLs, and URLs referring to the source itself are
@@ -249,7 +249,7 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non
return target
-def iter_targets(
+def iter_outgoing_targets(
content: Tag,
mf2_entry: dict[str, Any],
*,
@@ -277,18 +277,19 @@ def iter_targets(
)
reactions = chain.from_iterable(
- property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES
+ mf2_string_values(mf2_entry, property_name)
+ for property_name in REACTION_PROPERTIES
)
for value in chain(hrefs, reactions):
- target = normalize_target(value, base_url=base_url, source_url=source_url)
+ target = normalize_target_url(value, base_url=base_url, source_url=source_url)
if target is not None:
- hostname = hostname_of(target) or ""
+ hostname = normalized_hostname(target) or ""
if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames):
yield target
-def normalized_content(content: Tag) -> str:
+def stable_content_text(content: Tag) -> str:
"""Extract stable reader-facing content without generated HTML attributes.
:param content: Parsed e-content element to normalize.
@@ -312,7 +313,7 @@ def normalized_content(content: Tag) -> str:
return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] "
text = "".join(extract(child) for child in node.children)
- return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text
+ return f" {text} " if node.name in BLOCK_LEVEL_TAGS else text
return " ".join(extract(content).split())
@@ -346,22 +347,24 @@ def scan_source_file(
relative_path = path.relative_to(root)
document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
- source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
+ source_url = declared_canonical_url(
+ document, relative_path=relative_path, base_url=base_url
+ )
if source_url is None:
if base_url is None:
raise SourceScanError(
- "Document has no canonical URL and SOURCE_BASE_URL is not configured"
+ "Document has no canonical URL and SITE_BASE_URL is not configured"
)
source_url = source_url_for_path(relative_path, base_url)
- entry_element, mf2_entry = primary_entry(document, source_url)
+ entry_element, mf2_entry = find_primary_h_entry(document, source_url)
- content = content_element(entry_element)
+ content = find_e_content(entry_element)
targets = frozenset(
- iter_targets(
+ iter_outgoing_targets(
content,
mf2_entry,
base_url=source_url,
@@ -370,7 +373,7 @@ def scan_source_file(
)
)
- stable_content = normalized_content(content)
+ stable_content = stable_content_text(content)
fingerprint = json.dumps(
{"content": stable_content, "targets": sorted(targets)},
ensure_ascii=False,