diff options
Diffstat (limited to 'webmentions_ssg/tasks/source_parser.py')
| -rw-r--r-- | webmentions_ssg/tasks/source_parser.py | 53 |
1 files changed, 28 insertions, 25 deletions
diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index 31d0c98..e063525 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -18,11 +18,11 @@ import xxhash from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement -from ..net import hostname_of, is_http_url +from ..net import is_http_url, normalized_hostname REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") -CONTENT_BLOCK_TAGS = frozenset( +BLOCK_LEVEL_TAGS = frozenset( { "address", "article", @@ -83,7 +83,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: return urljoin(base_url, f"{quote(directory, safe='/')}/") -def canonical_url( +def declared_canonical_url( document: BeautifulSoup, *, relative_path: Path, base_url: str | None ) -> str | None: """Return the canonical URL declared by a source document. @@ -111,7 +111,7 @@ def canonical_url( if base_url is None: raise SourceScanError( "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" + "SITE_BASE_URL is not configured" ) resolved = urljoin(source_url_for_path(relative_path, base_url), href) @@ -124,7 +124,7 @@ def canonical_url( return resolved -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: +def mf2_string_values(entry: dict[str, Any], property_name: str) -> Iterator[str]: """Yield URL values from a microformats property. :param entry: Parsed microformats entry. @@ -144,7 +144,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: yield from (value for value in values if isinstance(value, str)) -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: +def parse_h_entry(element: Tag, base_url: str) -> dict[str, Any]: """Parse and validate an h-entry with mf2py. :param element: HTML element containing the h-entry. @@ -175,7 +175,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: return entry -def primary_entry( +def find_primary_h_entry( document: BeautifulSoup, source_url: str ) -> tuple[Tag, dict[str, Any]]: """Return and validate the primary h-entry of a source document. @@ -195,9 +195,9 @@ def primary_entry( raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) + mf2_entry = parse_h_entry(entry_element, source_url) - urls = tuple(property_urls(mf2_entry, "url")) + urls = tuple(mf2_string_values(mf2_entry, "url")) if source_url not in urls: raise SourceScanError( @@ -207,15 +207,15 @@ def primary_entry( return entry_element, mf2_entry -def content_element(entry: Tag) -> Tag: +def find_e_content(entry_element: Tag) -> Tag: """Return the e-content element of an h-entry. - :param entry: HTML element containing the h-entry. + :param entry_element: HTML element containing the h-entry. :returns: The h-entry's e-content element. :raises SourceScanError: If the h-entry does not contain exactly one e-content element. """ - contents = entry.find_all(class_="e-content") + contents = entry_element.find_all(class_="e-content") if len(contents) != 1: raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") @@ -223,7 +223,7 @@ def content_element(entry: Tag) -> Tag: return contents[0] -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: +def normalize_target_url(value: str, *, base_url: str, source_url: str) -> str | None: """Resolve and validate a possible Webmention target URL. Empty values, non-HTTP URLs, and URLs referring to the source itself are @@ -249,7 +249,7 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non return target -def iter_targets( +def iter_outgoing_targets( content: Tag, mf2_entry: dict[str, Any], *, @@ -277,18 +277,19 @@ def iter_targets( ) reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + mf2_string_values(mf2_entry, property_name) + for property_name in REACTION_PROPERTIES ) for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) + target = normalize_target_url(value, base_url=base_url, source_url=source_url) if target is not None: - hostname = hostname_of(target) or "" + hostname = normalized_hostname(target) or "" if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): yield target -def normalized_content(content: Tag) -> str: +def stable_content_text(content: Tag) -> str: """Extract stable reader-facing content without generated HTML attributes. :param content: Parsed e-content element to normalize. @@ -312,7 +313,7 @@ def normalized_content(content: Tag) -> str: return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + return f" {text} " if node.name in BLOCK_LEVEL_TAGS else text return " ".join(extract(content).split()) @@ -346,22 +347,24 @@ def scan_source_file( relative_path = path.relative_to(root) document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + source_url = declared_canonical_url( + document, relative_path=relative_path, base_url=base_url + ) if source_url is None: if base_url is None: raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" + "Document has no canonical URL and SITE_BASE_URL is not configured" ) source_url = source_url_for_path(relative_path, base_url) - entry_element, mf2_entry = primary_entry(document, source_url) + entry_element, mf2_entry = find_primary_h_entry(document, source_url) - content = content_element(entry_element) + content = find_e_content(entry_element) targets = frozenset( - iter_targets( + iter_outgoing_targets( content, mf2_entry, base_url=source_url, @@ -370,7 +373,7 @@ def scan_source_file( ) ) - stable_content = normalized_content(content) + stable_content = stable_content_text(content) fingerprint = json.dumps( {"content": stable_content, "targets": sorted(targets)}, ensure_ascii=False, |
