aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg/tasks/receiver.py
diff options
context:
space:
mode:
authorDennis Fink2026-11-12 19:57:08 +0100
committerDennis Fink2026-11-12 19:57:08 +0100
commit4bd1d9e348918b2e55db47a1ac3489d035cf55e2 (patch)
treef7ed41a374a8a4136792039a262dc62aa8cf8c59 /webmentions_ssg/tasks/receiver.py
parent7035e9f9467684cccae2b7ff3faaa62e6a9f91aa (diff)
downloadwebmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.tar.gz
webmentions-ssg-4bd1d9e348918b2e55db47a1ac3489d035cf55e2.zip
refactor(core): split modules by responsibility
Break up monolithic views, task helpers, networking, CLI, and application extension setup so each module owns a focused concern. Reorganize the test suite around the new boundaries while preserving existing Webmention behavior and coverage.
Diffstat (limited to '')
-rw-r--r--webmentions_ssg/tasks/receiver.py281
1 files changed, 95 insertions, 186 deletions
diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py
index acd66a1..bed923c 100644
--- a/webmentions_ssg/tasks/receiver.py
+++ b/webmentions_ssg/tasks/receiver.py
@@ -2,27 +2,27 @@
#
# SPDX-License-Identifier: BSD-3-Clause
-"""Fetch and verify incoming Webmentions against their source pages."""
+"""Fetch and verify incoming Webmentions with transactional state changes."""
import uuid
from datetime import UTC, datetime, timedelta
-from urllib.parse import urljoin
import httpx
-import rfc3987
import sqlalchemy as sa
-from bs4 import BeautifulSoup
from flask import current_app
from .. import APP_NAME, VERSION
-from .. import DATABASE as db
-from .. import HUEY as huey
-from ..models import ReceivedWebmention
-from ..public_transport import PublicHTTPTransport
-from ..url_security import AddressResolutionError, UnsafeAddressError
-
-
-RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
+from ..extensions import DATABASE as db
+from ..extensions import HUEY as huey
+from ..models import ReceivedStatus, ReceivedWebmention
+from ..net import (
+ AddressResolutionError,
+ UnsafeAddressError,
+ build_client,
+ is_temporary_status,
+ read_limited_body,
+)
+from .mentions import html_mentions_target, text_mentions_target
class VerificationError(Exception):
@@ -37,95 +37,7 @@ class TemporaryFetchError(Exception):
"""Fetching the source may succeed when retried later."""
-IRI_PATTERN = rfc3987.get_compiled_pattern("IRI")
-TRAILING_IRI_PUNCTUATION = ".,;:!?"
-
-
-HTML_URL_ATTRIBUTES = {
- "href": {"a", "area", "link"},
- "src": {
- "audio",
- "embed",
- "iframe",
- "img",
- 'input[type="image" i]',
- "script",
- "audio source",
- "video source",
- "track",
- "video",
- },
- "cite": {"blockquote", "del", "ins", "q"},
-}
-
-
-def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool:
- """
- Check whether an HTML document mentions the target URL.
-
- URL references are resolved against the document's ``base`` element when
- present, or against the source URL otherwise.
-
- :param body: HTML document body.
- :param source_url: URL from which the document was retrieved.
- :param target_url: Exact URL to look for.
- :return: Whether a supported HTML URL attribute references the target URL.
- """
-
- document = BeautifulSoup(body, "html.parser")
- base_url = source_url
-
- if (base_element := document.select_one("base[href]")) is not None and isinstance(
- base_href := base_element.get("href"), str
- ):
- base_url = urljoin(source_url, base_href.strip())
-
- for attribute, selectors in HTML_URL_ATTRIBUTES.items():
- selector = ", ".join(
- [f"{selector_string}[{attribute}]" for selector_string in selectors]
- )
-
- for element in document.select(selector):
- if not isinstance(reference := element.get(attribute), str):
- continue
-
- if urljoin(base_url, reference.strip()) == target_url:
- return True
-
- return False
-
-
-def text_mentions_target(body: str, target_url: str) -> bool:
- """
- Check whether plain text contains the target IRI, ignoring trailing prose
- punctuation but not extra URL path or query characters.
-
- :param body: Plain-text document body.
- :param target_url: Exact IRI to look for.
- :return: Whether the target IRI occurs in the document.
- """
- for match in IRI_PATTERN.finditer(body):
- candidate = match.group()
-
- # A URL in prose can be followed by punctuation that the IRI pattern
- # consumes as part of its match. Only remove trailing punctuation,
- # leaving longer URLs (e.g. ``target/more``) unmatched.
- if not candidate.startswith(target_url):
- continue
-
- while candidate != target_url:
- if candidate[-1] in TRAILING_IRI_PUNCTUATION:
- candidate = candidate[:-1]
- elif candidate[-1] == ")" and candidate.count(")") > candidate.count("("):
- # Keep balanced parentheses that belong to the URL itself.
- candidate = candidate[:-1]
- else:
- break
-
- if candidate == target_url:
- return True
-
- return False
+RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached"
def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
@@ -144,60 +56,37 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
"""
try:
- return _fetch_source(source_url)
+ with (
+ build_client(
+ f"{APP_NAME}/{VERSION} ReceivedWebmention",
+ accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
+ ) as client,
+ client.stream("GET", source_url) as response,
+ ):
+ match response.status_code:
+ case 200:
+ pass
+ case 410:
+ raise SourceGoneError("Source returned HTTP 410")
+ case status:
+ if is_temporary_status(status):
+ raise TemporaryFetchError(f"Source returned HTTP {status}")
+ else:
+ raise VerificationError(f"Source returned HTTP {status}")
+
+ try:
+ return response, read_limited_body(
+ response, current_app.config["MAX_SOURCE_BYTES"], label="Source"
+ )
+ except ValueError as exc:
+ raise VerificationError(str(exc)) from exc
+
except UnsafeAddressError as exc:
raise VerificationError("Source resolves to a non-public address") from exc
except AddressResolutionError as exc:
raise TemporaryFetchError("Source hostname could not be resolved") from exc
-def _fetch_source(source_url: str) -> tuple[httpx.Response, bytes]:
- with (
- httpx.Client(
- headers={
- "Accept": "text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8",
- "User-Agent": f"{APP_NAME}/{VERSION} ReceivedWebmention",
- },
- timeout=httpx.Timeout(current_app.config.get("REQUEST_TIMEOUT", 5.0)),
- follow_redirects=True,
- max_redirects=current_app.config.get("MAX_REDIRECTS", 20),
- trust_env=False,
- transport=PublicHTTPTransport(
- current_app.config.get("REQUEST_TOTAL_TIMEOUT", 30.0)
- ),
- ) as client,
- client.stream("GET", source_url) as response,
- ):
- match response.status_code:
- case 200:
- pass
- case 410:
- raise SourceGoneError("Source returned HTTP 410")
- case status:
- if status in {408, 425, 429} or 500 <= status <= 599:
- raise TemporaryFetchError(f"Source returned HTTP {status}")
- else:
- raise VerificationError(f"Source returned HTTP {status}")
-
- max_source_bytes = current_app.config.get("MAX_SOURCE_BYTES", 1_000_000)
-
- if (content_length := response.headers.get("Content-Length")) is not None:
- try:
- if int(content_length) > max_source_bytes:
- raise VerificationError("Source document is too large")
- except ValueError:
- pass
-
- body = bytearray()
- for chunk in response.iter_bytes(chunk_size=64 * 1024):
- body.extend(chunk)
-
- if len(body) > max_source_bytes:
- raise VerificationError("Source document is too large")
-
- return response, bytes(body)
-
-
def source_mentions_target(source_url: str, target_url: str) -> bool:
"""
Fetch a source and check whether it mentions the target URL.
@@ -260,13 +149,11 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
)
return
- # Failed tasks may be retried by Huey. Completed and already-running tasks
- # should not start another concurrent verification.
- if webmention.status not in {"received", "failed"}:
+ if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}:
return
- # A reaper-exhausted row must not be revived by a late Huey retry.
+
if (
- webmention.status == "failed"
+ webmention.status == ReceivedStatus.FAILED
and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON
):
return
@@ -283,39 +170,30 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
ReceivedWebmention.status == previous_status,
ReceivedWebmention.updated_at == previous_updated_at,
)
- .values(status="verifying", failure_reason=None, updated_at=started_at)
+ .values(
+ status=ReceivedStatus.VERIFYING, failure_reason=None, updated_at=started_at
+ )
)
+
if claimed.rowcount != 1:
db.session.rollback()
return
- db.session.commit()
- def finish(status: str, failure_reason: str | None) -> None:
- # A timeout or a newer verification may have superseded this attempt.
- # Do not let a late worker overwrite its state.
- db.session.execute(
- sa.update(ReceivedWebmention)
- .where(
- ReceivedWebmention.uuid == webmention_uuid,
- ReceivedWebmention.status == "verifying",
- ReceivedWebmention.updated_at == started_at,
- )
- .values(
- status=status,
- failure_reason=failure_reason,
- updated_at=datetime.now(UTC),
- )
- )
- db.session.commit()
+ db.session.commit()
+ status = None
+ failure_reason = None
try:
mentions_target = source_mentions_target(source, target)
except SourceGoneError as exc:
- finish("deleted", str(exc))
+ status = ReceivedStatus.DELETED
+ failure_reason = str(exc)
except VerificationError as exc:
- finish("failed", str(exc))
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc)
except (TemporaryFetchError, httpx.RequestError) as exc:
- finish("failed", str(exc) or "Source could not be fetched")
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or "Source could not be fetched"
raise
except Exception as exc:
# InvalidURL is not a RequestError; other unexpected source errors must
@@ -323,12 +201,31 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None:
current_app.logger.exception(
"Unexpected error verifying ReceivedWebmention %s", webmention_uuid
)
- finish("failed", str(exc) or type(exc).__name__)
+
+ status = ReceivedStatus.FAILED
+ failure_reason = str(exc) or type(exc).__name__
else:
if mentions_target:
- finish("verified", None)
+ status = ReceivedStatus.VERIFIED
else:
- finish("deleted", "Source does not mention target")
+ status = ReceivedStatus.DELETED
+ failure_reason = "Source does not mention target"
+ finally:
+ if status is not None:
+ db.session.execute(
+ sa.update(ReceivedWebmention)
+ .where(
+ ReceivedWebmention.uuid == webmention_uuid,
+ ReceivedWebmention.status == ReceivedStatus.VERIFYING,
+ ReceivedWebmention.updated_at == started_at,
+ )
+ .values(
+ status=status,
+ failure_reason=failure_reason,
+ updated_at=datetime.now(UTC),
+ )
+ )
+ db.session.commit()
def recover_stale_webmentions() -> int:
@@ -347,6 +244,7 @@ def recover_stale_webmentions() -> int:
"""
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"])
+
stale = db.session.execute(
sa.select(
ReceivedWebmention.uuid,
@@ -354,7 +252,9 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.updated_at,
ReceivedWebmention.recovery_attempts,
).where(
- ReceivedWebmention.status.in_(("verifying", "received")),
+ ReceivedWebmention.status.in_(
+ (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED)
+ ),
ReceivedWebmention.updated_at <= cutoff,
)
).all()
@@ -362,6 +262,7 @@ def recover_stale_webmentions() -> int:
recovered = 0
exhausted = 0
max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"]
+
for identifier, status, updated_at, recovery_attempts in stale:
at_limit = recovery_attempts >= max_recoveries
changed = db.session.execute(
@@ -373,19 +274,27 @@ def recover_stale_webmentions() -> int:
ReceivedWebmention.recovery_attempts == recovery_attempts,
)
.values(
- status="failed" if at_limit else "received",
- failure_reason=RECOVERY_EXHAUSTED_REASON if at_limit else None,
- updated_at=now,
- recovery_attempts=(
- recovery_attempts if at_limit else recovery_attempts + 1
- ),
+ **(
+ {
+ "status": ReceivedStatus.FAILED,
+ "failure_reason": RECOVERY_EXHAUSTED_REASON,
+ "updated_at": now,
+ "recovery_attempts": recovery_attempts,
+ }
+ if at_limit
+ else ReceivedWebmention.verification_reset_values(
+ at=now, recoveries=recovery_attempts + 1
+ )
+ )
)
)
+
if changed.rowcount != 1:
db.session.rollback()
continue
db.session.commit()
+
if at_limit:
exhausted += 1
continue