diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 3 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/discovery.py | 18 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 40 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/mentions.py | 7 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/notifications.py | 1 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 14 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/retention.py | 9 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 11 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/source_parser.py | 32 |
10 files changed, 53 insertions, 88 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index bf7189f..9010ef7 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -10,8 +10,7 @@ from .. import HUEY, create_app def create_consumer() -> Huey: - """ - Create and configure the Huey consumer. + """Create and configure the Huey consumer. The application is initialized, task modules are imported to register their tasks, and periodic scanner, notification and verification recovery tasks diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py index 98f3fd5..ddf5817 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/discovery.py @@ -30,8 +30,7 @@ class PermanentSenderError(SenderError): def resolve_endpoint(response: httpx.Response, href: str) -> str: - """ - Resolve and validate a discovered Webmention endpoint. + """Resolve and validate a discovered Webmention endpoint. :param response: Response from which the endpoint was discovered. :param href: Endpoint reference to resolve. @@ -51,8 +50,7 @@ def resolve_endpoint(response: httpx.Response, href: str) -> str: def parse_link_value(value: str) -> tuple[str, set[str]] | None: - """ - Parse a Link header value into its target and relations. + """Parse a Link header value into its target and relations. :param value: Link header value to parse. :return: Link target and relation names, or ``None`` if the value is invalid. @@ -81,8 +79,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None: def endpoint_from_headers(response: httpx.Response) -> str | None: - """ - Find a Webmention endpoint in the response Link headers. + """Find a Webmention endpoint in the response Link headers. :param response: HTTP response whose headers should be inspected. :return: First valid advertised Webmention endpoint, or ``None`` if none @@ -105,8 +102,7 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: - """ - Find a Webmention endpoint in an HTML document. + """Find a Webmention endpoint in an HTML document. ``link`` and ``a`` elements are inspected in document order. @@ -156,8 +152,7 @@ def read_target_body(response: httpx.Response) -> bytes: def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """ - Discover the Webmention endpoint advertised by a target URL. + """Discover the Webmention endpoint advertised by a target URL. Endpoint discovery first checks the response to a HEAD request and then falls back to a GET request, inspecting both HTTP Link headers and supported HTML @@ -202,8 +197,7 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non def post_webmention( client: httpx.Client, *, endpoint: str, source: str, target: str ) -> tuple[int, str | None]: - """ - Send a Webmention to a discovered endpoint. + """Send a Webmention to a discovered endpoint. :param client: HTTP client to use for the request. :param endpoint: Webmention endpoint URL. diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index 9f53f3b..f7072b2 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -20,16 +20,14 @@ R = TypeVar("R") class Huey: - """ - Provide Flask integration for a Huey instance. + """Provide Flask integration for a Huey instance. The extension initializes a Huey backend from the Flask configuration and wraps tasks so that they execute within an application context. """ def __init__(self, app: Flask | None = None) -> None: - """ - Initialize the Huey extension. + """Initialize the Huey extension. :param app: Flask application to initialize immediately, if provided. """ @@ -40,8 +38,7 @@ class Huey: self.init_app(app) def init_app(self, app: Flask) -> None: - """ - Initialize Huey for a Flask application. + """Initialize Huey for a Flask application. The backend and its storage options are derived from ``HUEY_URL`` and the resulting extension is registered with the application. @@ -80,8 +77,7 @@ class Huey: @property def huey(self) -> BaseHuey: - """ - Return the initialized Huey instance. + """Return the initialized Huey instance. :return: Configured Huey backend instance. :raises RuntimeError: If the extension has not been initialized. @@ -128,8 +124,7 @@ class Huey: return decorator def __getattr__(self, name: str) -> Any: - """ - Forward an unknown attribute to the underlying Huey instance. + """Forward an unknown attribute to the underlying Huey instance. :param name: Name of the attribute to retrieve. :return: Attribute from the initialized Huey instance. @@ -139,8 +134,7 @@ class Huey: @staticmethod def backend_from_url(url: str) -> tuple[type[BaseHuey], dict[str, Any]]: - """ - Determine the Huey backend and storage options from a URL. + """Determine the Huey backend and storage options from a URL. :param url: Huey backend URL. :return: Huey backend class and keyword arguments for its storage backend. @@ -157,20 +151,22 @@ class Huey: from huey import PriorityRedisExpireHuey return PriorityRedisExpireHuey, {"url": fixed_url} - elif scheme.endswith("priority"): + + if scheme.endswith("priority"): from huey import PriorityRedisHuey return PriorityRedisHuey, {"url": fixed_url} - elif scheme.endswith("expire"): + + if scheme.endswith("expire"): from huey import RedisExpireHuey return RedisExpireHuey, {"url": fixed_url} - else: - from huey import RedisHuey - return RedisHuey, {"url": url} + from huey import RedisHuey + + return RedisHuey, {"url": url} - elif scheme == "sqlite": + if scheme == "sqlite": from huey import SqliteHuey prefix = "sqlite:///" @@ -187,7 +183,7 @@ class Huey: return SqliteHuey, {"filename": filename} - elif scheme == "file": + if scheme == "file": from huey import FileHuey prefix = "file:///" @@ -204,17 +200,17 @@ class Huey: return FileHuey, {"path": path} - elif scheme in {"postgres", "postgresql"}: + if scheme in {"postgres", "postgresql"}: from huey import PostgresHuey return PostgresHuey, {"dsn": url} - elif scheme == "memory": + if scheme == "memory": from huey import MemoryHuey return MemoryHuey, {} - elif scheme == "blackhole": + if scheme == "blackhole": from huey import BlackHoleHuey return BlackHoleHuey, {} diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/mentions.py index 4e5a5ca..42dd0b6 100644 --- a/webmentions_ssg/tasks/mentions.py +++ b/webmentions_ssg/tasks/mentions.py @@ -31,8 +31,7 @@ HTML_URL_ATTRIBUTES = { def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: - """ - Check whether an HTML document mentions the target URL. + """Check whether an HTML document mentions the target URL. URL references are resolved against the document's ``base`` element when present, or against the source URL otherwise. @@ -42,7 +41,6 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: :param target_url: Exact URL to look for. :return: Whether a supported HTML URL attribute references the target URL. """ - document = BeautifulSoup(body, "html.parser") base_url = source_url @@ -67,8 +65,7 @@ def html_mentions_target(body: bytes, source_url: str, target_url: str) -> bool: def text_mentions_target(body: str, target_url: str) -> bool: - """ - Check whether plain text contains the target IRI, ignoring trailing prose + """Check whether plain text contains the target IRI, ignoring trailing prose punctuation but not extra URL path or query characters. :param body: Plain-text document body. diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index 889030c..a6a9b2b 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -19,7 +19,6 @@ from ..models import ReceivedStatus, ReceivedWebmention @huey.lock_task("send-webmention-digest") def send_webmention_digest() -> None: """Send one digest of verified, previously unreported Webmentions.""" - recipient = current_app.config.get("NOTIFICATION_EMAIL") if not recipient: return diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index feb6823..ada89f8 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -42,8 +42,7 @@ RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: - """ - Fetch a webmention source document. + """Fetch a webmention source document. The request follows redirects while enforcing the configured timeout, redirect limit, and maximum response size. @@ -55,7 +54,6 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: :raises VerificationError: If the source returns another unsuccessful HTTP status or exceeds the configured maximum response size. """ - try: with ( build_client( @@ -89,8 +87,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: def source_mentions_target(source_url: str, target_url: str) -> bool: - """ - Fetch a source and check whether it mentions the target URL. + """Fetch a source and check whether it mentions the target URL. HTML and XHTML sources are inspected for supported URL attributes, while plain-text sources are searched for the exact target IRI. @@ -103,7 +100,6 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: :raises SourceGoneError: If the source explicitly reports that it is gone. :raises TemporaryFetchError: If fetching the source fails temporarily. """ - response, body = fetch_source(source_url) media_type = ( @@ -129,8 +125,7 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: @huey.task(retries=2, retry_delay=50) def verify_webmention(webmention_uuid: uuid.UUID) -> None: - """ - Verify a received webmention and persist the result. + """Verify a received webmention and persist the result. The transition into ``verifying`` is an atomic claim. Its ``updated_at`` timestamp acts as a lease version: if a reaper has reclaimed the row while @@ -230,8 +225,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: def recover_stale_webmentions() -> int: - """ - Requeue verification attempts that were abandoned or lost from the queue. + """Requeue verification attempts that were abandoned or lost from the queue. Recover both stale ``verifying`` rows (workers that died) and stale ``received`` rows (tasks that were never enqueued or were lost). Conditional diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index bf79512..935f54b 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -39,8 +39,9 @@ def purge_old_received_webmentions() -> int: ) db.session.commit() - removed = result.rowcount - if removed: - current_app.logger.info("Purged %d old failed/deleted Webmentions", removed) + if result.rowcount: + current_app.logger.info( + "Purged %d old failed/deleted Webmentions", result.rowcount + ) - return removed + return result.rowcount diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 1b1c6ef..6365ff9 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -21,8 +21,7 @@ from .source_parser import ScannedSource, SourceScanError, scan_source_file def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: - """ - Create a source model from a newly discovered document. + """Create a source model from a newly discovered document. A sent Webmention is created for each discovered target at the initial source revision. @@ -52,8 +51,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: def update_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: - """ - Update an existing source from newly scanned data. + """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent Webmentions are updated to reflect the current targets, and newly discovered @@ -103,8 +101,7 @@ def update_source( def scan_sources() -> None: - """ - Scan generated source documents and queue pending Webmentions. + """Scan generated source documents and queue pending Webmentions. New and changed sources are persisted, missing sources are marked as deleted, and Webmentions requiring processing are queued for sending. A scan that @@ -157,7 +154,7 @@ def scan_sources_task() -> None: for index_path in root.rglob("index.html"): relative_path = index_path.relative_to(root) - if relative_path.parent == Path("."): + if relative_path.parent == Path(): continue seen_paths.add(relative_path.as_posix()) diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index a9db128..3803463 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -33,8 +33,7 @@ from .discovery import ( def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: - """ - Check whether a send attempt still represents the desired revision. + """Check whether a send attempt still represents the desired revision. The Webmention is refreshed from the database before comparing its desired and processed revisions. @@ -60,8 +59,7 @@ def retry_delay(attempt: int) -> timedelta: @huey.task() def send_webmention(webmention_uuid: uuid.UUID) -> None: - """ - Discover a receiver endpoint and send a Webmention. + """Discover a receiver endpoint and send a Webmention. The result of the attempt is stored on the corresponding sent Webmention. Temporary failures are recorded with an exponential retry deadline. The diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index cb15ab6..1bd498c 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -72,8 +72,7 @@ class ScannedSource: def source_url_for_path(relative_path: Path, base_url: str) -> str: - """ - Derive the public source URL from a relative filesystem path. + """Derive the public source URL from a relative filesystem path. :param relative_path: Path of the source document relative to the source root. :param base_url: Base URL under which source documents are published. @@ -87,8 +86,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: def canonical_url( document: BeautifulSoup, *, relative_path: Path, base_url: str | None ) -> str | None: - """ - Return the canonical URL declared by a source document. + """Return the canonical URL declared by a source document. Relative canonical URLs are resolved against the configured source base URL. @@ -122,11 +120,11 @@ def canonical_url( ) return resolved + return None def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: - """ - Yield URL values from a microformats property. + """Yield URL values from a microformats property. :param entry: Parsed microformats entry. :param property_name: Name of the property to inspect. @@ -146,8 +144,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: - """ - Parse and validate an h-entry with mf2py. + """Parse and validate an h-entry with mf2py. :param element: HTML element containing the h-entry. :param base_url: Base URL used when parsing the microformats data. @@ -180,8 +177,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: def primary_entry( document: BeautifulSoup, source_url: str ) -> tuple[Tag, dict[str, Any]]: - """ - Return and validate the primary h-entry of a source document. + """Return and validate the primary h-entry of a source document. The document must contain exactly one h-entry whose ``u-url`` matches the source URL. @@ -211,8 +207,7 @@ def primary_entry( def content_element(entry: Tag) -> Tag: - """ - Return the e-content element of an h-entry. + """Return the e-content element of an h-entry. :param entry: HTML element containing the h-entry. :return: The h-entry's e-content element. @@ -228,8 +223,7 @@ def content_element(entry: Tag) -> Tag: def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: - """ - Resolve and validate a possible Webmention target URL. + """Resolve and validate a possible Webmention target URL. Empty values, non-HTTP URLs, and URLs referring to the source itself are discarded. @@ -240,9 +234,7 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non :return: Normalized target URL, or ``None`` if the value is not a valid Webmention target. """ - value = value.strip() - - if not value: + if not (value := value.strip()): return None target = urljoin(base_url, value) @@ -264,8 +256,7 @@ def iter_targets( source_url: str, ignored_hostnames: Iterable[str] = (), ) -> Iterator[str]: - """ - Yield outgoing Webmention targets from an h-entry. + """Yield outgoing Webmention targets from an h-entry. Targets are collected from links in the entry content and supported microformats reaction properties. Invalid, self-referencing, and ignored @@ -328,8 +319,7 @@ def scan_source_file( base_url: str | None, ignored_hostnames: Iterable[str] = (), ) -> ScannedSource: - """ - Scan one generated source document for outgoing Webmentions. + """Scan one generated source document for outgoing Webmentions. :param path: Path of the generated HTML document. :param root: Root directory containing generated source documents. |
