diff options
Diffstat (limited to '')
| -rw-r--r-- | webmentions_ssg/tasks/consumer.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/discovery.py | 62 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/extension.py | 58 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/json_serializer.py | 2 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/notifications.py | 6 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/receiver.py | 72 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 48 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/sender.py | 78 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/source_parser.py | 53 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/target_matching.py (renamed from webmentions_ssg/tasks/mentions.py) | 0 |
10 files changed, 202 insertions, 183 deletions
diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index 13334bc..c24a883 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -22,9 +22,9 @@ def create_consumer() -> Huey: app = create_app() for config_key, task in ( - ("SCANNER_SCHEDULE", scanner.scan_sources), - ("VERIFICATION_REAPER_SCHEDULE", receiver.recover_stale_webmentions), - ("NOTIFICATION_SCHEDULE", notifications.send_webmention_digest), + ("SOURCE_SCAN_SCHEDULE", scanner.scan_sources), + ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications), + ("DIGEST_SCHEDULE", notifications.send_webmention_digest), ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), ): schedule = crontab(*app.config[config_key].split(), strict=True) diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py index 6d01e53..c683cb0 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/discovery.py @@ -14,48 +14,48 @@ from flask import current_app from ..net import ( ResponseTooLargeError, - content_type_of, is_http_url, - is_temporary_status, + is_retryable_status, + media_type_of, read_limited_body, ) -LINK_SPLIT = re.compile(r",\s*(?=<)") +LINK_HEADER_SPLIT = re.compile(r",\s*(?=<)") DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} -class SenderError(Exception): +class DeliveryError(Exception): """A Webmention cannot be sent.""" -class TemporarySenderError(SenderError): +class RetryableDeliveryError(DeliveryError): """A Webmention send failed for a potentially temporary reason.""" -class PermanentSenderError(SenderError): +class PermanentDeliveryError(DeliveryError): """A Webmention send failed permanently for this source revision.""" -def resolve_endpoint(response: httpx.Response, href: str) -> str: +def resolve_endpoint_url(response: httpx.Response, href: str) -> str: """Resolve and validate a discovered Webmention endpoint. :param response: Response from which the endpoint was discovered. :param href: Endpoint reference to resolve. :returns: Absolute Webmention endpoint URL. - :raises PermanentSenderError: If the resolved endpoint is not an HTTP or HTTPS URL. + :raises PermanentDeliveryError: If the resolved endpoint is not an HTTP or HTTPS URL. """ try: endpoint = urljoin(str(response.url), href.strip()) except ValueError as exc: - raise PermanentSenderError(f"Invalid Webmention endpoint: {href!r}") from exc + raise PermanentDeliveryError(f"Invalid Webmention endpoint: {href!r}") from exc if not is_http_url(endpoint): - raise PermanentSenderError(f"Invalid Webmention endpoint: {endpoint!r}") + raise PermanentDeliveryError(f"Invalid Webmention endpoint: {endpoint!r}") return endpoint -def parse_link_value(value: str) -> tuple[str, set[str]] | None: +def parse_link_header_value(value: str) -> tuple[str, set[str]] | None: """Parse a Link header value into its target and relations. :param value: Link header value to parse. @@ -84,7 +84,7 @@ def parse_link_value(value: str) -> tuple[str, set[str]] | None: return href, relations -def endpoint_from_headers(response: httpx.Response) -> str | None: +def find_endpoint_in_headers(response: httpx.Response) -> str | None: """Find a Webmention endpoint in the response Link headers. :param response: HTTP response whose headers should be inspected. @@ -92,22 +92,22 @@ def endpoint_from_headers(response: httpx.Response) -> str | None: Invalid candidates are ignored. """ for header in response.headers.get_list("Link"): - for value in LINK_SPLIT.split(header): - if (link := parse_link_value(value)) is None: + for value in LINK_HEADER_SPLIT.split(header): + if (link := parse_link_header_value(value)) is None: continue href, relations = link if "webmention" in relations: try: - return resolve_endpoint(response, href) - except PermanentSenderError: + return resolve_endpoint_url(response, href) + except PermanentDeliveryError: continue return None -def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: +def find_endpoint_in_html(response: httpx.Response, body: bytes) -> str | None: """Find a Webmention endpoint in an HTML document. ``link`` and ``a`` elements are inspected in document order. @@ -123,8 +123,8 @@ def endpoint_from_html(response: httpx.Response, body: bytes) -> str | None: 'link[rel~="webmention" i][href], a[rel~="webmention" i][href]' ): try: - return resolve_endpoint(response, str(element["href"])) - except PermanentSenderError: + return resolve_endpoint_url(response, str(element["href"])) + except PermanentDeliveryError: continue return None @@ -135,14 +135,14 @@ def read_target_body(response: httpx.Response) -> bytes: :param response: HTTP response to read with the configured target-size limit. :returns: Response body, provided it is within the target-size limit. - :raises PermanentSenderError: If the target document is too large. + :raises PermanentDeliveryError: If the target document is too large. """ try: return read_limited_body( - response, current_app.config["MAX_TARGET_BYTES"], label="Target" + response, current_app.config["MAX_DISCOVERY_TARGET_BYTES"], label="Target" ) except ResponseTooLargeError as exc: - raise PermanentSenderError(str(exc)) from exc + raise PermanentDeliveryError(str(exc)) from exc def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: @@ -155,8 +155,8 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non :param client: HTTP client to use for discovery requests. :param target: Target URL whose Webmention endpoint should be discovered. :returns: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises TemporarySenderError: If the target returns a temporary HTTP failure. - :raises PermanentSenderError: If GET returns a permanent HTTP failure. + :raises RetryableDeliveryError: If the target returns a temporary HTTP failure. + :raises PermanentDeliveryError: If GET returns a permanent HTTP failure. """ try: head_response = client.head(target, headers=DISCOVERY_HEADERS) @@ -165,25 +165,25 @@ def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | Non # or drop it even when fetching the document with GET would succeed. pass else: - if endpoint := endpoint_from_headers(head_response): + if endpoint := find_endpoint_in_headers(head_response): return endpoint with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := endpoint_from_headers(response): + if endpoint := find_endpoint_in_headers(response): return endpoint - if is_temporary_status(response.status_code): - raise TemporarySenderError(f"Target returned HTTP {response.status_code}") + if is_retryable_status(response.status_code): + raise RetryableDeliveryError(f"Target returned HTTP {response.status_code}") if not response.is_success: - raise PermanentSenderError(f"Target returned HTTP {response.status_code}") + raise PermanentDeliveryError(f"Target returned HTTP {response.status_code}") - media_type = content_type_of(response.headers) + media_type = media_type_of(response.headers) if media_type not in {"text/html", "application/xhtml+xml"}: return None - return endpoint_from_html(response, read_target_body(response)) + return find_endpoint_in_html(response, read_target_body(response)) def post_webmention( diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index 58e0036..faed364 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -15,10 +15,12 @@ from flask import Flask from huey import Huey as BaseHuey from huey.api import TaskWrapper -from .json_serializer import JSONSerializer +from .json_serializer import TaggedJSONSerializer -def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]]: +def _redis_class_and_options( + parsed: SplitResult, +) -> tuple[type[BaseHuey], dict[str, Any]]: """Select a Redis Huey flavour while retaining the underlying URL scheme. :param parsed: Parsed Redis backend URL, including any flavour suffix. @@ -47,7 +49,9 @@ def _redis_backend(parsed: SplitResult) -> tuple[type[BaseHuey], dict[str, Any]] return backends[flavour], {"url": fixed_url} -def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any]]: +def _file_class_and_options( + url: str, scheme: str +) -> tuple[type[BaseHuey], dict[str, Any]]: """Extract a relative or absolute SQLite/file storage path. :param url: Backend URL containing the storage filename or path. @@ -86,7 +90,7 @@ def _path_backend(url: str, scheme: str) -> tuple[type[BaseHuey], dict[str, Any] return backend, {option: path} -class _LazyTask: +class _DeferredTask: """Stable public task handle which follows the extension's current backend. Huey creates task classes and registers them when its decorator is applied. @@ -111,7 +115,7 @@ class _LazyTask: self._registered: TaskWrapper | None = None update_wrapper(self, func) - def register(self) -> None: + def bind(self) -> None: """Bind the original function to the latest initialized backend. Register the task decorator again so its stable handle uses the currently @@ -120,10 +124,10 @@ class _LazyTask: backend = self._extension.huey decorator = backend.periodic_task if self._periodic else backend.task self._registered = decorator(*self._args, **self._kwargs)( - self._extension._in_app_context(self._func) + self._extension._with_app_context(self._func) ) - def _current(self) -> TaskWrapper: + def _bound_task(self) -> TaskWrapper: if self._registered is None: raise RuntimeError( "Huey has not been initialized. Call huey.init_app(app) first." @@ -137,7 +141,7 @@ class _LazyTask: :param kwargs: Keyword arguments passed to the queued task. :returns: Result of enqueueing the task on the current backend. """ - return self._current()(*args, **kwargs) + return self._bound_task()(*args, **kwargs) def __getattr__(self, name: str) -> Any: """Delegate Huey task operations (call_local, schedule, s, etc.). @@ -145,7 +149,7 @@ class _LazyTask: :param name: Huey task attribute to delegate. :returns: Requested attribute of the currently bound Huey task. """ - return getattr(self._current(), name) + return getattr(self._bound_task(), name) class Huey: @@ -162,7 +166,7 @@ class Huey: """ self.app: Flask | None = None self._huey: BaseHuey | None = None - self._tasks: list[_LazyTask] = [] + self._task_handles: list[_DeferredTask] = [] if app is not None: self.init_app(app) @@ -192,7 +196,7 @@ class Huey: raise RuntimeError( "HUEY.serializer cannot be overridden; JSON is required." ) - config["serializer"] = JSONSerializer( + config["serializer"] = TaggedJSONSerializer( compression=config.pop("compression", False), use_zlib=config.pop("use_zlib", False), ) @@ -205,8 +209,8 @@ class Huey: # Import order is immaterial: tasks declared before initialization are # registered now, and previously imported handles follow this backend. - for task in self._tasks: - task.register() + for task in self._task_handles: + task.bind() app.extensions["huey"] = self @@ -224,7 +228,7 @@ class Huey: ) return self._huey - def _in_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]: + def _with_app_context[**P, R](self, func: Callable[P, R]) -> Callable[P, R]: """Return a task wrapper that executes with the configured Flask app. :param func: Task function to wrap in a Flask application context. @@ -241,23 +245,23 @@ class Huey: return wrapped - def _decorate_task[**P, R]( + def _register_task[**P, R]( self, func: Callable[P, R], task_args: tuple[Any, ...], task_kwargs: dict[str, Any], *, periodic: bool, - ) -> _LazyTask: - handle = _LazyTask(self, func, task_args, task_kwargs, periodic=periodic) - self._tasks.append(handle) + ) -> _DeferredTask: + handle = _DeferredTask(self, func, task_args, task_kwargs, periodic=periodic) + self._task_handles.append(handle) if self._huey is not None: - handle.register() + handle.bind() return handle def task[**P, R]( self, *task_args: Any, **task_kwargs: Any - ) -> Callable[[Callable[P, R]], _LazyTask]: + ) -> Callable[[Callable[P, R]], _DeferredTask]: """Declare a task even before :meth:`init_app` has run. :param task_args: Positional arguments for the Huey task decorator. @@ -265,14 +269,14 @@ class Huey: :returns: Decorator that registers and returns a lazily bound task. """ - def decorator(func: Callable[P, R]) -> _LazyTask: - return self._decorate_task(func, task_args, task_kwargs, periodic=False) + def decorator(func: Callable[P, R]) -> _DeferredTask: + return self._register_task(func, task_args, task_kwargs, periodic=False) return decorator def periodic_task[**P, R]( self, *task_args: Any, **task_kwargs: Any - ) -> Callable[[Callable[P, R]], _LazyTask]: + ) -> Callable[[Callable[P, R]], _DeferredTask]: """Declare a periodic task against the current or next backend. :param task_args: Positional arguments for the Huey periodic-task decorator. @@ -280,8 +284,8 @@ class Huey: :returns: Decorator that registers and returns a lazily bound periodic task. """ - def decorator(func: Callable[P, R]) -> _LazyTask: - return self._decorate_task(func, task_args, task_kwargs, periodic=True) + def decorator(func: Callable[P, R]) -> _DeferredTask: + return self._register_task(func, task_args, task_kwargs, periodic=True) return decorator @@ -309,9 +313,9 @@ class Huey: scheme = parsed.scheme.lower() if scheme.partition("+")[0] in {"redis", "rediss"}: - return _redis_backend(parsed) + return _redis_class_and_options(parsed) if scheme in {"sqlite", "file"}: - return _path_backend(url, scheme) + return _file_class_and_options(url, scheme) if scheme in {"postgres", "postgresql"}: from huey import PostgresHuey diff --git a/webmentions_ssg/tasks/json_serializer.py b/webmentions_ssg/tasks/json_serializer.py index 7e51358..804cfee 100644 --- a/webmentions_ssg/tasks/json_serializer.py +++ b/webmentions_ssg/tasks/json_serializer.py @@ -14,7 +14,7 @@ from huey.serializer import Serializer from huey.utils import SKIPPED, Error -class JSONSerializer(Serializer): +class TaggedJSONSerializer(Serializer): """A deliberately restricted, non-pickle serializer for our Huey tasks. JSON has no native UUID, datetime, tuple or Huey message representation. diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index b1762d5..eb540de 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -22,16 +22,16 @@ def send_webmention_digest() -> None: """ # Resolve the lock against the current Huey backend, not the import-time one. with huey.lock_task("send-webmention-digest"): - _send_webmention_digest() + _send_digest_locked() -def _send_webmention_digest() -> None: +def _send_digest_locked() -> None: """Do the work while the current backend's digest lock is held. Collect verified mentions, send the configured message, and persist notification timestamps. """ - recipient = current_app.config.get("NOTIFICATION_EMAIL") + recipient = current_app.config.get("DIGEST_EMAIL") if not recipient: return diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py index eb228b3..313a8e4 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/receiver.py @@ -20,13 +20,13 @@ from ..net import ( AddressResolutionError, ResponseTooLargeError, UnsafeAddressError, - build_client, - content_type_of, - is_temporary_status, + build_public_http_client, + is_retryable_status, + media_type_of, read_limited_body, ) from ..version import APP_NAME, VERSION -from .mentions import html_mentions_target, text_mentions_target +from .target_matching import html_mentions_target, text_mentions_target class VerificationError(Exception): @@ -37,14 +37,14 @@ class SourceGoneError(VerificationError): """The source explicitly reports that it has been removed.""" -class TemporaryFetchError(Exception): +class RetryableFetchError(Exception): """Fetching the source may succeed when retried later.""" RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" -def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: +def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: """Fetch a webmention source document. The request follows redirects while enforcing the configured timeout, @@ -53,13 +53,13 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: :param source_url: URL of the source document. :returns: HTTP response and response body. :raises SourceGoneError: If the source returns HTTP 410. - :raises TemporaryFetchError: If the source returns a temporary HTTP error. + :raises RetryableFetchError: If the source returns a temporary HTTP error. :raises VerificationError: If the source returns another unsuccessful HTTP status or exceeds the configured maximum response size. """ try: with ( - build_client( + build_public_http_client( f"{APP_NAME}/{VERSION} ReceivedWebmention", accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", ) as client, @@ -69,8 +69,8 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: pass elif response.status_code == HTTPStatus.GONE: raise SourceGoneError("Source returned HTTP 410") - elif is_temporary_status(response.status_code): - raise TemporaryFetchError( + elif is_retryable_status(response.status_code): + raise RetryableFetchError( f"Source returned HTTP {response.status_code}" ) else: @@ -78,7 +78,9 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: try: return response, read_limited_body( - response, current_app.config["MAX_SOURCE_BYTES"], label="Source" + response, + current_app.config["MAX_RECEIVED_SOURCE_BYTES"], + label="Source", ) except ResponseTooLargeError as exc: raise VerificationError(str(exc)) from exc @@ -86,7 +88,7 @@ def fetch_source(source_url: str) -> tuple[httpx.Response, bytes]: except UnsafeAddressError as exc: raise VerificationError("Source resolves to a non-public address") from exc except AddressResolutionError as exc: - raise TemporaryFetchError("Source hostname could not be resolved") from exc + raise RetryableFetchError("Source hostname could not be resolved") from exc def source_mentions_target(source_url: str, target_url: str) -> bool: @@ -100,11 +102,11 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: :returns: Whether the source mentions the target URL. :raises VerificationError: If the source has an unsupported or missing content type. :raises SourceGoneError: If the source explicitly reports that it is gone. - :raises TemporaryFetchError: If fetching the source fails temporarily. + :raises RetryableFetchError: If fetching the source fails temporarily. """ - response, body = fetch_source(source_url) + response, body = fetch_source_document(source_url) - media_type = content_type_of(response.headers) + media_type = media_type_of(response.headers) match media_type: case "text/html" | "application/xhtml+xml": @@ -127,13 +129,13 @@ def source_mentions_target(source_url: str, target_url: str) -> bool: class VerificationClaim: """Source and lease version for one atomically claimed verification.""" - identifier: uuid.UUID + webmention_uuid: uuid.UUID source: str target: str - started_at: datetime + lease_started_at: datetime -def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: +def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: """Claim an eligible row, returning its source, target, and lease version. :param webmention_uuid: Identifier of the received Webmention to claim. @@ -157,10 +159,10 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: return None claim = VerificationClaim( - identifier=webmention_uuid, + webmention_uuid=webmention_uuid, source=webmention.source, target=webmention.target, - started_at=datetime.now(UTC), + lease_started_at=datetime.now(UTC), ) changed = db.session.execute( sa.update(ReceivedWebmention) @@ -172,7 +174,7 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: .values( status=ReceivedStatus.VERIFYING, failure_reason=None, - updated_at=claim.started_at, + updated_at=claim.lease_started_at, ) ) @@ -184,12 +186,14 @@ def claim_eligible_row(webmention_uuid: uuid.UUID) -> VerificationClaim | None: return claim -def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]: +def determine_verification_outcome( + claim: VerificationClaim, +) -> tuple[ReceivedStatus, str | None]: """Map verification results and permanent errors to a final status/reason. :param claim: Claim containing the source and target URLs to verify. :returns: Final verification status and optional reason. - :raises TemporaryFetchError: If the source fetch fails transiently and should be + :raises RetryableFetchError: If the source fetch fails transiently and should be retried. :raises httpx.RequestError: If a transient HTTP request error should be retried. """ @@ -199,12 +203,12 @@ def check_source(claim: VerificationClaim) -> tuple[ReceivedStatus, str | None]: return ReceivedStatus.DELETED, str(exc) except VerificationError as exc: return ReceivedStatus.FAILED, str(exc) - except TemporaryFetchError, httpx.RequestError: + except RetryableFetchError, httpx.RequestError: raise # These errors must be persisted and then retried by Huey. except Exception as exc: # InvalidURL is not a RequestError; unexpected source errors are terminal. current_app.logger.exception( - "Unexpected error verifying ReceivedWebmention %s", claim.identifier + "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid ) return ReceivedStatus.FAILED, str(exc) or type(exc).__name__ @@ -226,9 +230,9 @@ def release_claim( changed = db.session.execute( sa.update(ReceivedWebmention) .where( - ReceivedWebmention.uuid == claim.identifier, + ReceivedWebmention.uuid == claim.webmention_uuid, ReceivedWebmention.status == ReceivedStatus.VERIFYING, - ReceivedWebmention.updated_at == claim.started_at, + ReceivedWebmention.updated_at == claim.lease_started_at, ) .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) ) @@ -246,15 +250,15 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: lease. :param webmention_uuid: Identifier of the received webmention to verify. - :raises TemporaryFetchError: If fetching the source fails temporarily. + :raises RetryableFetchError: If fetching the source fails temporarily. :raises httpx.RequestError: If an HTTP request fails while fetching. """ - if (claim := claim_eligible_row(webmention_uuid)) is None: + if (claim := claim_for_verification(webmention_uuid)) is None: return try: - status, reason = check_source(claim) - except (TemporaryFetchError, httpx.RequestError) as exc: + status, reason = determine_verification_outcome(claim) + except (RetryableFetchError, httpx.RequestError) as exc: release_claim( claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched" ) @@ -263,7 +267,7 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: release_claim(claim, status, reason) -def recover_stale_webmentions() -> int: +def recover_stale_verifications() -> int: """Requeue verification attempts that were abandoned or lost from the queue. Recover both stale ``verifying`` rows (workers that died) and stale @@ -280,7 +284,9 @@ def recover_stale_webmentions() -> int: :returns: Number of webmentions requeued. """ now = datetime.now(UTC) - cutoff = now - timedelta(seconds=current_app.config["VERIFICATION_TIMEOUT_SECONDS"]) + cutoff = now - timedelta( + seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"] + ) stale = db.session.execute( sa.select( diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 13d2ceb..034b2cf 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -20,15 +20,15 @@ from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file -def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: - """Create a source model from a newly discovered document. +def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: + """Build a source model from a newly discovered document. - A sent Webmention is created for each discovered target at the initial source + A sent Webmention is built for each discovered target at the initial source revision. :param scanned: Scanned source data. :param scan_time: Time at which the source was discovered. - :returns: Newly created source model. + :returns: Newly built source model. """ source = Source( path=scanned.path, @@ -48,7 +48,7 @@ def create_source(scanned: ScannedSource, scan_time: datetime) -> Source: return source -def update_source( +def apply_scan_to_source( source: Source, scanned: ScannedSource, scan_time: datetime ) -> Source: """Update an existing source from newly scanned data. @@ -113,24 +113,24 @@ def scan_sources() -> None: """ # Manual and periodic scans must contend for the current backend's lock. with huey.lock_task("scan-webmention-sources"): - _scan_locked() + _scan_and_enqueue() @dataclass(frozen=True) -class ScanResult: +class TreeScanResult: """Count and discovered paths from a traversal of the source tree.""" scanned_count: int seen_paths: frozenset[str] -def scan_tree( +def reconcile_source_tree( root: Path, base_url: str | None, sources_by_path: dict[str, Source], paths_by_url: dict[str, str], scan_time: datetime, -) -> ScanResult: +) -> TreeScanResult: """Reconcile found pages and return the paths seen, even on parse failures. :param root: Root of the generated static-site output tree. @@ -157,7 +157,7 @@ def scan_tree( index_path, root=root, base_url=base_url, - ignored_hostnames=current_app.config["IGNORED_HOSTNAMES"], + ignored_hostnames=current_app.config["IGNORED_TARGET_HOSTNAMES"], ) except (OSError, SourceScanError) as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) @@ -182,10 +182,10 @@ def scan_tree( try: if (source := sources_by_path.get(scanned.path)) is None: - source = create_source(scanned, scan_time) + source = build_source(scanned, scan_time) db.session.add(source) else: - source = update_source(source, scanned, scan_time) + source = apply_scan_to_source(source, scanned, scan_time) sources_by_path[source.path] = source paths_by_url[source.url] = source.path @@ -193,10 +193,10 @@ def scan_tree( except SourceScanError as exc: current_app.logger.warning("Could not scan source %s: %s", index_path, exc) - return ScanResult(scanned_count, frozenset(seen_paths)) + return TreeScanResult(scanned_count, frozenset(seen_paths)) -def mark_missing( +def mark_missing_sources_deleted( sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime ) -> int: """Mark absent, previously active sources deleted and return their count. @@ -225,7 +225,7 @@ def mark_missing( return marked -def queue_pending(scan_time: datetime) -> int: +def enqueue_due_sends(scan_time: datetime) -> int: """Enqueue due Webmentions and return the number of tasks submitted. :param scan_time: Current scan timestamp used to evaluate retry deadlines. @@ -249,7 +249,7 @@ def queue_pending(scan_time: datetime) -> int: return queued -def _scan_locked() -> None: +def _scan_and_enqueue() -> None: """Scan sources while the caller holds the shared scan lock. Reconcile the generated source tree, persist changes, and enqueue eligible outgoing @@ -257,7 +257,7 @@ def _scan_locked() -> None: :raises RuntimeError: If the source directory or configured base URL is invalid. """ - directory = current_app.config.get("SOURCE_DIRECTORY") + directory = current_app.config.get("SITE_OUTPUT_DIRECTORY") if directory is None: return @@ -265,9 +265,9 @@ def _scan_locked() -> None: if not root.is_dir(): raise RuntimeError(f"Source directory is not a directory: {root}") - if (base_url := current_app.config.get("SOURCE_BASE_URL")) is not None: + if (base_url := current_app.config.get("SITE_BASE_URL")) is not None: if not is_http_url(base_url): - raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") + raise RuntimeError("SITE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" sources_by_path = { @@ -281,7 +281,9 @@ def _scan_locked() -> None: # take ownership merely by appearing earlier in the traversal. paths_by_url = {source.url: source.path for source in sources_by_path.values()} scan_time = datetime.now(UTC) - scan = scan_tree(root, base_url, sources_by_path, paths_by_url, scan_time) + scan = reconcile_source_tree( + root, base_url, sources_by_path, paths_by_url, scan_time + ) if scan.scanned_count == 0: current_app.logger.warning( @@ -290,9 +292,11 @@ def _scan_locked() -> None: ) return - missing_count = mark_missing(sources_by_path, scan.seen_paths, scan_time) + missing_count = mark_missing_sources_deleted( + sources_by_path, scan.seen_paths, scan_time + ) db.session.commit() - pending_count = queue_pending(scan_time) + pending_count = enqueue_due_sends(scan_time) current_app.logger.info( "Webmention source scan complete: %d sources scanned, " diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index cf8ab9d..c33cf06 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -19,20 +19,20 @@ from ..models import SentWebmention, SentWebmentionStatus from ..net import ( AddressResolutionError, UnsafeAddressError, - build_client, + build_public_http_client, is_http_url, - is_temporary_status, + is_retryable_status, ) from ..version import APP_NAME, VERSION from .discovery import ( - PermanentSenderError, - TemporarySenderError, + PermanentDeliveryError, + RetryableDeliveryError, discover_webmention_endpoint, post_webmention, ) -def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: +def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """Check whether a send attempt still represents the desired revision. The Webmention is refreshed from the database before comparing its desired @@ -47,7 +47,7 @@ def attempt_is_current(webmention: SentWebmention, revision: int) -> bool: return webmention.desired_revision == revision and webmention.pending -def retry_delay(attempt: int) -> timedelta: +def backoff_delay(attempt: int) -> timedelta: """Calculate the capped exponential delay after a numbered send attempt. :param attempt: One-based send attempt number. @@ -76,7 +76,7 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: # The pending/retry checks must happen *under* this lock. try: with huey.lock_task(f"send-webmention-{webmention_uuid}"): - _send_locked(webmention_uuid) + _process_send_locked(webmention_uuid) except TaskLockedException: # Another worker is already sending this Webmention. The active worker # owns its result; a later scan will requeue it if it remains pending. @@ -84,12 +84,12 @@ def send_webmention(webmention_uuid: uuid.UUID) -> None: @dataclass(frozen=True) -class SendAttempt: +class ClaimedSendAttempt: """Claimed source revision and retry budget for one send.""" - identifier: uuid.UUID + webmention_uuid: uuid.UUID revision: int - number: int + attempt_number: int max_attempts: int source: str target: str @@ -112,7 +112,7 @@ class DeliveryResult: temporary_failure: bool = False -def is_due(webmention: SentWebmention, now: datetime) -> bool: +def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: """Return whether an unprocessed Webmention has reached its retry deadline. :param webmention: Outgoing Webmention whose retry eligibility is checked. @@ -130,7 +130,9 @@ def is_due(webmention: SentWebmention, now: datetime) -> bool: return aware_deadline <= now -def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | None: +def reserve_send_attempt( + webmention: SentWebmention, now: datetime +) -> ClaimedSendAttempt | None: """Atomically reserve one send attempt, or finish an exhausted revision. An attempt's next retry deadline is persisted *before* the HTTP request so @@ -172,10 +174,10 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No db.session.commit() return None - attempt = SendAttempt( - identifier=webmention.uuid, + attempt = ClaimedSendAttempt( + webmention_uuid=webmention.uuid, revision=revision, - number=webmention.attempt_count + 1, + attempt_number=webmention.attempt_count + 1, max_attempts=max_attempts, source=webmention.source.url, target=webmention.target, @@ -183,9 +185,9 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No changed = db.session.execute( sa.update(SentWebmention) .where( - SentWebmention.uuid == attempt.identifier, + SentWebmention.uuid == attempt.webmention_uuid, SentWebmention.desired_revision == revision, - SentWebmention.attempt_count == attempt.number - 1, + SentWebmention.attempt_count == attempt.attempt_number - 1, SentWebmention.pending, sa.or_( SentWebmention.next_retry_at.is_(None), @@ -193,9 +195,9 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No ), ) .values( - attempt_count=attempt.number, + attempt_count=attempt.attempt_number, last_attempted_at=now, - next_retry_at=now + retry_delay(attempt.number), + next_retry_at=now + backoff_delay(attempt.attempt_number), endpoint=None, response_status=None, status_url=None, @@ -211,7 +213,7 @@ def claim_attempt(webmention: SentWebmention, now: datetime) -> SendAttempt | No return attempt -def deliver_webmention( +def attempt_delivery( source: str, target: str, *, webmention_uuid: uuid.UUID ) -> DeliveryResult: """Discover and POST a Webmention, returning an outcome without DB writes. @@ -227,9 +229,9 @@ def deliver_webmention( try: if not is_http_url(target): - raise PermanentSenderError(f"Invalid target URL: {target!r}") + raise PermanentDeliveryError(f"Invalid target URL: {target!r}") - with build_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: + with build_public_http_client(f"{APP_NAME}/{VERSION} SentWebmention") as client: endpoint = discover_webmention_endpoint(client, target) if endpoint is None: @@ -243,9 +245,9 @@ def deliver_webmention( ) if not 200 <= response_status <= 299: message = f"Webmention endpoint returned HTTP {response_status}" - if is_temporary_status(response_status): - raise TemporarySenderError(message) - raise PermanentSenderError(message) + if is_retryable_status(response_status): + raise RetryableDeliveryError(message) + raise PermanentDeliveryError(message) return DeliveryResult( status=SentWebmentionStatus.SENT, @@ -254,7 +256,7 @@ def deliver_webmention( status_url=status_url, ) except ( - PermanentSenderError, + PermanentDeliveryError, UnsafeAddressError, httpx.InvalidURL, httpx.UnsupportedProtocol, @@ -267,7 +269,7 @@ def deliver_webmention( endpoint=endpoint, response_status=response_status, ) - except (TemporarySenderError, AddressResolutionError, httpx.RequestError) as exc: + except (RetryableDeliveryError, AddressResolutionError, httpx.RequestError) as exc: return DeliveryResult( status=SentWebmentionStatus.FAILED, reason=str(exc) or "Webmention request failed", @@ -289,9 +291,9 @@ def deliver_webmention( ) -def record_outcome( +def record_delivery_outcome( webmention: SentWebmention, - attempt: SendAttempt, + attempt: ClaimedSendAttempt, result: DeliveryResult, *, terminal: bool, @@ -305,7 +307,7 @@ def record_outcome( :returns: ``True`` if the outcome was recorded for the current revision. """ # A scanner may have reopened or superseded the revision during I/O. - if not attempt_is_current(webmention, attempt.revision): + if not revision_still_pending(webmention, attempt.revision): return False webmention.status = result.status @@ -330,7 +332,7 @@ def record_outcome( return True -def _send_locked(webmention_uuid: uuid.UUID) -> bool: +def _process_send_locked(webmention_uuid: uuid.UUID) -> bool: """Process a due Webmention while holding its per-UUID task lock. :param webmention_uuid: Identifier of the outgoing Webmention being processed. @@ -344,18 +346,18 @@ def _send_locked(webmention_uuid: uuid.UUID) -> bool: return False now = datetime.now(UTC) - if not is_due(webmention, now): + if not is_ready_to_send(webmention, now): return False - if (attempt := claim_attempt(webmention, now)) is None: + if (attempt := reserve_send_attempt(webmention, now)) is None: return False - result = deliver_webmention( + result = attempt_delivery( attempt.source, attempt.target, webmention_uuid=webmention_uuid ) - terminal = result.terminal or attempt.number >= attempt.max_attempts + terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts - if not record_outcome(webmention, attempt, result, terminal=terminal): + if not record_delivery_outcome(webmention, attempt, result, terminal=terminal): return False if result.temporary_failure and not terminal: @@ -363,13 +365,13 @@ def _send_locked(webmention_uuid: uuid.UUID) -> bool: current_app.logger.info( "Webmention %s send attempt %d failed; will retry", webmention_uuid, - attempt.number, + attempt.attempt_number, ) elif result.temporary_failure: current_app.logger.warning( "Webmention %s exhausted %d send attempts: %s", webmention_uuid, - attempt.number, + attempt.attempt_number, result.reason, ) diff --git a/webmentions_ssg/tasks/source_parser.py b/webmentions_ssg/tasks/source_parser.py index 31d0c98..e063525 100644 --- a/webmentions_ssg/tasks/source_parser.py +++ b/webmentions_ssg/tasks/source_parser.py @@ -18,11 +18,11 @@ import xxhash from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement -from ..net import hostname_of, is_http_url +from ..net import is_http_url, normalized_hostname REACTION_PROPERTIES = ("in-reply-to", "like-of", "repost-of", "bookmark-of") -CONTENT_BLOCK_TAGS = frozenset( +BLOCK_LEVEL_TAGS = frozenset( { "address", "article", @@ -83,7 +83,7 @@ def source_url_for_path(relative_path: Path, base_url: str) -> str: return urljoin(base_url, f"{quote(directory, safe='/')}/") -def canonical_url( +def declared_canonical_url( document: BeautifulSoup, *, relative_path: Path, base_url: str | None ) -> str | None: """Return the canonical URL declared by a source document. @@ -111,7 +111,7 @@ def canonical_url( if base_url is None: raise SourceScanError( "Document contains a relative canonical URL, but " - "SOURCE_BASE_URL is not configured" + "SITE_BASE_URL is not configured" ) resolved = urljoin(source_url_for_path(relative_path, base_url), href) @@ -124,7 +124,7 @@ def canonical_url( return resolved -def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: +def mf2_string_values(entry: dict[str, Any], property_name: str) -> Iterator[str]: """Yield URL values from a microformats property. :param entry: Parsed microformats entry. @@ -144,7 +144,7 @@ def property_urls(entry: dict[str, Any], property_name: str) -> Iterator[str]: yield from (value for value in values if isinstance(value, str)) -def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: +def parse_h_entry(element: Tag, base_url: str) -> dict[str, Any]: """Parse and validate an h-entry with mf2py. :param element: HTML element containing the h-entry. @@ -175,7 +175,7 @@ def parse_entry(element: Tag, base_url: str) -> dict[str, Any]: return entry -def primary_entry( +def find_primary_h_entry( document: BeautifulSoup, source_url: str ) -> tuple[Tag, dict[str, Any]]: """Return and validate the primary h-entry of a source document. @@ -195,9 +195,9 @@ def primary_entry( raise SourceScanError(f"Expected exactly one h-entry, found {len(entries)}") entry_element = entries[0] - mf2_entry = parse_entry(entry_element, source_url) + mf2_entry = parse_h_entry(entry_element, source_url) - urls = tuple(property_urls(mf2_entry, "url")) + urls = tuple(mf2_string_values(mf2_entry, "url")) if source_url not in urls: raise SourceScanError( @@ -207,15 +207,15 @@ def primary_entry( return entry_element, mf2_entry -def content_element(entry: Tag) -> Tag: +def find_e_content(entry_element: Tag) -> Tag: """Return the e-content element of an h-entry. - :param entry: HTML element containing the h-entry. + :param entry_element: HTML element containing the h-entry. :returns: The h-entry's e-content element. :raises SourceScanError: If the h-entry does not contain exactly one e-content element. """ - contents = entry.find_all(class_="e-content") + contents = entry_element.find_all(class_="e-content") if len(contents) != 1: raise SourceScanError(f"Expected exactly one e-content, found {len(contents)}") @@ -223,7 +223,7 @@ def content_element(entry: Tag) -> Tag: return contents[0] -def normalize_target(value: str, *, base_url: str, source_url: str) -> str | None: +def normalize_target_url(value: str, *, base_url: str, source_url: str) -> str | None: """Resolve and validate a possible Webmention target URL. Empty values, non-HTTP URLs, and URLs referring to the source itself are @@ -249,7 +249,7 @@ def normalize_target(value: str, *, base_url: str, source_url: str) -> str | Non return target -def iter_targets( +def iter_outgoing_targets( content: Tag, mf2_entry: dict[str, Any], *, @@ -277,18 +277,19 @@ def iter_targets( ) reactions = chain.from_iterable( - property_urls(mf2_entry, property_name) for property_name in REACTION_PROPERTIES + mf2_string_values(mf2_entry, property_name) + for property_name in REACTION_PROPERTIES ) for value in chain(hrefs, reactions): - target = normalize_target(value, base_url=base_url, source_url=source_url) + target = normalize_target_url(value, base_url=base_url, source_url=source_url) if target is not None: - hostname = hostname_of(target) or "" + hostname = normalized_hostname(target) or "" if not any(fnmatchcase(hostname, pattern) for pattern in ignored_hostnames): yield target -def normalized_content(content: Tag) -> str: +def stable_content_text(content: Tag) -> str: """Extract stable reader-facing content without generated HTML attributes. :param content: Parsed e-content element to normalize. @@ -312,7 +313,7 @@ def normalized_content(content: Tag) -> str: return f" [image:{node.get('src', '')!s}:{node.get('alt', '')!s}] " text = "".join(extract(child) for child in node.children) - return f" {text} " if node.name in CONTENT_BLOCK_TAGS else text + return f" {text} " if node.name in BLOCK_LEVEL_TAGS else text return " ".join(extract(content).split()) @@ -346,22 +347,24 @@ def scan_source_file( relative_path = path.relative_to(root) document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") - source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) + source_url = declared_canonical_url( + document, relative_path=relative_path, base_url=base_url + ) if source_url is None: if base_url is None: raise SourceScanError( - "Document has no canonical URL and SOURCE_BASE_URL is not configured" + "Document has no canonical URL and SITE_BASE_URL is not configured" ) source_url = source_url_for_path(relative_path, base_url) - entry_element, mf2_entry = primary_entry(document, source_url) + entry_element, mf2_entry = find_primary_h_entry(document, source_url) - content = content_element(entry_element) + content = find_e_content(entry_element) targets = frozenset( - iter_targets( + iter_outgoing_targets( content, mf2_entry, base_url=source_url, @@ -370,7 +373,7 @@ def scan_source_file( ) ) - stable_content = normalized_content(content) + stable_content = stable_content_text(content) fingerprint = json.dumps( {"content": stable_content, "targets": sorted(targets)}, ensure_ascii=False, diff --git a/webmentions_ssg/tasks/mentions.py b/webmentions_ssg/tasks/target_matching.py index 4c3918b..4c3918b 100644 --- a/webmentions_ssg/tasks/mentions.py +++ b/webmentions_ssg/tasks/target_matching.py |
