From 9b00526ab79257b65cedd71595848f09ecf84279 Mon Sep 17 00:00:00 2001 From: Dennis Fink Date: Sun, 15 Nov 2026 21:18:19 +0100 Subject: refactor(core)!: clarify Webmention domain terminology Rename ambiguous models, statuses, task modules, fields, and helpers so names reflect source documents, verification, endpoint discovery, and delivery state. Migrate persisted status values and database identifiers without losing existing Webmention data. BREAKING CHANGE: persisted status values and database schema names have changed, and several public Python names have been renamed. --- webmentions_ssg/config.py | 2 +- webmentions_ssg/extensions.py | 4 +- webmentions_ssg/forms.py | 9 +- .../9a4e0c8211d6_clarify_webmention_names.py | 178 ++++++++++ .../d845aa7fbc19_portable_url_fingerprints.py | 3 +- webmentions_ssg/models.py | 88 ++--- webmentions_ssg/tasks/__init__.py | 4 +- webmentions_ssg/tasks/consumer.py | 12 +- webmentions_ssg/tasks/discovery.py | 217 ------------ webmentions_ssg/tasks/endpoint_discovery.py | 217 ++++++++++++ webmentions_ssg/tasks/extension.py | 6 +- webmentions_ssg/tasks/notifications.py | 6 +- webmentions_ssg/tasks/receiver.py | 358 -------------------- webmentions_ssg/tasks/retention.py | 7 +- webmentions_ssg/tasks/scanner.py | 38 ++- webmentions_ssg/tasks/sender.py | 38 +-- webmentions_ssg/tasks/verification.py | 363 +++++++++++++++++++++ webmentions_ssg/templates/received.html | 2 +- webmentions_ssg/templates/sent_source.html | 12 +- webmentions_ssg/templates/status.html | 2 +- webmentions_ssg/views/admin.py | 20 +- webmentions_ssg/views/endpoint.py | 9 +- 22 files changed, 898 insertions(+), 697 deletions(-) create mode 100644 webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py delete mode 100644 webmentions_ssg/tasks/discovery.py create mode 100644 webmentions_ssg/tasks/endpoint_discovery.py delete mode 100644 webmentions_ssg/tasks/receiver.py create mode 100644 webmentions_ssg/tasks/verification.py (limited to 'webmentions_ssg') diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 7d09e7e..15667ab 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -36,7 +36,7 @@ class DefaultConfig: MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. MAX_WEBMENTION_URL_BYTES = 8192 - # Purge terminal failed/deleted received Webmentions after 30 days. + # Purge terminal failed/source-removed received Webmentions after 30 days. RECEIVED_RETENTION_DAYS = 30 RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py index c1cf396..d515d24 100644 --- a/webmentions_ssg/extensions.py +++ b/webmentions_ssg/extensions.py @@ -13,7 +13,7 @@ from flask_sqlalchemy import SQLAlchemy from flask_wtf.csrf import CSRFProtect from sqlalchemy.orm import DeclarativeBase -from .tasks.extension import Huey +from .tasks.extension import FlaskHuey class Base(DeclarativeBase): @@ -24,6 +24,6 @@ csrf = CSRFProtect() bootstrap = Bootstrap5() db = SQLAlchemy(model_class=Base) migrate = Migrate() -huey = Huey() +huey = FlaskHuey() mail = Mail() limiter = Limiter(key_func=get_remote_address) diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py index c8ad0ba..1ce591d 100644 --- a/webmentions_ssg/forms.py +++ b/webmentions_ssg/forms.py @@ -104,7 +104,7 @@ class AllowedTargetHostname: raise ValidationError(self.message or field.gettext("Invalid input.")) -class PublicIPLiteral: +class RejectNonPublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" def __call__(self, form: BaseForm, field: Field) -> None: @@ -138,7 +138,12 @@ class WebmentionSubmissionForm(FlaskForm): source = StringField( "source", - validators=[InputRequired(), MaxURLBytes(), StrictHTTPURL(), PublicIPLiteral()], + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + RejectNonPublicIPLiteral(), + ], ) target = StringField( "target", diff --git a/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py b/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py new file mode 100644 index 0000000..23ff56a --- /dev/null +++ b/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Clarify source-document and Webmention lifecycle names. + +Revision ID: 9a4e0c8211d6 +Revises: 7c38bb1a935e +""" + +import sqlalchemy as sa +from alembic import op + +revision = "9a4e0c8211d6" +down_revision = "7c38bb1a935e" +branch_labels = None +depends_on = None + + +_OLD_RECEIVED_STATUS = sa.Enum( + "received", + "verifying", + "verified", + "failed", + "deleted", + name="received_webmention_status", + native_enum=False, +) +_NEW_RECEIVED_STATUS = sa.Enum( + "received", + "verifying", + "verified", + "failed", + "source_removed", + name="received_webmention_status", + native_enum=False, +) + + +def upgrade() -> None: + # RENAME TABLE retains existing source UUIDs and updates referencing FKs. + op.rename_table("sources", "source_documents") + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_url_digest", type_="unique") + batch.alter_column( + "url_digest", + new_column_name="source_target_digest", + existing_type=sa.String(64), + existing_nullable=False, + ) + # The new enum string is 14 characters, not the old VARCHAR(9). + batch.alter_column( + "status", + existing_type=_OLD_RECEIVED_STATUS, + type_=_NEW_RECEIVED_STATUS, + existing_nullable=False, + ) + + with op.batch_alter_table("received_webmentions") as batch: + batch.create_unique_constraint( + "uq_webmention_source_target_digest", ["source_target_digest"] + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_index("ix_sent_webmentions_source_id") + batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique") + batch.alter_column( + "source_id", + new_column_name="source_uuid", + existing_type=sa.Uuid(), + existing_nullable=False, + ) + batch.alter_column( + "active", + new_column_name="linked", + existing_type=sa.Boolean(), + existing_nullable=False, + ) + batch.alter_column( + "endpoint", + new_column_name="webmention_endpoint", + existing_type=sa.Text(), + existing_nullable=True, + ) + batch.alter_column( + "response_status", + new_column_name="endpoint_response_status", + existing_type=sa.Integer(), + existing_nullable=True, + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.create_unique_constraint( + "uq_sent_webmention_source_uuid_target_digest", + ["source_uuid", "target_digest"], + ) + + op.create_index( + "ix_sent_webmentions_source_uuid", "sent_webmentions", ["source_uuid"] + ) + + op.execute( + "UPDATE received_webmentions SET status = 'source_removed' " + "WHERE status = 'deleted'" + ) + op.execute( + "UPDATE sent_webmentions SET status = 'no_endpoint' " + "WHERE status = 'unsupported'" + ) + + +def downgrade() -> None: + op.execute( + "UPDATE received_webmentions SET status = 'deleted' " + "WHERE status = 'source_removed'" + ) + op.execute( + "UPDATE sent_webmentions SET status = 'unsupported' " + "WHERE status = 'no_endpoint'" + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_index("ix_sent_webmentions_source_uuid") + batch.drop_constraint( + "uq_sent_webmention_source_uuid_target_digest", type_="unique" + ) + batch.alter_column( + "source_uuid", + new_column_name="source_id", + existing_type=sa.Uuid(), + existing_nullable=False, + ) + batch.alter_column( + "linked", + new_column_name="active", + existing_type=sa.Boolean(), + existing_nullable=False, + ) + batch.alter_column( + "webmention_endpoint", + new_column_name="endpoint", + existing_type=sa.Text(), + existing_nullable=True, + ) + batch.alter_column( + "endpoint_response_status", + new_column_name="response_status", + existing_type=sa.Integer(), + existing_nullable=True, + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.create_unique_constraint( + "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"] + ) + + op.create_index("ix_sent_webmentions_source_id", "sent_webmentions", ["source_id"]) + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_source_target_digest", type_="unique") + batch.alter_column( + "source_target_digest", + new_column_name="url_digest", + existing_type=sa.String(64), + existing_nullable=False, + ) + batch.alter_column( + "status", + existing_type=_NEW_RECEIVED_STATUS, + type_=_OLD_RECEIVED_STATUS, + existing_nullable=False, + ) + + with op.batch_alter_table("received_webmentions") as batch: + batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"]) + + op.rename_table("source_documents", "sources") diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 41c5376..8518b15 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,7 +11,8 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.models import received_url_digest, sent_target_digest +from webmentions_ssg.models import source_target_digest as received_url_digest +from webmentions_ssg.models import target_digest as sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 33f27d1..2b375dd 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -32,21 +32,21 @@ from werkzeug.security import check_password_hash, generate_password_hash from .extensions import Base -class ReceivedStatus(StrEnum): +class ReceivedWebmentionStatus(StrEnum): """Persisted lifecycle state of an incoming Webmention.""" RECEIVED = "received" VERIFYING = "verifying" VERIFIED = "verified" FAILED = "failed" - DELETED = "deleted" + SOURCE_REMOVED = "source_removed" class SentWebmentionStatus(StrEnum): """Represent the outcome of a Webmention send attempt.""" SENT = "sent" - UNSUPPORTED = "unsupported" + NO_ENDPOINT = "no_endpoint" FAILED = "failed" @@ -110,25 +110,25 @@ class ReceivedWebmention(Base): source: Mapped[str] = mapped_column(Text(), nullable=False) target: Mapped[str] = mapped_column(Text(), nullable=False) - url_digest: Mapped[str] = mapped_column(String(64), nullable=False) + source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) @validates("source", "target") - def _update_url_digest(self, key: str, value: str) -> str: + def _refresh_url_digest(self, key: str, value: str) -> str: source = value if key == "source" else self.__dict__.get("source") target = value if key == "target" else self.__dict__.get("target") if source is not None and target is not None: - self.url_digest = received_url_digest(source, target) + self.source_target_digest = source_target_digest(source, target) return value - status: Mapped[ReceivedStatus] = mapped_column( + status: Mapped[ReceivedWebmentionStatus] = mapped_column( Enum( - ReceivedStatus, + ReceivedWebmentionStatus, name="received_webmention_status", values_callable=lambda enum_type: [member.value for member in enum_type], native_enum=False, ), nullable=False, - default=ReceivedStatus.RECEIVED, + default=ReceivedWebmentionStatus.RECEIVED, ) # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh # explicit submission resets this counter. @@ -151,7 +151,9 @@ class ReceivedWebmention(Base): ) __table_args__ = ( - UniqueConstraint("url_digest", name="uq_webmention_url_digest"), + UniqueConstraint( + "source_target_digest", name="uq_webmention_source_target_digest" + ), Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) @@ -160,35 +162,35 @@ class ReceivedWebmention(Base): return f" {self.target!r}>" @staticmethod - def verification_reset_values( - *, at: datetime, recoveries: int = 0 + def reopen_verification_values( + *, at: datetime, recovery_attempts: int = 0 ) -> dict[str, object]: """Return the columns to reset when verification is explicitly reopened. :param at: Timestamp to record for the newly opened verification. - :param recoveries: Number of stale-lease recovery attempts to retain. + :param recovery_attempts: Number of stale-lease recovery attempts to retain. :returns: Column values for resetting the verification state. """ return { - "status": ReceivedStatus.RECEIVED, + "status": ReceivedWebmentionStatus.RECEIVED, "failure_reason": None, - "recovery_attempts": recoveries, + "recovery_attempts": recovery_attempts, "updated_at": at, } - def reset_for_verification(self) -> None: + def reopen_verification(self) -> None: """Reopen an existing row for manual verification. Clear the previous outcome and restore the received state with a fresh timestamp. """ - for key, value in self.verification_reset_values(at=datetime.now(UTC)).items(): + for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): setattr(self, key, value) @property def verified(self) -> bool: """Return whether the Webmention has been successfully verified.""" - return self.status == ReceivedStatus.VERIFIED + return self.status == ReceivedWebmentionStatus.VERIFIED @property def created_at(self) -> datetime: @@ -196,10 +198,10 @@ class ReceivedWebmention(Base): return uuid7_to_datetime(self.uuid) -class Source(Base): +class SourceDocument(Base): """Represent a generated source document containing outgoing Webmentions.""" - __tablename__ = "sources" + __tablename__ = "source_documents" uuid: Mapped[UUID] = mapped_column( Uuid(as_uuid=True), primary_key=True, default=uuid7 @@ -227,7 +229,7 @@ class Source(Base): def __repr__(self) -> str: """Represent the source by its public URL.""" - return f"" + return f"" @property def created_at(self) -> datetime: @@ -244,9 +246,9 @@ class SentWebmention(Base): Uuid(as_uuid=True), primary_key=True, default=uuid7 ) - source_id: Mapped[UUID] = mapped_column( + source_uuid: Mapped[UUID] = mapped_column( Uuid(as_uuid=True), - ForeignKey("sources.uuid", ondelete="CASCADE"), + ForeignKey("source_documents.uuid", ondelete="CASCADE"), nullable=False, index=True, ) @@ -256,10 +258,10 @@ class SentWebmention(Base): @validates("target") def _update_target_digest(self, key: str, value: str) -> str: - self.target_digest = sent_target_digest(value) + self.target_digest = target_digest(value) return value - active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) processed_revision: Mapped[int | None] = mapped_column( @@ -284,8 +286,10 @@ class SentWebmention(Base): Text(), nullable=True, default=None ) - endpoint: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) - response_status: Mapped[int | None] = mapped_column( + webmention_endpoint: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + endpoint_response_status: Mapped[int | None] = mapped_column( Integer(), nullable=True, default=None ) status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) @@ -303,11 +307,13 @@ class SentWebmention(Base): DateTime(timezone=True), nullable=True, default=None ) - source: Mapped[Source] = relationship(back_populates="sent_webmentions") + source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") __table_args__ = ( UniqueConstraint( - "source_id", "target_digest", name="uq_sent_webmention_source_target_digest" + "source_uuid", + "target_digest", + name="uq_sent_webmention_source_uuid_target_digest", ), ) @@ -315,41 +321,41 @@ class SentWebmention(Base): """Represent the sent Webmention by its target URL.""" return f"" - def reopen(self, revision: int, active: bool) -> None: + def reset_for_revision(self, revision: int, linked: bool) -> None: """Reopen delivery for a revision while preserving send history. :param revision: Source revision to make the desired delivery revision. - :param active: Whether the target still appears in the current source. + :param linked: Whether the target still appears in the current source. """ self.desired_revision = revision - self.active = active + self.linked = linked self.attempt_count = 0 self.next_retry_at = None - def resend(self) -> None: + def clear_result_for_resend(self) -> None: """Retry the current revision, clearing its previous delivery result. Reset attempt and response state without changing the desired source revision. """ - self.reopen(self.desired_revision, self.active) + self.reset_for_revision(self.desired_revision, self.linked) self.processed_revision = None self.status = None self.failure_reason = None - self.endpoint = None - self.response_status = None + self.webmention_endpoint = None + self.endpoint_response_status = None self.status_url = None @hybrid_property - def pending(self) -> bool: + def awaiting_processing(self) -> bool: """Return whether the Webmention has an unprocessed source revision.""" return ( self.processed_revision is None or self.processed_revision < self.desired_revision ) - @pending.inplace.expression + @awaiting_processing.inplace.expression @classmethod - def _pending_expression(cls) -> sa.ColumnElement[bool]: + def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: """Express pending-revision status in database queries. :returns: SQLAlchemy expression selecting unprocessed source revisions. @@ -374,7 +380,7 @@ def uuid7_to_datetime(identifier: UUID) -> datetime: return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) -def received_url_digest(source: str, target: str) -> str: +def source_target_digest(source: str, target: str) -> str: """Fingerprint an unambiguous, ordered source/target pair. :param source: Webmention source URL. @@ -387,7 +393,7 @@ def received_url_digest(source: str, target: str) -> str: return sha256(encoded).hexdigest() -def sent_target_digest(target: str) -> str: +def target_digest(target: str) -> str: """Fingerprint an outgoing target (unique within each source ID). :param target: Outgoing Webmention target URL. diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py index c744b49..37ab4d2 100644 --- a/webmentions_ssg/tasks/__init__.py +++ b/webmentions_ssg/tasks/__init__.py @@ -4,6 +4,6 @@ """Expose the background task queue integration.""" -from .extension import Huey +from .extension import FlaskHuey -__all__ = ["Huey"] +__all__ = ["FlaskHuey"] diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index c24a883..7991b6a 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -8,11 +8,11 @@ from huey import Huey, crontab from .. import create_app from ..extensions import huey as queue -from . import notifications, receiver, retention, scanner +from . import notifications, retention, scanner, verification -def create_consumer() -> Huey: - """Create and configure the Huey consumer. +def create_huey() -> Huey: + """Create and configure the Huey instance for the task worker. The application is initialized and the periodic scanner, notification, verification recovery and retention tasks are registered. @@ -23,8 +23,8 @@ def create_consumer() -> Huey: for config_key, task in ( ("SOURCE_SCAN_SCHEDULE", scanner.scan_sources), - ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications), - ("DIGEST_SCHEDULE", notifications.send_webmention_digest), + ("VERIFICATION_RECOVERY_SCHEDULE", verification.recover_stale_verifications), + ("DIGEST_SCHEDULE", notifications.email_verified_digest), ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), ): schedule = crontab(*app.config[config_key].split(), strict=True) @@ -33,4 +33,4 @@ def create_consumer() -> Huey: return queue.huey -huey = create_consumer() +huey = create_huey() diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/discovery.py deleted file mode 100644 index c683cb0..0000000 --- a/webmentions_ssg/tasks/discovery.py +++ /dev/null @@ -1,217 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Discover a target Webmention endpoint and submit a POST.""" - -import re -from http import HTTPStatus -from urllib.parse import urljoin - -import httpx -from bs4 import BeautifulSoup -from flask import current_app - -from ..net import ( - ResponseTooLargeError, - is_http_url, - is_retryable_status, - media_type_of, - read_limited_body, -) - -LINK_HEADER_SPLIT = re.compile(r",\s*(?=<)") -DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} - - -class DeliveryError(Exception): - """A Webmention cannot be sent.""" - - -class RetryableDeliveryError(DeliveryError): - """A Webmention send failed for a potentially temporary reason.""" - - -class PermanentDeliveryError(DeliveryError): - """A Webmention send failed permanently for this source revision.""" - - -def resolve_endpoint_url(response: httpx.Response, href: str) -> str: - """Resolve and validate a discovered Webmention endpoint. - - :param response: Response from which the endpoint was discovered. - :param href: Endpoint reference to resolve. - :returns: Absolute Webmention endpoint URL. - :raises PermanentDeliveryError: If the resolved endpoint is not an HTTP or HTTPS URL. - """ - try: - endpoint = urljoin(str(response.url), href.strip()) - except ValueError as exc: - raise PermanentDeliveryError(f"Invalid Webmention endpoint: {href!r}") from exc - - if not is_http_url(endpoint): - raise PermanentDeliveryError(f"Invalid Webmention endpoint: {endpoint!r}") - - return endpoint - - -def parse_link_header_value(value: str) -> tuple[str, set[str]] | None: - """Parse a Link header value into its target and relations. - - :param value: Link header value to parse. - :returns: Link target and relation names, or ``None`` if the value is invalid. - """ - value = value.strip() - - if not value.startswith("<"): - return None - - href, separator, parameters = value[1:].partition(">") - - if not separator: - return None - - relations: set[str] = set() - - for parameter in parameters.split(";"): - name, separator, value = parameter.partition("=") - - if separator and name.strip().lower() == "rel": - relations.update( - relation.lower() for relation in value.strip().strip("\"'").split() - ) - - return href, relations - - -def find_endpoint_in_headers(response: httpx.Response) -> str | None: - """Find a Webmention endpoint in the response Link headers. - - :param response: HTTP response whose headers should be inspected. - :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. - Invalid candidates are ignored. - """ - for header in response.headers.get_list("Link"): - for value in LINK_HEADER_SPLIT.split(header): - if (link := parse_link_header_value(value)) is None: - continue - - href, relations = link - - if "webmention" in relations: - try: - return resolve_endpoint_url(response, href) - except PermanentDeliveryError: - continue - - return None - - -def find_endpoint_in_html(response: httpx.Response, body: bytes) -> str | None: - """Find a Webmention endpoint in an HTML document. - - ``link`` and ``a`` elements are inspected in document order. - - :param response: HTTP response from which the document was retrieved. - :param body: HTML document body. - :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. - Invalid candidates are ignored. - """ - document = BeautifulSoup(body, "html5lib") - - for element in document.select( - 'link[rel~="webmention" i][href], a[rel~="webmention" i][href]' - ): - try: - return resolve_endpoint_url(response, str(element["href"])) - except PermanentDeliveryError: - continue - - return None - - -def read_target_body(response: httpx.Response) -> bytes: - """Return a bounded target body, using the shared HTTP size guard. - - :param response: HTTP response to read with the configured target-size limit. - :returns: Response body, provided it is within the target-size limit. - :raises PermanentDeliveryError: If the target document is too large. - """ - try: - return read_limited_body( - response, current_app.config["MAX_DISCOVERY_TARGET_BYTES"], label="Target" - ) - except ResponseTooLargeError as exc: - raise PermanentDeliveryError(str(exc)) from exc - - -def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: - """Discover the Webmention endpoint advertised by a target URL. - - Endpoint discovery first checks the response to a HEAD request and then falls - back to a GET request, inspecting both HTTP Link headers and supported HTML - documents. - - :param client: HTTP client to use for discovery requests. - :param target: Target URL whose Webmention endpoint should be discovered. - :returns: Discovered Webmention endpoint, or ``None`` if none is advertised. - :raises RetryableDeliveryError: If the target returns a temporary HTTP failure. - :raises PermanentDeliveryError: If GET returns a permanent HTTP failure. - """ - try: - head_response = client.head(target, headers=DISCOVERY_HEADERS) - except httpx.RequestError: - # HEAD is only an optimization. Servers and intermediaries may reject - # or drop it even when fetching the document with GET would succeed. - pass - else: - if endpoint := find_endpoint_in_headers(head_response): - return endpoint - - with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: - if endpoint := find_endpoint_in_headers(response): - return endpoint - - if is_retryable_status(response.status_code): - raise RetryableDeliveryError(f"Target returned HTTP {response.status_code}") - - if not response.is_success: - raise PermanentDeliveryError(f"Target returned HTTP {response.status_code}") - - media_type = media_type_of(response.headers) - - if media_type not in {"text/html", "application/xhtml+xml"}: - return None - - return find_endpoint_in_html(response, read_target_body(response)) - - -def post_webmention( - client: httpx.Client, *, endpoint: str, source: str, target: str -) -> tuple[int, str | None]: - """Send a Webmention to a discovered endpoint. - - :param client: HTTP client to use for the request. - :param endpoint: Webmention endpoint URL. - :param source: Source URL of the Webmention. - :param target: Target URL of the Webmention. - :returns: HTTP response status code and optional status URL. - """ - with client.stream( - "POST", endpoint, data={"source": source, "target": target} - ) as response: - status_url = None - - if response.status_code == HTTPStatus.CREATED and ( - location := response.headers.get("Location") - ): - try: - candidate = urljoin(str(response.url), location) - except ValueError: - # A malformed Location must not fail an otherwise successful send. - pass - else: - if is_http_url(candidate): - status_url = candidate - - return (response.status_code, status_url) diff --git a/webmentions_ssg/tasks/endpoint_discovery.py b/webmentions_ssg/tasks/endpoint_discovery.py new file mode 100644 index 0000000..c683cb0 --- /dev/null +++ b/webmentions_ssg/tasks/endpoint_discovery.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Discover a target Webmention endpoint and submit a POST.""" + +import re +from http import HTTPStatus +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup +from flask import current_app + +from ..net import ( + ResponseTooLargeError, + is_http_url, + is_retryable_status, + media_type_of, + read_limited_body, +) + +LINK_HEADER_SPLIT = re.compile(r",\s*(?=<)") +DISCOVERY_HEADERS = {"Accept": "text/html, application/xhtml+xml;q=0.9"} + + +class DeliveryError(Exception): + """A Webmention cannot be sent.""" + + +class RetryableDeliveryError(DeliveryError): + """A Webmention send failed for a potentially temporary reason.""" + + +class PermanentDeliveryError(DeliveryError): + """A Webmention send failed permanently for this source revision.""" + + +def resolve_endpoint_url(response: httpx.Response, href: str) -> str: + """Resolve and validate a discovered Webmention endpoint. + + :param response: Response from which the endpoint was discovered. + :param href: Endpoint reference to resolve. + :returns: Absolute Webmention endpoint URL. + :raises PermanentDeliveryError: If the resolved endpoint is not an HTTP or HTTPS URL. + """ + try: + endpoint = urljoin(str(response.url), href.strip()) + except ValueError as exc: + raise PermanentDeliveryError(f"Invalid Webmention endpoint: {href!r}") from exc + + if not is_http_url(endpoint): + raise PermanentDeliveryError(f"Invalid Webmention endpoint: {endpoint!r}") + + return endpoint + + +def parse_link_header_value(value: str) -> tuple[str, set[str]] | None: + """Parse a Link header value into its target and relations. + + :param value: Link header value to parse. + :returns: Link target and relation names, or ``None`` if the value is invalid. + """ + value = value.strip() + + if not value.startswith("<"): + return None + + href, separator, parameters = value[1:].partition(">") + + if not separator: + return None + + relations: set[str] = set() + + for parameter in parameters.split(";"): + name, separator, value = parameter.partition("=") + + if separator and name.strip().lower() == "rel": + relations.update( + relation.lower() for relation in value.strip().strip("\"'").split() + ) + + return href, relations + + +def find_endpoint_in_headers(response: httpx.Response) -> str | None: + """Find a Webmention endpoint in the response Link headers. + + :param response: HTTP response whose headers should be inspected. + :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. + Invalid candidates are ignored. + """ + for header in response.headers.get_list("Link"): + for value in LINK_HEADER_SPLIT.split(header): + if (link := parse_link_header_value(value)) is None: + continue + + href, relations = link + + if "webmention" in relations: + try: + return resolve_endpoint_url(response, href) + except PermanentDeliveryError: + continue + + return None + + +def find_endpoint_in_html(response: httpx.Response, body: bytes) -> str | None: + """Find a Webmention endpoint in an HTML document. + + ``link`` and ``a`` elements are inspected in document order. + + :param response: HTTP response from which the document was retrieved. + :param body: HTML document body. + :returns: First valid advertised Webmention endpoint, or ``None`` if none is found. + Invalid candidates are ignored. + """ + document = BeautifulSoup(body, "html5lib") + + for element in document.select( + 'link[rel~="webmention" i][href], a[rel~="webmention" i][href]' + ): + try: + return resolve_endpoint_url(response, str(element["href"])) + except PermanentDeliveryError: + continue + + return None + + +def read_target_body(response: httpx.Response) -> bytes: + """Return a bounded target body, using the shared HTTP size guard. + + :param response: HTTP response to read with the configured target-size limit. + :returns: Response body, provided it is within the target-size limit. + :raises PermanentDeliveryError: If the target document is too large. + """ + try: + return read_limited_body( + response, current_app.config["MAX_DISCOVERY_TARGET_BYTES"], label="Target" + ) + except ResponseTooLargeError as exc: + raise PermanentDeliveryError(str(exc)) from exc + + +def discover_webmention_endpoint(client: httpx.Client, target: str) -> str | None: + """Discover the Webmention endpoint advertised by a target URL. + + Endpoint discovery first checks the response to a HEAD request and then falls + back to a GET request, inspecting both HTTP Link headers and supported HTML + documents. + + :param client: HTTP client to use for discovery requests. + :param target: Target URL whose Webmention endpoint should be discovered. + :returns: Discovered Webmention endpoint, or ``None`` if none is advertised. + :raises RetryableDeliveryError: If the target returns a temporary HTTP failure. + :raises PermanentDeliveryError: If GET returns a permanent HTTP failure. + """ + try: + head_response = client.head(target, headers=DISCOVERY_HEADERS) + except httpx.RequestError: + # HEAD is only an optimization. Servers and intermediaries may reject + # or drop it even when fetching the document with GET would succeed. + pass + else: + if endpoint := find_endpoint_in_headers(head_response): + return endpoint + + with client.stream("GET", target, headers=DISCOVERY_HEADERS) as response: + if endpoint := find_endpoint_in_headers(response): + return endpoint + + if is_retryable_status(response.status_code): + raise RetryableDeliveryError(f"Target returned HTTP {response.status_code}") + + if not response.is_success: + raise PermanentDeliveryError(f"Target returned HTTP {response.status_code}") + + media_type = media_type_of(response.headers) + + if media_type not in {"text/html", "application/xhtml+xml"}: + return None + + return find_endpoint_in_html(response, read_target_body(response)) + + +def post_webmention( + client: httpx.Client, *, endpoint: str, source: str, target: str +) -> tuple[int, str | None]: + """Send a Webmention to a discovered endpoint. + + :param client: HTTP client to use for the request. + :param endpoint: Webmention endpoint URL. + :param source: Source URL of the Webmention. + :param target: Target URL of the Webmention. + :returns: HTTP response status code and optional status URL. + """ + with client.stream( + "POST", endpoint, data={"source": source, "target": target} + ) as response: + status_url = None + + if response.status_code == HTTPStatus.CREATED and ( + location := response.headers.get("Location") + ): + try: + candidate = urljoin(str(response.url), location) + except ValueError: + # A malformed Location must not fail an otherwise successful send. + pass + else: + if is_http_url(candidate): + status_url = candidate + + return (response.status_code, status_url) diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index faed364..3f172f4 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -4,8 +4,6 @@ """Integrate Huey tasks with Flask application contexts.""" -from __future__ import annotations - from collections.abc import Callable from functools import update_wrapper, wraps from typing import Any @@ -100,7 +98,7 @@ class _DeferredTask: def __init__( self, - extension: Huey, + extension: FlaskHuey, func: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any], @@ -152,7 +150,7 @@ class _DeferredTask: return getattr(self._bound_task(), name) -class Huey: +class FlaskHuey: """Provide Flask integration for a Huey instance. The extension initializes a Huey backend from the Flask configuration and diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index eb540de..c503e1f 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,10 +11,10 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus -def send_webmention_digest() -> None: +def email_verified_digest() -> None: """Send one digest of verified, previously unreported Webmentions. Acquire the digest lock before notifying recipients about previously unreported @@ -38,7 +38,7 @@ def _send_digest_locked() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == ReceivedStatus.VERIFIED, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/receiver.py deleted file mode 100644 index 313a8e4..0000000 --- a/webmentions_ssg/tasks/receiver.py +++ /dev/null @@ -1,358 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Dennis Fink -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Fetch and verify incoming Webmentions with transactional state changes.""" - -import uuid -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from http import HTTPStatus - -import httpx -import sqlalchemy as sa -from flask import current_app - -from ..db_helpers import rows_affected -from ..extensions import db, huey -from ..models import ReceivedStatus, ReceivedWebmention -from ..net import ( - AddressResolutionError, - ResponseTooLargeError, - UnsafeAddressError, - build_public_http_client, - is_retryable_status, - media_type_of, - read_limited_body, -) -from ..version import APP_NAME, VERSION -from .target_matching import html_mentions_target, text_mentions_target - - -class VerificationError(Exception): - """The source permanently failed ReceivedWebmention verification.""" - - -class SourceGoneError(VerificationError): - """The source explicitly reports that it has been removed.""" - - -class RetryableFetchError(Exception): - """Fetching the source may succeed when retried later.""" - - -RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" - - -def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: - """Fetch a webmention source document. - - The request follows redirects while enforcing the configured timeout, - redirect limit, and maximum response size. - - :param source_url: URL of the source document. - :returns: HTTP response and response body. - :raises SourceGoneError: If the source returns HTTP 410. - :raises RetryableFetchError: If the source returns a temporary HTTP error. - :raises VerificationError: If the source returns another unsuccessful HTTP status or - exceeds the configured maximum response size. - """ - try: - with ( - build_public_http_client( - f"{APP_NAME}/{VERSION} ReceivedWebmention", - accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", - ) as client, - client.stream("GET", source_url) as response, - ): - if response.status_code == HTTPStatus.OK: - pass - elif response.status_code == HTTPStatus.GONE: - raise SourceGoneError("Source returned HTTP 410") - elif is_retryable_status(response.status_code): - raise RetryableFetchError( - f"Source returned HTTP {response.status_code}" - ) - else: - raise VerificationError(f"Source returned HTTP {response.status_code}") - - try: - return response, read_limited_body( - response, - current_app.config["MAX_RECEIVED_SOURCE_BYTES"], - label="Source", - ) - except ResponseTooLargeError as exc: - raise VerificationError(str(exc)) from exc - - except UnsafeAddressError as exc: - raise VerificationError("Source resolves to a non-public address") from exc - except AddressResolutionError as exc: - raise RetryableFetchError("Source hostname could not be resolved") from exc - - -def source_mentions_target(source_url: str, target_url: str) -> bool: - """Fetch a source and check whether it mentions the target URL. - - HTML and XHTML sources are inspected for supported URL attributes, while - plain-text sources are searched for the exact target IRI. - - :param source_url: URL of the source document. - :param target_url: Exact target URL to look for. - :returns: Whether the source mentions the target URL. - :raises VerificationError: If the source has an unsupported or missing content type. - :raises SourceGoneError: If the source explicitly reports that it is gone. - :raises RetryableFetchError: If fetching the source fails temporarily. - """ - response, body = fetch_source_document(source_url) - - media_type = media_type_of(response.headers) - - match media_type: - case "text/html" | "application/xhtml+xml": - return html_mentions_target(body, str(response.url), target_url) - case "text/plain": - try: - decoded_body = body.decode( - response.encoding or "utf-8", errors="replace" - ) - except LookupError: - decoded_body = body.decode("utf-8", errors="replace") - return text_mentions_target(decoded_body, target_url) - case _: - raise VerificationError( - f"Unsupported source content type: {media_type or 'missing'}" - ) - - -@dataclass(frozen=True) -class VerificationClaim: - """Source and lease version for one atomically claimed verification.""" - - webmention_uuid: uuid.UUID - source: str - target: str - lease_started_at: datetime - - -def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: - """Claim an eligible row, returning its source, target, and lease version. - - :param webmention_uuid: Identifier of the received Webmention to claim. - :returns: Verification claim, or ``None`` if the row cannot be claimed. - """ - webmention = db.session.get(ReceivedWebmention, webmention_uuid) - - if webmention is None: - current_app.logger.warning( - "Cannot verify unknown ReceivedWebmention %s", webmention_uuid - ) - return None - - if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: - return None - - if ( - webmention.status == ReceivedStatus.FAILED - and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON - ): - return None - - claim = VerificationClaim( - webmention_uuid=webmention_uuid, - source=webmention.source, - target=webmention.target, - lease_started_at=datetime.now(UTC), - ) - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == webmention_uuid, - ReceivedWebmention.status == webmention.status, - ReceivedWebmention.updated_at == webmention.updated_at, - ) - .values( - status=ReceivedStatus.VERIFYING, - failure_reason=None, - updated_at=claim.lease_started_at, - ) - ) - - if rows_affected(changed) != 1: - db.session.rollback() - return None - - db.session.commit() - return claim - - -def determine_verification_outcome( - claim: VerificationClaim, -) -> tuple[ReceivedStatus, str | None]: - """Map verification results and permanent errors to a final status/reason. - - :param claim: Claim containing the source and target URLs to verify. - :returns: Final verification status and optional reason. - :raises RetryableFetchError: If the source fetch fails transiently and should be - retried. - :raises httpx.RequestError: If a transient HTTP request error should be retried. - """ - try: - mentions_target = source_mentions_target(claim.source, claim.target) - except SourceGoneError as exc: - return ReceivedStatus.DELETED, str(exc) - except VerificationError as exc: - return ReceivedStatus.FAILED, str(exc) - except RetryableFetchError, httpx.RequestError: - raise # These errors must be persisted and then retried by Huey. - except Exception as exc: - # InvalidURL is not a RequestError; unexpected source errors are terminal. - current_app.logger.exception( - "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid - ) - return ReceivedStatus.FAILED, str(exc) or type(exc).__name__ - - if mentions_target: - return ReceivedStatus.VERIFIED, None - return ReceivedStatus.DELETED, "Source does not mention target" - - -def release_claim( - claim: VerificationClaim, status: ReceivedStatus, reason: str | None -) -> bool: - """Persist an outcome only if this worker still owns the verification lease. - - :param claim: Verification lease acquired by this worker. - :param status: Final status to persist for the claimed Webmention. - :param reason: Optional explanation for a failed or deleted Webmention. - :returns: ``True`` if the worker still owned the lease and persisted the outcome. - """ - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == claim.webmention_uuid, - ReceivedWebmention.status == ReceivedStatus.VERIFYING, - ReceivedWebmention.updated_at == claim.lease_started_at, - ) - .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) - ) - db.session.commit() - return rows_affected(changed) == 1 - - -@huey.task(retries=2, retry_delay=50) -def verify_webmention(webmention_uuid: uuid.UUID) -> None: - """Claim and verify a received Webmention, preserving its lease on release. - - Temporary fetch failures are recorded and re-raised for Huey's retries. - - Retry transient fetch errors; persist only results belonging to the currently held - lease. - - :param webmention_uuid: Identifier of the received webmention to verify. - :raises RetryableFetchError: If fetching the source fails temporarily. - :raises httpx.RequestError: If an HTTP request fails while fetching. - """ - if (claim := claim_for_verification(webmention_uuid)) is None: - return - - try: - status, reason = determine_verification_outcome(claim) - except (RetryableFetchError, httpx.RequestError) as exc: - release_claim( - claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched" - ) - raise - - release_claim(claim, status, reason) - - -def recover_stale_verifications() -> int: - """Requeue verification attempts that were abandoned or lost from the queue. - - Recover both stale ``verifying`` rows (workers that died) and stale - ``received`` rows (tasks that were never enqueued or were lost). Conditional - updates prevent overlapping reapers from enqueueing the same row. Each - reclaimed row is committed *before* enqueueing, so the worker can claim it. - A crash between commit and enqueue is recovered on a later sweep. - Persist a recovery count so repeatedly lost or crashing tasks eventually - terminate rather than being requeued forever. - - Eligible expired verification leases are returned to the queue, subject to recovery - limits. - - :returns: Number of webmentions requeued. - """ - now = datetime.now(UTC) - cutoff = now - timedelta( - seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"] - ) - - stale = db.session.execute( - sa.select( - ReceivedWebmention.uuid, - ReceivedWebmention.status, - ReceivedWebmention.updated_at, - ReceivedWebmention.recovery_attempts, - ).where( - ReceivedWebmention.status.in_( - (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) - ), - ReceivedWebmention.updated_at <= cutoff, - ) - ).all() - - recovered = 0 - exhausted = 0 - max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] - - for identifier, status, updated_at, recovery_attempts in stale: - at_limit = recovery_attempts >= max_recoveries - changed = db.session.execute( - sa.update(ReceivedWebmention) - .where( - ReceivedWebmention.uuid == identifier, - ReceivedWebmention.status == status, - ReceivedWebmention.updated_at == updated_at, - ReceivedWebmention.recovery_attempts == recovery_attempts, - ) - .values( - **( - { - "status": ReceivedStatus.FAILED, - "failure_reason": RECOVERY_EXHAUSTED_REASON, - "updated_at": now, - "recovery_attempts": recovery_attempts, - } - if at_limit - else ReceivedWebmention.verification_reset_values( - at=now, recoveries=recovery_attempts + 1 - ) - ) - ) - ) - - if rows_affected(changed) != 1: - db.session.rollback() - continue - - db.session.commit() - - if at_limit: - exhausted += 1 - continue - verify_webmention(identifier) - recovered += 1 - - if exhausted: - current_app.logger.warning( - "Failed %d stale webmentions after maximum verification recoveries", - exhausted, - ) - - if recovered: - current_app.logger.warning( - "Requeued %d stale received webmention verifications", recovered - ) - - return recovered diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 2c35cc7..b882493 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: @@ -33,7 +33,10 @@ def purge_old_received_webmentions() -> int: result = db.session.execute( sa.delete(ReceivedWebmention).where( ReceivedWebmention.status.in_( - (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ( + ReceivedWebmentionStatus.FAILED, + ReceivedWebmentionStatus.SOURCE_REMOVED, + ) ), ReceivedWebmention.updated_at < cutoff, ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 034b2cf..fb0c770 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,13 +14,13 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, Source +from ..models import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file -def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: +def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. A sent Webmention is built for each discovered target at the initial source @@ -30,7 +30,7 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: :param scan_time: Time at which the source was discovered. :returns: Newly built source model. """ - source = Source( + source = SourceDocument( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, @@ -42,15 +42,15 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: for target in scanned.targets: source.sent_webmentions.append( - SentWebmention(target=target, active=True, desired_revision=1) + SentWebmention(target=target, linked=True, desired_revision=1) ) return source def apply_scan_to_source( - source: Source, scanned: ScannedSource, scan_time: datetime -) -> Source: + source: SourceDocument, scanned: ScannedSource, scan_time: datetime +) -> SourceDocument: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent @@ -82,16 +82,18 @@ def apply_scan_to_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.reopen(source.revision, webmention.target in scanned.targets) + webmention.reset_for_revision( + source.revision, webmention.target in scanned.targets + ) - if not webmention.active and webmention.sent_revision is None: + if not webmention.linked and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( - target=target, active=True, desired_revision=source.revision + target=target, linked=True, desired_revision=source.revision ) ) @@ -127,7 +129,7 @@ class TreeScanResult: def reconcile_source_tree( root: Path, base_url: str | None, - sources_by_path: dict[str, Source], + sources_by_path: dict[str, SourceDocument], paths_by_url: dict[str, str], scan_time: datetime, ) -> TreeScanResult: @@ -197,14 +199,16 @@ def reconcile_source_tree( def mark_missing_sources_deleted( - sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime + sources_by_path: dict[str, SourceDocument], + seen_paths: frozenset[str], + scan_time: datetime, ) -> int: - """Mark absent, previously active sources deleted and return their count. + """Mark absent, previously present sources deleted and return their count. :param sources_by_path: Known source records indexed by relative path. :param seen_paths: Paths found during this scan, including failed parses. :param scan_time: Timestamp to record for sources newly marked missing. - :returns: Number of previously active sources marked as deleted. + :returns: Number of previously linked sources marked as deleted. """ marked = 0 for path, source in sources_by_path.items(): @@ -217,7 +221,7 @@ def mark_missing_sources_deleted( marked += 1 for webmention in source.sent_webmentions: - webmention.reopen(source.revision, False) + webmention.reset_for_revision(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -235,7 +239,7 @@ def enqueue_due_sends(scan_time: datetime) -> int: for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, @@ -273,7 +277,9 @@ def _scan_and_enqueue() -> None: sources_by_path = { source.path: source for source in db.session.scalars( - sa.select(Source).options(selectinload(Source.sent_webmentions)) + sa.select(SourceDocument).options( + selectinload(SourceDocument.sent_webmentions) + ) ) } diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index c33cf06..3f64bed 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -24,7 +24,7 @@ from ..net import ( is_retryable_status, ) from ..version import APP_NAME, VERSION -from .discovery import ( +from .endpoint_discovery import ( PermanentDeliveryError, RetryableDeliveryError, discover_webmention_endpoint, @@ -44,7 +44,7 @@ def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """ db.session.refresh(webmention) - return webmention.desired_revision == revision and webmention.pending + return webmention.desired_revision == revision and webmention.awaiting_processing def backoff_delay(attempt: int) -> timedelta: @@ -99,8 +99,8 @@ class ClaimedSendAttempt: class DeliveryResult: """HTTP outcome, with whether the failure is eligible for another attempt. - ``terminal`` describes the failure type. The retry budget can also make a - temporary failure terminal when its last allowed attempt has been used. + ``retryable`` describes the failure type; it does not account for the + remaining retry budget, which may already have been exhausted. """ status: SentWebmentionStatus @@ -108,8 +108,7 @@ class DeliveryResult: endpoint: str | None = None response_status: int | None = None status_url: str | None = None - terminal: bool = True - temporary_failure: bool = False + retryable: bool = False def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: @@ -119,7 +118,7 @@ def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: :param now: Current timestamp against which the retry deadline is compared. :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed. """ - if not webmention.pending: + if not webmention.awaiting_processing: return False deadline = webmention.next_retry_at @@ -154,7 +153,7 @@ def reserve_send_attempt( SentWebmention.uuid == webmention.uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count >= max_attempts, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -188,7 +187,7 @@ def reserve_send_attempt( SentWebmention.uuid == attempt.webmention_uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count == attempt.attempt_number - 1, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -198,8 +197,8 @@ def reserve_send_attempt( attempt_count=attempt.attempt_number, last_attempted_at=now, next_retry_at=now + backoff_delay(attempt.attempt_number), - endpoint=None, - response_status=None, + webmention_endpoint=None, + endpoint_response_status=None, status_url=None, ) .execution_options(synchronize_session=False) @@ -236,7 +235,7 @@ def attempt_delivery( if endpoint is None: return DeliveryResult( - status=SentWebmentionStatus.UNSUPPORTED, + status=SentWebmentionStatus.NO_ENDPOINT, reason="No Webmention endpoint discovered", ) @@ -275,11 +274,10 @@ def attempt_delivery( reason=str(exc) or "Webmention request failed", endpoint=endpoint, response_status=response_status, - terminal=False, - temporary_failure=True, + retryable=True, ) except Exception as exc: - # Unexpected errors should be terminal and keep their traceback. + # Unexpected errors should not be retried; keep their traceback. current_app.logger.exception( "Unexpected error sending SentWebmention %s", webmention_uuid ) @@ -312,8 +310,8 @@ def record_delivery_outcome( webmention.status = result.status webmention.failure_reason = result.reason - webmention.endpoint = result.endpoint - webmention.response_status = result.response_status + webmention.webmention_endpoint = result.endpoint + webmention.endpoint_response_status = result.response_status if terminal: webmention.processed_revision = attempt.revision @@ -355,19 +353,19 @@ def _process_send_locked(webmention_uuid: uuid.UUID) -> bool: result = attempt_delivery( attempt.source, attempt.target, webmention_uuid=webmention_uuid ) - terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts + terminal = not result.retryable or attempt.attempt_number >= attempt.max_attempts if not record_delivery_outcome(webmention, attempt, result, terminal=terminal): return False - if result.temporary_failure and not terminal: + if result.retryable and not terminal: # Huey does not retry: the scanner requeues after next_retry_at. current_app.logger.info( "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt.attempt_number, ) - elif result.temporary_failure: + elif result.retryable: current_app.logger.warning( "Webmention %s exhausted %d send attempts: %s", webmention_uuid, diff --git a/webmentions_ssg/tasks/verification.py b/webmentions_ssg/tasks/verification.py new file mode 100644 index 0000000..a614815 --- /dev/null +++ b/webmentions_ssg/tasks/verification.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Fetch and verify incoming Webmentions with transactional state changes.""" + +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import httpx +import sqlalchemy as sa +from flask import current_app + +from ..db_helpers import rows_affected +from ..extensions import db, huey +from ..models import ReceivedWebmention, ReceivedWebmentionStatus +from ..net import ( + AddressResolutionError, + ResponseTooLargeError, + UnsafeAddressError, + build_public_http_client, + is_retryable_status, + media_type_of, + read_limited_body, +) +from ..version import APP_NAME, VERSION +from .target_matching import html_mentions_target, text_mentions_target + + +class VerificationError(Exception): + """The source permanently failed ReceivedWebmention verification.""" + + +class SourceGoneError(VerificationError): + """The source explicitly reports that it has been removed.""" + + +class RetryableFetchError(Exception): + """Fetching the source may succeed when retried later.""" + + +RECOVERY_EXHAUSTED_REASON = "Verification recovery limit reached" + + +def fetch_source_document(source_url: str) -> tuple[httpx.Response, bytes]: + """Fetch a webmention source document. + + The request follows redirects while enforcing the configured timeout, + redirect limit, and maximum response size. + + :param source_url: URL of the source document. + :returns: HTTP response and response body. + :raises SourceGoneError: If the source returns HTTP 410. + :raises RetryableFetchError: If the source returns a temporary HTTP error. + :raises VerificationError: If the source returns another unsuccessful HTTP status or + exceeds the configured maximum response size. + """ + try: + with ( + build_public_http_client( + f"{APP_NAME}/{VERSION} ReceivedWebmention", + accept="text/html, application/xhtml+xml;q=0.9, text/plain;q=0.8", + ) as client, + client.stream("GET", source_url) as response, + ): + if response.status_code == HTTPStatus.OK: + pass + elif response.status_code == HTTPStatus.GONE: + raise SourceGoneError("Source returned HTTP 410") + elif is_retryable_status(response.status_code): + raise RetryableFetchError( + f"Source returned HTTP {response.status_code}" + ) + else: + raise VerificationError(f"Source returned HTTP {response.status_code}") + + try: + return response, read_limited_body( + response, + current_app.config["MAX_RECEIVED_SOURCE_BYTES"], + label="Source", + ) + except ResponseTooLargeError as exc: + raise VerificationError(str(exc)) from exc + + except UnsafeAddressError as exc: + raise VerificationError("Source resolves to a non-public address") from exc + except AddressResolutionError as exc: + raise RetryableFetchError("Source hostname could not be resolved") from exc + + +def source_mentions_target(source_url: str, target_url: str) -> bool: + """Fetch a source and check whether it mentions the target URL. + + HTML and XHTML sources are inspected for supported URL attributes, while + plain-text sources are searched for the exact target IRI. + + :param source_url: URL of the source document. + :param target_url: Exact target URL to look for. + :returns: Whether the source mentions the target URL. + :raises VerificationError: If the source has an unsupported or missing content type. + :raises SourceGoneError: If the source explicitly reports that it is gone. + :raises RetryableFetchError: If fetching the source fails temporarily. + """ + response, body = fetch_source_document(source_url) + + media_type = media_type_of(response.headers) + + match media_type: + case "text/html" | "application/xhtml+xml": + return html_mentions_target(body, str(response.url), target_url) + case "text/plain": + try: + decoded_body = body.decode( + response.encoding or "utf-8", errors="replace" + ) + except LookupError: + decoded_body = body.decode("utf-8", errors="replace") + return text_mentions_target(decoded_body, target_url) + case _: + raise VerificationError( + f"Unsupported source content type: {media_type or 'missing'}" + ) + + +@dataclass(frozen=True) +class VerificationClaim: + """Source and lease version for one atomically claimed verification.""" + + webmention_uuid: uuid.UUID + source: str + target: str + lease_started_at: datetime + + +def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | None: + """Claim an eligible row, returning its source, target, and lease version. + + :param webmention_uuid: Identifier of the received Webmention to claim. + :returns: Verification claim, or ``None`` if the row cannot be claimed. + """ + webmention = db.session.get(ReceivedWebmention, webmention_uuid) + + if webmention is None: + current_app.logger.warning( + "Cannot verify unknown ReceivedWebmention %s", webmention_uuid + ) + return None + + if webmention.status not in { + ReceivedWebmentionStatus.RECEIVED, + ReceivedWebmentionStatus.FAILED, + }: + return None + + if ( + webmention.status == ReceivedWebmentionStatus.FAILED + and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON + ): + return None + + claim = VerificationClaim( + webmention_uuid=webmention_uuid, + source=webmention.source, + target=webmention.target, + lease_started_at=datetime.now(UTC), + ) + changed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == webmention_uuid, + ReceivedWebmention.status == webmention.status, + ReceivedWebmention.updated_at == webmention.updated_at, + ) + .values( + status=ReceivedWebmentionStatus.VERIFYING, + failure_reason=None, + updated_at=claim.lease_started_at, + ) + ) + + if rows_affected(changed) != 1: + db.session.rollback() + return None + + db.session.commit() + return claim + + +def determine_verification_outcome( + claim: VerificationClaim, +) -> tuple[ReceivedWebmentionStatus, str | None]: + """Map verification results and permanent errors to a final status/reason. + + :param claim: Claim containing the source and target URLs to verify. + :returns: Final verification status and optional reason. + :raises RetryableFetchError: If the source fetch fails transiently and should be + retried. + :raises httpx.RequestError: If a transient HTTP request error should be retried. + """ + try: + mentions_target = source_mentions_target(claim.source, claim.target) + except SourceGoneError as exc: + return ReceivedWebmentionStatus.SOURCE_REMOVED, str(exc) + except VerificationError as exc: + return ReceivedWebmentionStatus.FAILED, str(exc) + except RetryableFetchError, httpx.RequestError: + raise # These errors must be persisted and then retried by Huey. + except Exception as exc: + # InvalidURL is not a RequestError; unexpected source errors are terminal. + current_app.logger.exception( + "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid + ) + return ReceivedWebmentionStatus.FAILED, str(exc) or type(exc).__name__ + + if mentions_target: + return ReceivedWebmentionStatus.VERIFIED, None + return ReceivedWebmentionStatus.SOURCE_REMOVED, "Source does not mention target" + + +def store_verification_outcome( + claim: VerificationClaim, status: ReceivedWebmentionStatus, reason: str | None +) -> bool: + """Persist an outcome only if this worker still owns the verification lease. + + :param claim: Verification lease acquired by this worker. + :param status: Final status to persist for the claimed Webmention. + :param reason: Optional explanation for a failed or source-removed Webmention. + :returns: ``True`` if the worker still owned the lease and persisted the outcome. + """ + changed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == claim.webmention_uuid, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFYING, + ReceivedWebmention.updated_at == claim.lease_started_at, + ) + .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) + ) + db.session.commit() + return rows_affected(changed) == 1 + + +@huey.task(retries=2, retry_delay=50) +def verify_webmention(webmention_uuid: uuid.UUID) -> None: + """Claim and verify a received Webmention, preserving its lease on release. + + Temporary fetch failures are recorded and re-raised for Huey's retries. + + Retry transient fetch errors; persist only results belonging to the currently held + lease. + + :param webmention_uuid: Identifier of the received webmention to verify. + :raises RetryableFetchError: If fetching the source fails temporarily. + :raises httpx.RequestError: If an HTTP request fails while fetching. + """ + if (claim := claim_for_verification(webmention_uuid)) is None: + return + + try: + status, reason = determine_verification_outcome(claim) + except (RetryableFetchError, httpx.RequestError) as exc: + store_verification_outcome( + claim, + ReceivedWebmentionStatus.FAILED, + str(exc) or "Source could not be fetched", + ) + raise + + store_verification_outcome(claim, status, reason) + + +def recover_stale_verifications() -> int: + """Requeue verification attempts that were abandoned or lost from the queue. + + Recover both stale ``verifying`` rows (workers that died) and stale + ``received`` rows (tasks that were never enqueued or were lost). Conditional + updates prevent overlapping reapers from enqueueing the same row. Each + reclaimed row is committed *before* enqueueing, so the worker can claim it. + A crash between commit and enqueue is recovered on a later sweep. + Persist a recovery count so repeatedly lost or crashing tasks eventually + terminate rather than being requeued forever. + + Eligible expired verification leases are returned to the queue, subject to recovery + limits. + + :returns: Number of webmentions requeued. + """ + now = datetime.now(UTC) + cutoff = now - timedelta( + seconds=current_app.config["VERIFICATION_STALE_AFTER_SECONDS"] + ) + + stale = db.session.execute( + sa.select( + ReceivedWebmention.uuid, + ReceivedWebmention.status, + ReceivedWebmention.updated_at, + ReceivedWebmention.recovery_attempts, + ).where( + ReceivedWebmention.status.in_( + (ReceivedWebmentionStatus.VERIFYING, ReceivedWebmentionStatus.RECEIVED) + ), + ReceivedWebmention.updated_at <= cutoff, + ) + ).all() + + recovered = 0 + exhausted = 0 + max_recoveries = current_app.config["VERIFICATION_MAX_RECOVERIES"] + + for identifier, status, updated_at, recovery_attempts in stale: + at_limit = recovery_attempts >= max_recoveries + changed = db.session.execute( + sa.update(ReceivedWebmention) + .where( + ReceivedWebmention.uuid == identifier, + ReceivedWebmention.status == status, + ReceivedWebmention.updated_at == updated_at, + ReceivedWebmention.recovery_attempts == recovery_attempts, + ) + .values( + **( + { + "status": ReceivedWebmentionStatus.FAILED, + "failure_reason": RECOVERY_EXHAUSTED_REASON, + "updated_at": now, + "recovery_attempts": recovery_attempts, + } + if at_limit + else ReceivedWebmention.reopen_verification_values( + at=now, recovery_attempts=recovery_attempts + 1 + ) + ) + ) + ) + + if rows_affected(changed) != 1: + db.session.rollback() + continue + + db.session.commit() + + if at_limit: + exhausted += 1 + continue + verify_webmention(identifier) + recovered += 1 + + if exhausted: + current_app.logger.warning( + "Failed %d stale webmentions after maximum verification recoveries", + exhausted, + ) + + if recovered: + current_app.logger.warning( + "Requeued %d stale received webmention verifications", recovered + ) + + return recovered diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index f00a5f9..f94d104 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -27,7 +27,7 @@ SPDX-License-Identifier: BSD-3-Clause "badge_class": "text-bg-danger", "label": "Declined" }, - "deleted": { + "source_removed": { "row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined" diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 31d5c81..2b7a377 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -12,7 +12,7 @@ SPDX-License-Identifier: BSD-3-Clause "badge_class": "text-bg-success", "label": "Sent" }, - "unsupported": { + "no_endpoint": { "row_class": "table-warning", "badge_class": "text-bg-warning", "label": "Unsupported" @@ -114,7 +114,7 @@ SPDX-License-Identifier: BSD-3-Clause {% endif %} - {% if webmention.active %} + {% if webmention.linked %} Active {% else %} Removed @@ -139,16 +139,16 @@ SPDX-License-Identifier: BSD-3-Clause - {% if webmention.endpoint is not none %} + {% if webmention.webmention_endpoint is not none %} {% endif %} - {% if webmention.response_status is not none %} + {% if webmention.endpoint_response_status is not none %}
HTTP: - {{ webmention.response_status }} + {{ webmention.endpoint_response_status }}
{% endif %} {% if webmention.status_url is not none %} diff --git a/webmentions_ssg/templates/status.html b/webmentions_ssg/templates/status.html index 2db2741..a9a39b4 100644 --- a/webmentions_ssg/templates/status.html +++ b/webmentions_ssg/templates/status.html @@ -16,7 +16,7 @@ SPDX-License-Identifier: BSD-3-Clause {% elif webmention.status in ("received", "verifying") %}
{{ render_icon("hourglass-split", size="5rem", title="Verification pending") }}

Webmention verification pending

- {% elif webmention.status == "deleted" %} + {% elif webmention.status == "source_removed" %}
{{ render_icon("x-circle-fill", size="5rem", title="Not verified") }}

Webmention not verified

{% elif webmention.status == "failed" %} diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py index 9b7c418..23abe65 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/admin.py @@ -15,10 +15,10 @@ from sqlalchemy.orm import selectinload from .. import forms from ..extensions import db -from ..models import ReceivedWebmention, SentWebmention, Source -from ..tasks.receiver import verify_webmention +from ..models import ReceivedWebmention, SentWebmention, SourceDocument from ..tasks.scanner import manual_scan_sources from ..tasks.sender import send_webmention +from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -80,7 +80,7 @@ def reverify_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnVa webmention = db.get_or_404(ReceivedWebmention, webmention_uuid) - webmention.reset_for_verification() + webmention.reopen_verification() db.session.commit() @@ -101,9 +101,9 @@ def source_list() -> ResponseReturnValue: :returns: Rendered page containing the paginated source list. """ sources = db.paginate( - sa.select(Source) - .options(selectinload(Source.sent_webmentions)) - .order_by(Source.last_seen_at.desc()), + sa.select(SourceDocument) + .options(selectinload(SourceDocument.sent_webmentions)) + .order_by(SourceDocument.last_seen_at.desc()), per_page=25, ) return render_template( @@ -138,11 +138,11 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: :param source_uuid: UUID of the source to display. :returns: Rendered source details page. """ - source = db.get_or_404(Source, source_uuid) + source = db.get_or_404(SourceDocument, source_uuid) webmentions = db.paginate( sa.select(SentWebmention) - .where(SentWebmention.source_id == source.uuid) + .where(SentWebmention.source_uuid == source.uuid) .order_by(SentWebmention.target, SentWebmention.uuid), per_page=25, ) @@ -177,13 +177,13 @@ def resend_webmention( webmention = db.get_or_404(SentWebmention, webmention_uuid) - if webmention.source_id != source_uuid: + if webmention.source_uuid != source_uuid: abort(HTTPStatus.NOT_FOUND) # Requeue the existing desired revision instead of inventing a new source # revision. send_webmention() otherwise exits when this revision has already # been processed. - webmention.resend() + webmention.clear_result_for_resend() db.session.commit() diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index be2a9ab..934ffad 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -17,8 +17,8 @@ from flask.typing import ResponseReturnValue from .. import forms from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter -from ..models import ReceivedWebmention, received_url_digest -from ..tasks.receiver import verify_webmention +from ..models import ReceivedWebmention, source_target_digest +from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -37,7 +37,8 @@ def find_received_webmention(source: str, target: str) -> ReceivedWebmention | N """ webmention = db.session.execute( sa.select(ReceivedWebmention).where( - ReceivedWebmention.url_digest == received_url_digest(source, target) + ReceivedWebmention.source_target_digest + == source_target_digest(source, target) ) ).scalar_one_or_none() @@ -92,7 +93,7 @@ def reopen_if_cooldown_elapsed(webmention: ReceivedWebmention) -> bool: ReceivedWebmention.uuid == webmention.uuid, ReceivedWebmention.updated_at <= cutoff, ) - .values(ReceivedWebmention.verification_reset_values(at=now)) + .values(ReceivedWebmention.reopen_verification_values(at=now)) .execution_options(synchronize_session=False) ) db.session.commit() -- cgit v1.3.1