diff options
Diffstat (limited to '')
20 files changed, 344 insertions, 143 deletions
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index 7d09e7e..15667ab 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -36,7 +36,7 @@ class DefaultConfig: MAX_CONTENT_LENGTH = 64 * 1024 # Independent UTF-8 byte limit for each submitted source/target URL. MAX_WEBMENTION_URL_BYTES = 8192 - # Purge terminal failed/deleted received Webmentions after 30 days. + # Purge terminal failed/source-removed received Webmentions after 30 days. RECEIVED_RETENTION_DAYS = 30 RECEIVED_RETENTION_SCHEDULE = "0 3 * * *" MAX_REDIRECTS = 20 diff --git a/webmentions_ssg/extensions.py b/webmentions_ssg/extensions.py index c1cf396..d515d24 100644 --- a/webmentions_ssg/extensions.py +++ b/webmentions_ssg/extensions.py @@ -13,7 +13,7 @@ from flask_sqlalchemy import SQLAlchemy from flask_wtf.csrf import CSRFProtect from sqlalchemy.orm import DeclarativeBase -from .tasks.extension import Huey +from .tasks.extension import FlaskHuey class Base(DeclarativeBase): @@ -24,6 +24,6 @@ csrf = CSRFProtect() bootstrap = Bootstrap5() db = SQLAlchemy(model_class=Base) migrate = Migrate() -huey = Huey() +huey = FlaskHuey() mail = Mail() limiter = Limiter(key_func=get_remote_address) diff --git a/webmentions_ssg/forms.py b/webmentions_ssg/forms.py index c8ad0ba..1ce591d 100644 --- a/webmentions_ssg/forms.py +++ b/webmentions_ssg/forms.py @@ -104,7 +104,7 @@ class AllowedTargetHostname: raise ValidationError(self.message or field.gettext("Invalid input.")) -class PublicIPLiteral: +class RejectNonPublicIPLiteral: """Reject non-public IP literals without performing DNS in the web worker.""" def __call__(self, form: BaseForm, field: Field) -> None: @@ -138,7 +138,12 @@ class WebmentionSubmissionForm(FlaskForm): source = StringField( "source", - validators=[InputRequired(), MaxURLBytes(), StrictHTTPURL(), PublicIPLiteral()], + validators=[ + InputRequired(), + MaxURLBytes(), + StrictHTTPURL(), + RejectNonPublicIPLiteral(), + ], ) target = StringField( "target", diff --git a/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py b/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py new file mode 100644 index 0000000..23ff56a --- /dev/null +++ b/webmentions_ssg/migrations/versions/9a4e0c8211d6_clarify_webmention_names.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026 Dennis Fink <me+coding@dennisfink.me> +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Clarify source-document and Webmention lifecycle names. + +Revision ID: 9a4e0c8211d6 +Revises: 7c38bb1a935e +""" + +import sqlalchemy as sa +from alembic import op + +revision = "9a4e0c8211d6" +down_revision = "7c38bb1a935e" +branch_labels = None +depends_on = None + + +_OLD_RECEIVED_STATUS = sa.Enum( + "received", + "verifying", + "verified", + "failed", + "deleted", + name="received_webmention_status", + native_enum=False, +) +_NEW_RECEIVED_STATUS = sa.Enum( + "received", + "verifying", + "verified", + "failed", + "source_removed", + name="received_webmention_status", + native_enum=False, +) + + +def upgrade() -> None: + # RENAME TABLE retains existing source UUIDs and updates referencing FKs. + op.rename_table("sources", "source_documents") + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_url_digest", type_="unique") + batch.alter_column( + "url_digest", + new_column_name="source_target_digest", + existing_type=sa.String(64), + existing_nullable=False, + ) + # The new enum string is 14 characters, not the old VARCHAR(9). + batch.alter_column( + "status", + existing_type=_OLD_RECEIVED_STATUS, + type_=_NEW_RECEIVED_STATUS, + existing_nullable=False, + ) + + with op.batch_alter_table("received_webmentions") as batch: + batch.create_unique_constraint( + "uq_webmention_source_target_digest", ["source_target_digest"] + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_index("ix_sent_webmentions_source_id") + batch.drop_constraint("uq_sent_webmention_source_target_digest", type_="unique") + batch.alter_column( + "source_id", + new_column_name="source_uuid", + existing_type=sa.Uuid(), + existing_nullable=False, + ) + batch.alter_column( + "active", + new_column_name="linked", + existing_type=sa.Boolean(), + existing_nullable=False, + ) + batch.alter_column( + "endpoint", + new_column_name="webmention_endpoint", + existing_type=sa.Text(), + existing_nullable=True, + ) + batch.alter_column( + "response_status", + new_column_name="endpoint_response_status", + existing_type=sa.Integer(), + existing_nullable=True, + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.create_unique_constraint( + "uq_sent_webmention_source_uuid_target_digest", + ["source_uuid", "target_digest"], + ) + + op.create_index( + "ix_sent_webmentions_source_uuid", "sent_webmentions", ["source_uuid"] + ) + + op.execute( + "UPDATE received_webmentions SET status = 'source_removed' " + "WHERE status = 'deleted'" + ) + op.execute( + "UPDATE sent_webmentions SET status = 'no_endpoint' " + "WHERE status = 'unsupported'" + ) + + +def downgrade() -> None: + op.execute( + "UPDATE received_webmentions SET status = 'deleted' " + "WHERE status = 'source_removed'" + ) + op.execute( + "UPDATE sent_webmentions SET status = 'unsupported' " + "WHERE status = 'no_endpoint'" + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.drop_index("ix_sent_webmentions_source_uuid") + batch.drop_constraint( + "uq_sent_webmention_source_uuid_target_digest", type_="unique" + ) + batch.alter_column( + "source_uuid", + new_column_name="source_id", + existing_type=sa.Uuid(), + existing_nullable=False, + ) + batch.alter_column( + "linked", + new_column_name="active", + existing_type=sa.Boolean(), + existing_nullable=False, + ) + batch.alter_column( + "webmention_endpoint", + new_column_name="endpoint", + existing_type=sa.Text(), + existing_nullable=True, + ) + batch.alter_column( + "endpoint_response_status", + new_column_name="response_status", + existing_type=sa.Integer(), + existing_nullable=True, + ) + + with op.batch_alter_table("sent_webmentions") as batch: + batch.create_unique_constraint( + "uq_sent_webmention_source_target_digest", ["source_id", "target_digest"] + ) + + op.create_index("ix_sent_webmentions_source_id", "sent_webmentions", ["source_id"]) + + with op.batch_alter_table("received_webmentions") as batch: + batch.drop_constraint("uq_webmention_source_target_digest", type_="unique") + batch.alter_column( + "source_target_digest", + new_column_name="url_digest", + existing_type=sa.String(64), + existing_nullable=False, + ) + batch.alter_column( + "status", + existing_type=_NEW_RECEIVED_STATUS, + type_=_OLD_RECEIVED_STATUS, + existing_nullable=False, + ) + + with op.batch_alter_table("received_webmentions") as batch: + batch.create_unique_constraint("uq_webmention_url_digest", ["url_digest"]) + + op.rename_table("source_documents", "sources") diff --git a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py index 41c5376..8518b15 100644 --- a/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py +++ b/webmentions_ssg/migrations/versions/d845aa7fbc19_portable_url_fingerprints.py @@ -11,7 +11,8 @@ Revises: c0272f25ce17 import sqlalchemy as sa from alembic import op -from webmentions_ssg.models import received_url_digest, sent_target_digest +from webmentions_ssg.models import source_target_digest as received_url_digest +from webmentions_ssg.models import target_digest as sent_target_digest revision = "d845aa7fbc19" down_revision = "c0272f25ce17" diff --git a/webmentions_ssg/models.py b/webmentions_ssg/models.py index 33f27d1..2b375dd 100644 --- a/webmentions_ssg/models.py +++ b/webmentions_ssg/models.py @@ -32,21 +32,21 @@ from werkzeug.security import check_password_hash, generate_password_hash from .extensions import Base -class ReceivedStatus(StrEnum): +class ReceivedWebmentionStatus(StrEnum): """Persisted lifecycle state of an incoming Webmention.""" RECEIVED = "received" VERIFYING = "verifying" VERIFIED = "verified" FAILED = "failed" - DELETED = "deleted" + SOURCE_REMOVED = "source_removed" class SentWebmentionStatus(StrEnum): """Represent the outcome of a Webmention send attempt.""" SENT = "sent" - UNSUPPORTED = "unsupported" + NO_ENDPOINT = "no_endpoint" FAILED = "failed" @@ -110,25 +110,25 @@ class ReceivedWebmention(Base): source: Mapped[str] = mapped_column(Text(), nullable=False) target: Mapped[str] = mapped_column(Text(), nullable=False) - url_digest: Mapped[str] = mapped_column(String(64), nullable=False) + source_target_digest: Mapped[str] = mapped_column(String(64), nullable=False) @validates("source", "target") - def _update_url_digest(self, key: str, value: str) -> str: + def _refresh_url_digest(self, key: str, value: str) -> str: source = value if key == "source" else self.__dict__.get("source") target = value if key == "target" else self.__dict__.get("target") if source is not None and target is not None: - self.url_digest = received_url_digest(source, target) + self.source_target_digest = source_target_digest(source, target) return value - status: Mapped[ReceivedStatus] = mapped_column( + status: Mapped[ReceivedWebmentionStatus] = mapped_column( Enum( - ReceivedStatus, + ReceivedWebmentionStatus, name="received_webmention_status", values_callable=lambda enum_type: [member.value for member in enum_type], native_enum=False, ), nullable=False, - default=ReceivedStatus.RECEIVED, + default=ReceivedWebmentionStatus.RECEIVED, ) # Counts stale-row requeues, not Huey's bounded in-task retries. A fresh # explicit submission resets this counter. @@ -151,7 +151,9 @@ class ReceivedWebmention(Base): ) __table_args__ = ( - UniqueConstraint("url_digest", name="uq_webmention_url_digest"), + UniqueConstraint( + "source_target_digest", name="uq_webmention_source_target_digest" + ), Index("ix_received_webmentions_status_updated_at", "status", "updated_at"), ) @@ -160,35 +162,35 @@ class ReceivedWebmention(Base): return f"<ReceivedWebmention {self.source!r} -> {self.target!r}>" @staticmethod - def verification_reset_values( - *, at: datetime, recoveries: int = 0 + def reopen_verification_values( + *, at: datetime, recovery_attempts: int = 0 ) -> dict[str, object]: """Return the columns to reset when verification is explicitly reopened. :param at: Timestamp to record for the newly opened verification. - :param recoveries: Number of stale-lease recovery attempts to retain. + :param recovery_attempts: Number of stale-lease recovery attempts to retain. :returns: Column values for resetting the verification state. """ return { - "status": ReceivedStatus.RECEIVED, + "status": ReceivedWebmentionStatus.RECEIVED, "failure_reason": None, - "recovery_attempts": recoveries, + "recovery_attempts": recovery_attempts, "updated_at": at, } - def reset_for_verification(self) -> None: + def reopen_verification(self) -> None: """Reopen an existing row for manual verification. Clear the previous outcome and restore the received state with a fresh timestamp. """ - for key, value in self.verification_reset_values(at=datetime.now(UTC)).items(): + for key, value in self.reopen_verification_values(at=datetime.now(UTC)).items(): setattr(self, key, value) @property def verified(self) -> bool: """Return whether the Webmention has been successfully verified.""" - return self.status == ReceivedStatus.VERIFIED + return self.status == ReceivedWebmentionStatus.VERIFIED @property def created_at(self) -> datetime: @@ -196,10 +198,10 @@ class ReceivedWebmention(Base): return uuid7_to_datetime(self.uuid) -class Source(Base): +class SourceDocument(Base): """Represent a generated source document containing outgoing Webmentions.""" - __tablename__ = "sources" + __tablename__ = "source_documents" uuid: Mapped[UUID] = mapped_column( Uuid(as_uuid=True), primary_key=True, default=uuid7 @@ -227,7 +229,7 @@ class Source(Base): def __repr__(self) -> str: """Represent the source by its public URL.""" - return f"<Source {self.url!r}>" + return f"<SourceDocument {self.url!r}>" @property def created_at(self) -> datetime: @@ -244,9 +246,9 @@ class SentWebmention(Base): Uuid(as_uuid=True), primary_key=True, default=uuid7 ) - source_id: Mapped[UUID] = mapped_column( + source_uuid: Mapped[UUID] = mapped_column( Uuid(as_uuid=True), - ForeignKey("sources.uuid", ondelete="CASCADE"), + ForeignKey("source_documents.uuid", ondelete="CASCADE"), nullable=False, index=True, ) @@ -256,10 +258,10 @@ class SentWebmention(Base): @validates("target") def _update_target_digest(self, key: str, value: str) -> str: - self.target_digest = sent_target_digest(value) + self.target_digest = target_digest(value) return value - active: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) + linked: Mapped[bool] = mapped_column(Boolean(), nullable=False, default=True) desired_revision: Mapped[int] = mapped_column(Integer(), nullable=False) processed_revision: Mapped[int | None] = mapped_column( @@ -284,8 +286,10 @@ class SentWebmention(Base): Text(), nullable=True, default=None ) - endpoint: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) - response_status: Mapped[int | None] = mapped_column( + webmention_endpoint: Mapped[str | None] = mapped_column( + Text(), nullable=True, default=None + ) + endpoint_response_status: Mapped[int | None] = mapped_column( Integer(), nullable=True, default=None ) status_url: Mapped[str | None] = mapped_column(Text(), nullable=True, default=None) @@ -303,11 +307,13 @@ class SentWebmention(Base): DateTime(timezone=True), nullable=True, default=None ) - source: Mapped[Source] = relationship(back_populates="sent_webmentions") + source: Mapped[SourceDocument] = relationship(back_populates="sent_webmentions") __table_args__ = ( UniqueConstraint( - "source_id", "target_digest", name="uq_sent_webmention_source_target_digest" + "source_uuid", + "target_digest", + name="uq_sent_webmention_source_uuid_target_digest", ), ) @@ -315,41 +321,41 @@ class SentWebmention(Base): """Represent the sent Webmention by its target URL.""" return f"<SentWebmention {self.target!r}>" - def reopen(self, revision: int, active: bool) -> None: + def reset_for_revision(self, revision: int, linked: bool) -> None: """Reopen delivery for a revision while preserving send history. :param revision: Source revision to make the desired delivery revision. - :param active: Whether the target still appears in the current source. + :param linked: Whether the target still appears in the current source. """ self.desired_revision = revision - self.active = active + self.linked = linked self.attempt_count = 0 self.next_retry_at = None - def resend(self) -> None: + def clear_result_for_resend(self) -> None: """Retry the current revision, clearing its previous delivery result. Reset attempt and response state without changing the desired source revision. """ - self.reopen(self.desired_revision, self.active) + self.reset_for_revision(self.desired_revision, self.linked) self.processed_revision = None self.status = None self.failure_reason = None - self.endpoint = None - self.response_status = None + self.webmention_endpoint = None + self.endpoint_response_status = None self.status_url = None @hybrid_property - def pending(self) -> bool: + def awaiting_processing(self) -> bool: """Return whether the Webmention has an unprocessed source revision.""" return ( self.processed_revision is None or self.processed_revision < self.desired_revision ) - @pending.inplace.expression + @awaiting_processing.inplace.expression @classmethod - def _pending_expression(cls) -> sa.ColumnElement[bool]: + def _awaiting_processing_expression(cls) -> sa.ColumnElement[bool]: """Express pending-revision status in database queries. :returns: SQLAlchemy expression selecting unprocessed source revisions. @@ -374,7 +380,7 @@ def uuid7_to_datetime(identifier: UUID) -> datetime: return datetime.fromtimestamp(identifier.time / 1000, tz=UTC) -def received_url_digest(source: str, target: str) -> str: +def source_target_digest(source: str, target: str) -> str: """Fingerprint an unambiguous, ordered source/target pair. :param source: Webmention source URL. @@ -387,7 +393,7 @@ def received_url_digest(source: str, target: str) -> str: return sha256(encoded).hexdigest() -def sent_target_digest(target: str) -> str: +def target_digest(target: str) -> str: """Fingerprint an outgoing target (unique within each source ID). :param target: Outgoing Webmention target URL. diff --git a/webmentions_ssg/tasks/__init__.py b/webmentions_ssg/tasks/__init__.py index c744b49..37ab4d2 100644 --- a/webmentions_ssg/tasks/__init__.py +++ b/webmentions_ssg/tasks/__init__.py @@ -4,6 +4,6 @@ """Expose the background task queue integration.""" -from .extension import Huey +from .extension import FlaskHuey -__all__ = ["Huey"] +__all__ = ["FlaskHuey"] diff --git a/webmentions_ssg/tasks/consumer.py b/webmentions_ssg/tasks/consumer.py index c24a883..7991b6a 100644 --- a/webmentions_ssg/tasks/consumer.py +++ b/webmentions_ssg/tasks/consumer.py @@ -8,11 +8,11 @@ from huey import Huey, crontab from .. import create_app from ..extensions import huey as queue -from . import notifications, receiver, retention, scanner +from . import notifications, retention, scanner, verification -def create_consumer() -> Huey: - """Create and configure the Huey consumer. +def create_huey() -> Huey: + """Create and configure the Huey instance for the task worker. The application is initialized and the periodic scanner, notification, verification recovery and retention tasks are registered. @@ -23,8 +23,8 @@ def create_consumer() -> Huey: for config_key, task in ( ("SOURCE_SCAN_SCHEDULE", scanner.scan_sources), - ("VERIFICATION_RECOVERY_SCHEDULE", receiver.recover_stale_verifications), - ("DIGEST_SCHEDULE", notifications.send_webmention_digest), + ("VERIFICATION_RECOVERY_SCHEDULE", verification.recover_stale_verifications), + ("DIGEST_SCHEDULE", notifications.email_verified_digest), ("RECEIVED_RETENTION_SCHEDULE", retention.purge_old_received_webmentions), ): schedule = crontab(*app.config[config_key].split(), strict=True) @@ -33,4 +33,4 @@ def create_consumer() -> Huey: return queue.huey -huey = create_consumer() +huey = create_huey() diff --git a/webmentions_ssg/tasks/discovery.py b/webmentions_ssg/tasks/endpoint_discovery.py index c683cb0..c683cb0 100644 --- a/webmentions_ssg/tasks/discovery.py +++ b/webmentions_ssg/tasks/endpoint_discovery.py diff --git a/webmentions_ssg/tasks/extension.py b/webmentions_ssg/tasks/extension.py index faed364..3f172f4 100644 --- a/webmentions_ssg/tasks/extension.py +++ b/webmentions_ssg/tasks/extension.py @@ -4,8 +4,6 @@ """Integrate Huey tasks with Flask application contexts.""" -from __future__ import annotations - from collections.abc import Callable from functools import update_wrapper, wraps from typing import Any @@ -100,7 +98,7 @@ class _DeferredTask: def __init__( self, - extension: Huey, + extension: FlaskHuey, func: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any], @@ -152,7 +150,7 @@ class _DeferredTask: return getattr(self._bound_task(), name) -class Huey: +class FlaskHuey: """Provide Flask integration for a Huey instance. The extension initializes a Huey backend from the Flask configuration and diff --git a/webmentions_ssg/tasks/notifications.py b/webmentions_ssg/tasks/notifications.py index eb540de..c503e1f 100644 --- a/webmentions_ssg/tasks/notifications.py +++ b/webmentions_ssg/tasks/notifications.py @@ -11,10 +11,10 @@ from flask import current_app from flask_mail import Message from ..extensions import db, huey, mail -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus -def send_webmention_digest() -> None: +def email_verified_digest() -> None: """Send one digest of verified, previously unreported Webmentions. Acquire the digest lock before notifying recipients about previously unreported @@ -38,7 +38,7 @@ def _send_digest_locked() -> None: cutoff = datetime.now(UTC) conditions = ( - ReceivedWebmention.status == ReceivedStatus.VERIFIED, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFIED, ReceivedWebmention.notification_sent_at.is_(None), ReceivedWebmention.updated_at <= cutoff, ) diff --git a/webmentions_ssg/tasks/retention.py b/webmentions_ssg/tasks/retention.py index 2c35cc7..b882493 100644 --- a/webmentions_ssg/tasks/retention.py +++ b/webmentions_ssg/tasks/retention.py @@ -11,7 +11,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus def purge_old_received_webmentions() -> int: @@ -33,7 +33,10 @@ def purge_old_received_webmentions() -> int: result = db.session.execute( sa.delete(ReceivedWebmention).where( ReceivedWebmention.status.in_( - (ReceivedStatus.FAILED, ReceivedStatus.DELETED) + ( + ReceivedWebmentionStatus.FAILED, + ReceivedWebmentionStatus.SOURCE_REMOVED, + ) ), ReceivedWebmention.updated_at < cutoff, ) diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index 034b2cf..fb0c770 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -14,13 +14,13 @@ from huey.exceptions import RetryTask, TaskLockedException from sqlalchemy.orm import selectinload from ..extensions import db, huey -from ..models import SentWebmention, Source +from ..models import SentWebmention, SourceDocument from ..net import is_http_url from .sender import send_webmention from .source_parser import ScannedSource, SourceScanError, scan_source_file -def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: +def build_source(scanned: ScannedSource, scan_time: datetime) -> SourceDocument: """Build a source model from a newly discovered document. A sent Webmention is built for each discovered target at the initial source @@ -30,7 +30,7 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: :param scan_time: Time at which the source was discovered. :returns: Newly built source model. """ - source = Source( + source = SourceDocument( path=scanned.path, url=scanned.url, content_hash=scanned.content_hash, @@ -42,15 +42,15 @@ def build_source(scanned: ScannedSource, scan_time: datetime) -> Source: for target in scanned.targets: source.sent_webmentions.append( - SentWebmention(target=target, active=True, desired_revision=1) + SentWebmention(target=target, linked=True, desired_revision=1) ) return source def apply_scan_to_source( - source: Source, scanned: ScannedSource, scan_time: datetime -) -> Source: + source: SourceDocument, scanned: ScannedSource, scan_time: datetime +) -> SourceDocument: """Update an existing source from newly scanned data. A changed or previously deleted source receives a new revision. Existing sent @@ -82,16 +82,18 @@ def apply_scan_to_source( existing = {webmention.target: webmention for webmention in source.sent_webmentions} for webmention in existing.values(): - webmention.reopen(source.revision, webmention.target in scanned.targets) + webmention.reset_for_revision( + source.revision, webmention.target in scanned.targets + ) - if not webmention.active and webmention.sent_revision is None: + if not webmention.linked and webmention.sent_revision is None: webmention.processed_revision = source.revision for target in scanned.targets: if target not in existing: source.sent_webmentions.append( SentWebmention( - target=target, active=True, desired_revision=source.revision + target=target, linked=True, desired_revision=source.revision ) ) @@ -127,7 +129,7 @@ class TreeScanResult: def reconcile_source_tree( root: Path, base_url: str | None, - sources_by_path: dict[str, Source], + sources_by_path: dict[str, SourceDocument], paths_by_url: dict[str, str], scan_time: datetime, ) -> TreeScanResult: @@ -197,14 +199,16 @@ def reconcile_source_tree( def mark_missing_sources_deleted( - sources_by_path: dict[str, Source], seen_paths: frozenset[str], scan_time: datetime + sources_by_path: dict[str, SourceDocument], + seen_paths: frozenset[str], + scan_time: datetime, ) -> int: - """Mark absent, previously active sources deleted and return their count. + """Mark absent, previously present sources deleted and return their count. :param sources_by_path: Known source records indexed by relative path. :param seen_paths: Paths found during this scan, including failed parses. :param scan_time: Timestamp to record for sources newly marked missing. - :returns: Number of previously active sources marked as deleted. + :returns: Number of previously linked sources marked as deleted. """ marked = 0 for path, source in sources_by_path.items(): @@ -217,7 +221,7 @@ def mark_missing_sources_deleted( marked += 1 for webmention in source.sent_webmentions: - webmention.reopen(source.revision, False) + webmention.reset_for_revision(source.revision, False) if webmention.sent_revision is None: webmention.processed_revision = source.revision @@ -235,7 +239,7 @@ def enqueue_due_sends(scan_time: datetime) -> int: for identifier in db.session.scalars( sa.select(SentWebmention.uuid) .where( - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= scan_time, @@ -273,7 +277,9 @@ def _scan_and_enqueue() -> None: sources_by_path = { source.path: source for source in db.session.scalars( - sa.select(Source).options(selectinload(Source.sent_webmentions)) + sa.select(SourceDocument).options( + selectinload(SourceDocument.sent_webmentions) + ) ) } diff --git a/webmentions_ssg/tasks/sender.py b/webmentions_ssg/tasks/sender.py index c33cf06..3f64bed 100644 --- a/webmentions_ssg/tasks/sender.py +++ b/webmentions_ssg/tasks/sender.py @@ -24,7 +24,7 @@ from ..net import ( is_retryable_status, ) from ..version import APP_NAME, VERSION -from .discovery import ( +from .endpoint_discovery import ( PermanentDeliveryError, RetryableDeliveryError, discover_webmention_endpoint, @@ -44,7 +44,7 @@ def revision_still_pending(webmention: SentWebmention, revision: int) -> bool: """ db.session.refresh(webmention) - return webmention.desired_revision == revision and webmention.pending + return webmention.desired_revision == revision and webmention.awaiting_processing def backoff_delay(attempt: int) -> timedelta: @@ -99,8 +99,8 @@ class ClaimedSendAttempt: class DeliveryResult: """HTTP outcome, with whether the failure is eligible for another attempt. - ``terminal`` describes the failure type. The retry budget can also make a - temporary failure terminal when its last allowed attempt has been used. + ``retryable`` describes the failure type; it does not account for the + remaining retry budget, which may already have been exhausted. """ status: SentWebmentionStatus @@ -108,8 +108,7 @@ class DeliveryResult: endpoint: str | None = None response_status: int | None = None status_url: str | None = None - terminal: bool = True - temporary_failure: bool = False + retryable: bool = False def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: @@ -119,7 +118,7 @@ def is_ready_to_send(webmention: SentWebmention, now: datetime) -> bool: :param now: Current timestamp against which the retry deadline is compared. :returns: ``True`` if the Webmention is pending and its retry deadline has elapsed. """ - if not webmention.pending: + if not webmention.awaiting_processing: return False deadline = webmention.next_retry_at @@ -154,7 +153,7 @@ def reserve_send_attempt( SentWebmention.uuid == webmention.uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count >= max_attempts, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -188,7 +187,7 @@ def reserve_send_attempt( SentWebmention.uuid == attempt.webmention_uuid, SentWebmention.desired_revision == revision, SentWebmention.attempt_count == attempt.attempt_number - 1, - SentWebmention.pending, + SentWebmention.awaiting_processing, sa.or_( SentWebmention.next_retry_at.is_(None), SentWebmention.next_retry_at <= now, @@ -198,8 +197,8 @@ def reserve_send_attempt( attempt_count=attempt.attempt_number, last_attempted_at=now, next_retry_at=now + backoff_delay(attempt.attempt_number), - endpoint=None, - response_status=None, + webmention_endpoint=None, + endpoint_response_status=None, status_url=None, ) .execution_options(synchronize_session=False) @@ -236,7 +235,7 @@ def attempt_delivery( if endpoint is None: return DeliveryResult( - status=SentWebmentionStatus.UNSUPPORTED, + status=SentWebmentionStatus.NO_ENDPOINT, reason="No Webmention endpoint discovered", ) @@ -275,11 +274,10 @@ def attempt_delivery( reason=str(exc) or "Webmention request failed", endpoint=endpoint, response_status=response_status, - terminal=False, - temporary_failure=True, + retryable=True, ) except Exception as exc: - # Unexpected errors should be terminal and keep their traceback. + # Unexpected errors should not be retried; keep their traceback. current_app.logger.exception( "Unexpected error sending SentWebmention %s", webmention_uuid ) @@ -312,8 +310,8 @@ def record_delivery_outcome( webmention.status = result.status webmention.failure_reason = result.reason - webmention.endpoint = result.endpoint - webmention.response_status = result.response_status + webmention.webmention_endpoint = result.endpoint + webmention.endpoint_response_status = result.response_status if terminal: webmention.processed_revision = attempt.revision @@ -355,19 +353,19 @@ def _process_send_locked(webmention_uuid: uuid.UUID) -> bool: result = attempt_delivery( attempt.source, attempt.target, webmention_uuid=webmention_uuid ) - terminal = result.terminal or attempt.attempt_number >= attempt.max_attempts + terminal = not result.retryable or attempt.attempt_number >= attempt.max_attempts if not record_delivery_outcome(webmention, attempt, result, terminal=terminal): return False - if result.temporary_failure and not terminal: + if result.retryable and not terminal: # Huey does not retry: the scanner requeues after next_retry_at. current_app.logger.info( "Webmention %s send attempt %d failed; will retry", webmention_uuid, attempt.attempt_number, ) - elif result.temporary_failure: + elif result.retryable: current_app.logger.warning( "Webmention %s exhausted %d send attempts: %s", webmention_uuid, diff --git a/webmentions_ssg/tasks/receiver.py b/webmentions_ssg/tasks/verification.py index 313a8e4..a614815 100644 --- a/webmentions_ssg/tasks/receiver.py +++ b/webmentions_ssg/tasks/verification.py @@ -15,7 +15,7 @@ from flask import current_app from ..db_helpers import rows_affected from ..extensions import db, huey -from ..models import ReceivedStatus, ReceivedWebmention +from ..models import ReceivedWebmention, ReceivedWebmentionStatus from ..net import ( AddressResolutionError, ResponseTooLargeError, @@ -149,11 +149,14 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No ) return None - if webmention.status not in {ReceivedStatus.RECEIVED, ReceivedStatus.FAILED}: + if webmention.status not in { + ReceivedWebmentionStatus.RECEIVED, + ReceivedWebmentionStatus.FAILED, + }: return None if ( - webmention.status == ReceivedStatus.FAILED + webmention.status == ReceivedWebmentionStatus.FAILED and webmention.failure_reason == RECOVERY_EXHAUSTED_REASON ): return None @@ -172,7 +175,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No ReceivedWebmention.updated_at == webmention.updated_at, ) .values( - status=ReceivedStatus.VERIFYING, + status=ReceivedWebmentionStatus.VERIFYING, failure_reason=None, updated_at=claim.lease_started_at, ) @@ -188,7 +191,7 @@ def claim_for_verification(webmention_uuid: uuid.UUID) -> VerificationClaim | No def determine_verification_outcome( claim: VerificationClaim, -) -> tuple[ReceivedStatus, str | None]: +) -> tuple[ReceivedWebmentionStatus, str | None]: """Map verification results and permanent errors to a final status/reason. :param claim: Claim containing the source and target URLs to verify. @@ -200,9 +203,9 @@ def determine_verification_outcome( try: mentions_target = source_mentions_target(claim.source, claim.target) except SourceGoneError as exc: - return ReceivedStatus.DELETED, str(exc) + return ReceivedWebmentionStatus.SOURCE_REMOVED, str(exc) except VerificationError as exc: - return ReceivedStatus.FAILED, str(exc) + return ReceivedWebmentionStatus.FAILED, str(exc) except RetryableFetchError, httpx.RequestError: raise # These errors must be persisted and then retried by Huey. except Exception as exc: @@ -210,28 +213,28 @@ def determine_verification_outcome( current_app.logger.exception( "Unexpected error verifying ReceivedWebmention %s", claim.webmention_uuid ) - return ReceivedStatus.FAILED, str(exc) or type(exc).__name__ + return ReceivedWebmentionStatus.FAILED, str(exc) or type(exc).__name__ if mentions_target: - return ReceivedStatus.VERIFIED, None - return ReceivedStatus.DELETED, "Source does not mention target" + return ReceivedWebmentionStatus.VERIFIED, None + return ReceivedWebmentionStatus.SOURCE_REMOVED, "Source does not mention target" -def release_claim( - claim: VerificationClaim, status: ReceivedStatus, reason: str | None +def store_verification_outcome( + claim: VerificationClaim, status: ReceivedWebmentionStatus, reason: str | None ) -> bool: """Persist an outcome only if this worker still owns the verification lease. :param claim: Verification lease acquired by this worker. :param status: Final status to persist for the claimed Webmention. - :param reason: Optional explanation for a failed or deleted Webmention. + :param reason: Optional explanation for a failed or source-removed Webmention. :returns: ``True`` if the worker still owned the lease and persisted the outcome. """ changed = db.session.execute( sa.update(ReceivedWebmention) .where( ReceivedWebmention.uuid == claim.webmention_uuid, - ReceivedWebmention.status == ReceivedStatus.VERIFYING, + ReceivedWebmention.status == ReceivedWebmentionStatus.VERIFYING, ReceivedWebmention.updated_at == claim.lease_started_at, ) .values(status=status, failure_reason=reason, updated_at=datetime.now(UTC)) @@ -259,12 +262,14 @@ def verify_webmention(webmention_uuid: uuid.UUID) -> None: try: status, reason = determine_verification_outcome(claim) except (RetryableFetchError, httpx.RequestError) as exc: - release_claim( - claim, ReceivedStatus.FAILED, str(exc) or "Source could not be fetched" + store_verification_outcome( + claim, + ReceivedWebmentionStatus.FAILED, + str(exc) or "Source could not be fetched", ) raise - release_claim(claim, status, reason) + store_verification_outcome(claim, status, reason) def recover_stale_verifications() -> int: @@ -296,7 +301,7 @@ def recover_stale_verifications() -> int: ReceivedWebmention.recovery_attempts, ).where( ReceivedWebmention.status.in_( - (ReceivedStatus.VERIFYING, ReceivedStatus.RECEIVED) + (ReceivedWebmentionStatus.VERIFYING, ReceivedWebmentionStatus.RECEIVED) ), ReceivedWebmention.updated_at <= cutoff, ) @@ -319,14 +324,14 @@ def recover_stale_verifications() -> int: .values( **( { - "status": ReceivedStatus.FAILED, + "status": ReceivedWebmentionStatus.FAILED, "failure_reason": RECOVERY_EXHAUSTED_REASON, "updated_at": now, "recovery_attempts": recovery_attempts, } if at_limit - else ReceivedWebmention.verification_reset_values( - at=now, recoveries=recovery_attempts + 1 + else ReceivedWebmention.reopen_verification_values( + at=now, recovery_attempts=recovery_attempts + 1 ) ) ) diff --git a/webmentions_ssg/templates/received.html b/webmentions_ssg/templates/received.html index f00a5f9..f94d104 100644 --- a/webmentions_ssg/templates/received.html +++ b/webmentions_ssg/templates/received.html @@ -27,7 +27,7 @@ SPDX-License-Identifier: BSD-3-Clause "badge_class": "text-bg-danger", "label": "Declined" }, - "deleted": { + "source_removed": { "row_class": "table-danger", "badge_class": "text-bg-danger", "label": "Declined" diff --git a/webmentions_ssg/templates/sent_source.html b/webmentions_ssg/templates/sent_source.html index 31d5c81..2b7a377 100644 --- a/webmentions_ssg/templates/sent_source.html +++ b/webmentions_ssg/templates/sent_source.html @@ -12,7 +12,7 @@ SPDX-License-Identifier: BSD-3-Clause "badge_class": "text-bg-success", "label": "Sent" }, - "unsupported": { + "no_endpoint": { "row_class": "table-warning", "badge_class": "text-bg-warning", "label": "Unsupported" @@ -114,7 +114,7 @@ SPDX-License-Identifier: BSD-3-Clause {% endif %} </td> <td> - {% if webmention.active %} + {% if webmention.linked %} <span class="badge text-bg-success">Active</span> {% else %} <span class="badge text-bg-danger">Removed</span> @@ -139,16 +139,16 @@ SPDX-License-Identifier: BSD-3-Clause </div> </td> <td class="small"> - {% if webmention.endpoint is not none %} + {% if webmention.webmention_endpoint is not none %} <div class="mb-1"> <strong>Endpoint:</strong> - <a href="{{ webmention.endpoint }}" class="text-break">{{ webmention.endpoint }}</a> + <a href="{{ webmention.webmention_endpoint }}" class="text-break">{{ webmention.webmention_endpoint }}</a> </div> {% endif %} - {% if webmention.response_status is not none %} + {% if webmention.endpoint_response_status is not none %} <div class="mb-1"> <strong>HTTP:</strong> - {{ webmention.response_status }} + {{ webmention.endpoint_response_status }} </div> {% endif %} {% if webmention.status_url is not none %} diff --git a/webmentions_ssg/templates/status.html b/webmentions_ssg/templates/status.html index 2db2741..a9a39b4 100644 --- a/webmentions_ssg/templates/status.html +++ b/webmentions_ssg/templates/status.html @@ -16,7 +16,7 @@ SPDX-License-Identifier: BSD-3-Clause {% elif webmention.status in ("received", "verifying") %} <div class="text-warning mb-3">{{ render_icon("hourglass-split", size="5rem", title="Verification pending") }}</div> <h1 class="h3">Webmention verification pending</h1> - {% elif webmention.status == "deleted" %} + {% elif webmention.status == "source_removed" %} <div class="text-danger mb-3">{{ render_icon("x-circle-fill", size="5rem", title="Not verified") }}</div> <h1 class="h3">Webmention not verified</h1> {% elif webmention.status == "failed" %} diff --git a/webmentions_ssg/views/admin.py b/webmentions_ssg/views/admin.py index 9b7c418..23abe65 100644 --- a/webmentions_ssg/views/admin.py +++ b/webmentions_ssg/views/admin.py @@ -15,10 +15,10 @@ from sqlalchemy.orm import selectinload from .. import forms from ..extensions import db -from ..models import ReceivedWebmention, SentWebmention, Source -from ..tasks.receiver import verify_webmention +from ..models import ReceivedWebmention, SentWebmention, SourceDocument from ..tasks.scanner import manual_scan_sources from ..tasks.sender import send_webmention +from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -80,7 +80,7 @@ def reverify_received_webmention(webmention_uuid: uuid.UUID) -> ResponseReturnVa webmention = db.get_or_404(ReceivedWebmention, webmention_uuid) - webmention.reset_for_verification() + webmention.reopen_verification() db.session.commit() @@ -101,9 +101,9 @@ def source_list() -> ResponseReturnValue: :returns: Rendered page containing the paginated source list. """ sources = db.paginate( - sa.select(Source) - .options(selectinload(Source.sent_webmentions)) - .order_by(Source.last_seen_at.desc()), + sa.select(SourceDocument) + .options(selectinload(SourceDocument.sent_webmentions)) + .order_by(SourceDocument.last_seen_at.desc()), per_page=25, ) return render_template( @@ -138,11 +138,11 @@ def source_detail(source_uuid: uuid.UUID) -> ResponseReturnValue: :param source_uuid: UUID of the source to display. :returns: Rendered source details page. """ - source = db.get_or_404(Source, source_uuid) + source = db.get_or_404(SourceDocument, source_uuid) webmentions = db.paginate( sa.select(SentWebmention) - .where(SentWebmention.source_id == source.uuid) + .where(SentWebmention.source_uuid == source.uuid) .order_by(SentWebmention.target, SentWebmention.uuid), per_page=25, ) @@ -177,13 +177,13 @@ def resend_webmention( webmention = db.get_or_404(SentWebmention, webmention_uuid) - if webmention.source_id != source_uuid: + if webmention.source_uuid != source_uuid: abort(HTTPStatus.NOT_FOUND) # Requeue the existing desired revision instead of inventing a new source # revision. send_webmention() otherwise exits when this revision has already # been processed. - webmention.resend() + webmention.clear_result_for_resend() db.session.commit() diff --git a/webmentions_ssg/views/endpoint.py b/webmentions_ssg/views/endpoint.py index be2a9ab..934ffad 100644 --- a/webmentions_ssg/views/endpoint.py +++ b/webmentions_ssg/views/endpoint.py @@ -17,8 +17,8 @@ from flask.typing import ResponseReturnValue from .. import forms from ..db_helpers import rows_affected from ..extensions import csrf, db, limiter -from ..models import ReceivedWebmention, received_url_digest -from ..tasks.receiver import verify_webmention +from ..models import ReceivedWebmention, source_target_digest +from ..tasks.verification import verify_webmention from .blueprint import webmentions_bp @@ -37,7 +37,8 @@ def find_received_webmention(source: str, target: str) -> ReceivedWebmention | N """ webmention = db.session.execute( sa.select(ReceivedWebmention).where( - ReceivedWebmention.url_digest == received_url_digest(source, target) + ReceivedWebmention.source_target_digest + == source_target_digest(source, target) ) ).scalar_one_or_none() @@ -92,7 +93,7 @@ def reopen_if_cooldown_elapsed(webmention: ReceivedWebmention) -> bool: ReceivedWebmention.uuid == webmention.uuid, ReceivedWebmention.updated_at <= cutoff, ) - .values(ReceivedWebmention.verification_reset_values(at=now)) + .values(ReceivedWebmention.reopen_verification_values(at=now)) .execution_options(synchronize_session=False) ) db.session.commit() |
