aboutsummaryrefslogtreecommitdiff
path: root/webmentions_ssg
diff options
context:
space:
mode:
Diffstat (limited to 'webmentions_ssg')
-rw-r--r--webmentions_ssg/__init__.py4
-rw-r--r--webmentions_ssg/config.py19
-rw-r--r--webmentions_ssg/tasks/scanner.py15
3 files changed, 34 insertions, 4 deletions
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index e4a0a30..4c185c7 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -25,6 +25,7 @@ from webmentions_ssg.config import (
DevelopmentConfig,
ProductionConfig,
normalize_allowed_hostnames,
+ normalize_ignored_hostnames,
)
from .security import validate_secret_key
@@ -96,6 +97,9 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
app.config["ALLOWED_HOSTNAMES"]
)
+ app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames(
+ app.config["IGNORED_HOSTNAMES"]
+ )
for key in (
"MAX_CONTENT_LENGTH",
"MAX_WEBMENTION_URL_BYTES",
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index e91c651..b1d588b 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -22,6 +22,24 @@ def normalize_allowed_hostnames(value: object) -> set[str]:
return hostnames
+def normalize_ignored_hostnames(value: object) -> tuple[str, ...]:
+ """Validate and normalize hostname patterns ignored by the source scanner."""
+ if not isinstance(value, (list, tuple, set, frozenset)):
+ raise TypeError(
+ "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, "
+ "not a string."
+ )
+
+ patterns: set[str] = set()
+ for pattern in value:
+ if not isinstance(pattern, str) or not (
+ normalized := pattern.strip().lower().rstrip(".")
+ ):
+ raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.")
+ patterns.add(normalized)
+ return tuple(sorted(patterns))
+
+
class DefaultConfig:
TESTING = False
DEBUG = False
@@ -62,6 +80,7 @@ class DefaultConfig:
SOURCE_DIRECTORY = None
SOURCE_BASE_URL = None
+ IGNORED_HOSTNAMES = ()
SCANNER_SCHEDULE = "0 * * * *"
# Total send attempts per source revision (not additional retries).
SEND_MAX_ATTEMPTS = 6
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index e45085d..666eba7 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -25,6 +25,7 @@ from sqlalchemy.orm import selectinload
from .. import DATABASE as db
from .. import HUEY as huey
+from ..config import normalize_ignored_hostnames
from ..models import SentWebmention, Source
from ..url_security import is_http_url
from .sender import send_webmention
@@ -348,9 +349,16 @@ def scan_source_file(
Webmention source.
:raises OSError: If the source document cannot be read.
"""
- document = BeautifulSoup(path.read_bytes(), "html5lib")
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
@@ -515,9 +523,8 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = tuple(
- pattern.lower().rstrip(".")
- for pattern in current_app.config.get("IGNORED_HOSTNAMES", ())
+ ignored_hostnames = normalize_ignored_hostnames(
+ current_app.config["IGNORED_HOSTNAMES"]
)
sources_by_path = {