aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--tests/tasks/test_scanner.py74
-rw-r--r--tests/test_config.py34
-rw-r--r--webmentions_ssg/__init__.py4
-rw-r--r--webmentions_ssg/config.py19
-rw-r--r--webmentions_ssg/tasks/scanner.py15
5 files changed, 142 insertions, 4 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py
index 5a1e5f2..26e81c4 100644
--- a/tests/tasks/test_scanner.py
+++ b/tests/tasks/test_scanner.py
@@ -1979,6 +1979,80 @@ def test_unexpected_persistence_error_is_not_suppressed(
run_scan(app, scanner_module)
+def test_scan_source_rejects_symlink_outside_root_before_read(
+ scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ external = write_post(tmp_path / "outside", "private", "Sensitive content")
+ linked = root / "escaped" / "index.html"
+ linked.parent.mkdir()
+ linked.symlink_to(external)
+
+ read_bytes = Mock(side_effect=AssertionError("External file must not be read"))
+ monkeypatch.setattr(Path, "read_bytes", read_bytes)
+
+ with pytest.raises(
+ scanner_module.SourceScanError, match="escapes source directory"
+ ):
+ scanner_module.scan_source_file(linked, root=root, base_url=None)
+
+ read_bytes.assert_not_called()
+
+
+def test_scan_source_accepts_symlink_within_root(
+ scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ linked = write_post(
+ root, "example", '<a href="https://example.com/target">Target</a>'
+ )
+ shared = root / "shared.html"
+ linked.rename(shared)
+ linked.symlink_to(shared)
+
+ scanned = scanner_module.scan_source_file(linked, root=root, base_url=None)
+
+ assert scanned.path == "example/index.html"
+ assert scanned.url == SOURCE_URL
+ assert scanned.targets == frozenset({"https://example.com/target"})
+
+
+def test_scan_sources_skips_symlink_outside_root(
+ app: Flask,
+ scanner_module: ModuleType,
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ root = tmp_path / "site"
+ root.mkdir()
+ write_post(root, "example", "Normal content")
+ external = write_post(tmp_path / "outside", "private", "Outside content")
+ linked = root / "escaped" / "index.html"
+ linked.parent.mkdir()
+ linked.symlink_to(external)
+ configure_scanner(app, root)
+
+ run_scan(app, scanner_module)
+
+ with app.app_context():
+ sources = list(db.session.scalars(sa.select(Source)))
+ assert len(sources) == 1
+ assert sources[0].path == "example/index.html"
+ assert "escapes source directory" in caplog.text
+
+
+def test_scan_sources_rejects_malformed_runtime_ignored_hostnames(
+ app: Flask, scanner_module: ModuleType, tmp_path: Path
+) -> None:
+ configure_scanner(app, tmp_path)
+ app.config["IGNORED_HOSTNAMES"] = "example.com"
+
+ with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"):
+ run_scan(app, scanner_module)
+
+
def test_scan_sources_accepts_valid_base_url(
app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch
) -> None:
diff --git a/tests/test_config.py b/tests/test_config.py
index 933f437..aae7cce 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -75,6 +75,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None:
assert config.REQUEST_TOTAL_TIMEOUT == 30.0
assert DefaultConfig.ALLOWED_HOSTNAMES == set()
+ assert DefaultConfig.IGNORED_HOSTNAMES == ()
def test_allowed_hostnames_are_normalized_at_startup(tmp_path: Path) -> None:
@@ -124,6 +125,39 @@ def test_invalid_allowed_hostnames_fail_at_startup(
create_app(InvalidHostnameConfig)
+def test_ignored_hostnames_are_normalized_at_startup(tmp_path: Path) -> None:
+ class MixedCaseConfig(TestingConfig):
+ LOG_FILE = str(tmp_path / "test.log")
+ IGNORED_HOSTNAMES = [" Example.COM. ", "*.EXAMPLE.com.", "example.com"]
+
+ app = create_app(MixedCaseConfig)
+ assert app.config["IGNORED_HOSTNAMES"] == ("*.example.com", "example.com")
+
+
+@pytest.mark.parametrize(
+ "hostnames,expected_exception",
+ [
+ ("example.com", TypeError),
+ (None, TypeError),
+ ({"example.com": True}, TypeError),
+ (123, TypeError),
+ (["example.com", None], ValueError),
+ (["example.com", ""], ValueError),
+ (["example.com", " "], ValueError),
+ (["example.com", "..."], ValueError),
+ ],
+)
+def test_invalid_ignored_hostnames_fail_at_startup(
+ tmp_path: Path, hostnames: object, expected_exception: type[Exception]
+) -> None:
+ class InvalidHostnameConfig(TestingConfig):
+ LOG_FILE = str(tmp_path / "test.log")
+ IGNORED_HOSTNAMES = hostnames
+
+ with pytest.raises(expected_exception, match="IGNORED_HOSTNAMES"):
+ create_app(InvalidHostnameConfig)
+
+
def test_production_requires_nonempty_allowed_hostnames(tmp_path: Path) -> None:
config_path = tmp_path / "production.cfg"
config_path.write_text(
diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py
index e4a0a30..4c185c7 100644
--- a/webmentions_ssg/__init__.py
+++ b/webmentions_ssg/__init__.py
@@ -25,6 +25,7 @@ from webmentions_ssg.config import (
DevelopmentConfig,
ProductionConfig,
normalize_allowed_hostnames,
+ normalize_ignored_hostnames,
)
from .security import validate_secret_key
@@ -96,6 +97,9 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask:
app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames(
app.config["ALLOWED_HOSTNAMES"]
)
+ app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames(
+ app.config["IGNORED_HOSTNAMES"]
+ )
for key in (
"MAX_CONTENT_LENGTH",
"MAX_WEBMENTION_URL_BYTES",
diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py
index e91c651..b1d588b 100644
--- a/webmentions_ssg/config.py
+++ b/webmentions_ssg/config.py
@@ -22,6 +22,24 @@ def normalize_allowed_hostnames(value: object) -> set[str]:
return hostnames
+def normalize_ignored_hostnames(value: object) -> tuple[str, ...]:
+ """Validate and normalize hostname patterns ignored by the source scanner."""
+ if not isinstance(value, (list, tuple, set, frozenset)):
+ raise TypeError(
+ "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, "
+ "not a string."
+ )
+
+ patterns: set[str] = set()
+ for pattern in value:
+ if not isinstance(pattern, str) or not (
+ normalized := pattern.strip().lower().rstrip(".")
+ ):
+ raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.")
+ patterns.add(normalized)
+ return tuple(sorted(patterns))
+
+
class DefaultConfig:
TESTING = False
DEBUG = False
@@ -62,6 +80,7 @@ class DefaultConfig:
SOURCE_DIRECTORY = None
SOURCE_BASE_URL = None
+ IGNORED_HOSTNAMES = ()
SCANNER_SCHEDULE = "0 * * * *"
# Total send attempts per source revision (not additional retries).
SEND_MAX_ATTEMPTS = 6
diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py
index e45085d..666eba7 100644
--- a/webmentions_ssg/tasks/scanner.py
+++ b/webmentions_ssg/tasks/scanner.py
@@ -25,6 +25,7 @@ from sqlalchemy.orm import selectinload
from .. import DATABASE as db
from .. import HUEY as huey
+from ..config import normalize_ignored_hostnames
from ..models import SentWebmention, Source
from ..url_security import is_http_url
from .sender import send_webmention
@@ -348,9 +349,16 @@ def scan_source_file(
Webmention source.
:raises OSError: If the source document cannot be read.
"""
- document = BeautifulSoup(path.read_bytes(), "html5lib")
+ # Use the path as published for its URL, but read only files whose
+ # resolved target stays inside the configured source directory. Resolving
+ # before reading also protects against symlinked index.html files.
+ resolved_root = root.resolve()
+ resolved_path = path.resolve()
+ if not resolved_path.is_relative_to(resolved_root):
+ raise SourceScanError(f"Source document escapes source directory: {path}")
relative_path = path.relative_to(root)
+ document = BeautifulSoup(resolved_path.read_bytes(), "html5lib")
source_url = canonical_url(document, relative_path=relative_path, base_url=base_url)
@@ -515,9 +523,8 @@ def scan_sources_task() -> None:
raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL")
base_url = f"{base_url.rstrip('/')}/"
- ignored_hostnames = tuple(
- pattern.lower().rstrip(".")
- for pattern in current_app.config.get("IGNORED_HOSTNAMES", ())
+ ignored_hostnames = normalize_ignored_hostnames(
+ current_app.config["IGNORED_HOSTNAMES"]
)
sources_by_path = {