diff options
Diffstat (limited to '')
| -rw-r--r-- | tests/tasks/test_scanner.py | 74 | ||||
| -rw-r--r-- | tests/test_config.py | 34 | ||||
| -rw-r--r-- | webmentions_ssg/__init__.py | 4 | ||||
| -rw-r--r-- | webmentions_ssg/config.py | 19 | ||||
| -rw-r--r-- | webmentions_ssg/tasks/scanner.py | 15 |
5 files changed, 142 insertions, 4 deletions
diff --git a/tests/tasks/test_scanner.py b/tests/tasks/test_scanner.py index 5a1e5f2..26e81c4 100644 --- a/tests/tasks/test_scanner.py +++ b/tests/tasks/test_scanner.py @@ -1979,6 +1979,80 @@ def test_unexpected_persistence_error_is_not_suppressed( run_scan(app, scanner_module) +def test_scan_source_rejects_symlink_outside_root_before_read( + scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + root = tmp_path / "site" + root.mkdir() + external = write_post(tmp_path / "outside", "private", "Sensitive content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + + read_bytes = Mock(side_effect=AssertionError("External file must not be read")) + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + with pytest.raises( + scanner_module.SourceScanError, match="escapes source directory" + ): + scanner_module.scan_source_file(linked, root=root, base_url=None) + + read_bytes.assert_not_called() + + +def test_scan_source_accepts_symlink_within_root( + scanner_module: ModuleType, tmp_path: Path +) -> None: + root = tmp_path / "site" + root.mkdir() + linked = write_post( + root, "example", '<a href="https://example.com/target">Target</a>' + ) + shared = root / "shared.html" + linked.rename(shared) + linked.symlink_to(shared) + + scanned = scanner_module.scan_source_file(linked, root=root, base_url=None) + + assert scanned.path == "example/index.html" + assert scanned.url == SOURCE_URL + assert scanned.targets == frozenset({"https://example.com/target"}) + + +def test_scan_sources_skips_symlink_outside_root( + app: Flask, + scanner_module: ModuleType, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + root = tmp_path / "site" + root.mkdir() + write_post(root, "example", "Normal content") + external = write_post(tmp_path / "outside", "private", "Outside content") + linked = root / "escaped" / "index.html" + linked.parent.mkdir() + linked.symlink_to(external) + configure_scanner(app, root) + + run_scan(app, scanner_module) + + with app.app_context(): + sources = list(db.session.scalars(sa.select(Source))) + assert len(sources) == 1 + assert sources[0].path == "example/index.html" + assert "escapes source directory" in caplog.text + + +def test_scan_sources_rejects_malformed_runtime_ignored_hostnames( + app: Flask, scanner_module: ModuleType, tmp_path: Path +) -> None: + configure_scanner(app, tmp_path) + app.config["IGNORED_HOSTNAMES"] = "example.com" + + with pytest.raises(TypeError, match="IGNORED_HOSTNAMES"): + run_scan(app, scanner_module) + + def test_scan_sources_accepts_valid_base_url( app: Flask, scanner_module: ModuleType, tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: diff --git a/tests/test_config.py b/tests/test_config.py index 933f437..aae7cce 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -75,6 +75,7 @@ def test_endpoint_limits_are_inherited_from_defaults() -> None: assert config.REQUEST_TOTAL_TIMEOUT == 30.0 assert DefaultConfig.ALLOWED_HOSTNAMES == set() + assert DefaultConfig.IGNORED_HOSTNAMES == () def test_allowed_hostnames_are_normalized_at_startup(tmp_path: Path) -> None: @@ -124,6 +125,39 @@ def test_invalid_allowed_hostnames_fail_at_startup( create_app(InvalidHostnameConfig) +def test_ignored_hostnames_are_normalized_at_startup(tmp_path: Path) -> None: + class MixedCaseConfig(TestingConfig): + LOG_FILE = str(tmp_path / "test.log") + IGNORED_HOSTNAMES = [" Example.COM. ", "*.EXAMPLE.com.", "example.com"] + + app = create_app(MixedCaseConfig) + assert app.config["IGNORED_HOSTNAMES"] == ("*.example.com", "example.com") + + +@pytest.mark.parametrize( + "hostnames,expected_exception", + [ + ("example.com", TypeError), + (None, TypeError), + ({"example.com": True}, TypeError), + (123, TypeError), + (["example.com", None], ValueError), + (["example.com", ""], ValueError), + (["example.com", " "], ValueError), + (["example.com", "..."], ValueError), + ], +) +def test_invalid_ignored_hostnames_fail_at_startup( + tmp_path: Path, hostnames: object, expected_exception: type[Exception] +) -> None: + class InvalidHostnameConfig(TestingConfig): + LOG_FILE = str(tmp_path / "test.log") + IGNORED_HOSTNAMES = hostnames + + with pytest.raises(expected_exception, match="IGNORED_HOSTNAMES"): + create_app(InvalidHostnameConfig) + + def test_production_requires_nonempty_allowed_hostnames(tmp_path: Path) -> None: config_path = tmp_path / "production.cfg" config_path.write_text( diff --git a/webmentions_ssg/__init__.py b/webmentions_ssg/__init__.py index e4a0a30..4c185c7 100644 --- a/webmentions_ssg/__init__.py +++ b/webmentions_ssg/__init__.py @@ -25,6 +25,7 @@ from webmentions_ssg.config import ( DevelopmentConfig, ProductionConfig, normalize_allowed_hostnames, + normalize_ignored_hostnames, ) from .security import validate_secret_key @@ -96,6 +97,9 @@ def create_app(config_class: type[DefaultConfig] | None = None) -> Flask: app.config["ALLOWED_HOSTNAMES"] = normalize_allowed_hostnames( app.config["ALLOWED_HOSTNAMES"] ) + app.config["IGNORED_HOSTNAMES"] = normalize_ignored_hostnames( + app.config["IGNORED_HOSTNAMES"] + ) for key in ( "MAX_CONTENT_LENGTH", "MAX_WEBMENTION_URL_BYTES", diff --git a/webmentions_ssg/config.py b/webmentions_ssg/config.py index e91c651..b1d588b 100644 --- a/webmentions_ssg/config.py +++ b/webmentions_ssg/config.py @@ -22,6 +22,24 @@ def normalize_allowed_hostnames(value: object) -> set[str]: return hostnames +def normalize_ignored_hostnames(value: object) -> tuple[str, ...]: + """Validate and normalize hostname patterns ignored by the source scanner.""" + if not isinstance(value, (list, tuple, set, frozenset)): + raise TypeError( + "IGNORED_HOSTNAMES must be a list, tuple, or set of hostname patterns, " + "not a string." + ) + + patterns: set[str] = set() + for pattern in value: + if not isinstance(pattern, str) or not ( + normalized := pattern.strip().lower().rstrip(".") + ): + raise ValueError("IGNORED_HOSTNAMES must contain non-empty strings only.") + patterns.add(normalized) + return tuple(sorted(patterns)) + + class DefaultConfig: TESTING = False DEBUG = False @@ -62,6 +80,7 @@ class DefaultConfig: SOURCE_DIRECTORY = None SOURCE_BASE_URL = None + IGNORED_HOSTNAMES = () SCANNER_SCHEDULE = "0 * * * *" # Total send attempts per source revision (not additional retries). SEND_MAX_ATTEMPTS = 6 diff --git a/webmentions_ssg/tasks/scanner.py b/webmentions_ssg/tasks/scanner.py index e45085d..666eba7 100644 --- a/webmentions_ssg/tasks/scanner.py +++ b/webmentions_ssg/tasks/scanner.py @@ -25,6 +25,7 @@ from sqlalchemy.orm import selectinload from .. import DATABASE as db from .. import HUEY as huey +from ..config import normalize_ignored_hostnames from ..models import SentWebmention, Source from ..url_security import is_http_url from .sender import send_webmention @@ -348,9 +349,16 @@ def scan_source_file( Webmention source. :raises OSError: If the source document cannot be read. """ - document = BeautifulSoup(path.read_bytes(), "html5lib") + # Use the path as published for its URL, but read only files whose + # resolved target stays inside the configured source directory. Resolving + # before reading also protects against symlinked index.html files. + resolved_root = root.resolve() + resolved_path = path.resolve() + if not resolved_path.is_relative_to(resolved_root): + raise SourceScanError(f"Source document escapes source directory: {path}") relative_path = path.relative_to(root) + document = BeautifulSoup(resolved_path.read_bytes(), "html5lib") source_url = canonical_url(document, relative_path=relative_path, base_url=base_url) @@ -515,9 +523,8 @@ def scan_sources_task() -> None: raise RuntimeError("SOURCE_BASE_URL must be an absolute HTTP or HTTPS URL") base_url = f"{base_url.rstrip('/')}/" - ignored_hostnames = tuple( - pattern.lower().rstrip(".") - for pattern in current_app.config.get("IGNORED_HOSTNAMES", ()) + ignored_hostnames = normalize_ignored_hostnames( + current_app.config["IGNORED_HOSTNAMES"] ) sources_by_path = { |
