From 423b60c90ce030f02f0bfa560464d4016301da59 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 26 May 2026 19:56:44 +1200 Subject: [PATCH] [packages] Resolve git symlinks on Windows when materialized as text (#16657) --- esphome/components/packages/__init__.py | 42 +++- esphome/git.py | 87 +++++++ tests/unit_tests/test_git.py | 303 +++++++++++++++++++++++- tests/unit_tests/test_substitutions.py | 83 +++++++ 4 files changed, 507 insertions(+), 8 deletions(-) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 06a64208b6..f3e0e0db8f 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: If loading fails after cloning, attempts a revert and retry in case a prior cached checkout is stale. """ - repo_dir, revert = git.clone_or_update( + repo_root, revert = git.clone_or_update( url=config[CONF_URL], ref=config.get(CONF_REF), refresh=config[CONF_REFRESH], @@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: ) files: list[dict[str, Any]] = [] + # ``repo_root`` is the directory containing ``.git`` and must be passed + # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a + # subdirectory via the user's CONF_PATH and is used for file lookups. + repo_dir = repo_root if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path @@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: def _load_package_yaml(yaml_file: Path, filename: str) -> dict: """Load a YAML file from a remote package, validating min_version.""" - try: - new_yaml = yaml_util.load_yaml(yaml_file) - except EsphomeError as e: + + def _load(path: Path) -> dict | str | None: + try: + return yaml_util.load_yaml(path) + except EsphomeError as e: + raise cv.Invalid( + f"{filename} is not a valid YAML file." + f" Please check the file contents.\n{e}" + ) from e + + new_yaml = _load(yaml_file) + if not isinstance(new_yaml, dict): + # On Windows, git defaults to core.symlinks=false unless the user + # has Developer Mode enabled or is running elevated. Files stored + # in the repo as symlinks (tree mode 120000) are then checked out + # as plain text files containing the symlink target path, so + # parsing them as YAML yields a bare scalar instead of a mapping. + # Best-effort: follow the symlink target ourselves and re-load. + target = git.resolve_symlink_stub(repo_root, yaml_file) + if target is not None: + new_yaml = _load(target) + if not isinstance(new_yaml, dict): raise cv.Invalid( - f"{filename} is not a valid YAML file." - f" Please check the file contents.\n{e}" - ) from e + f"{filename} does not contain a YAML mapping at the top level " + f"(got {type(new_yaml).__name__}). " + f"If this file is a git symlink in the source repository, it " + f"may not have been materialized correctly on your platform " + f"(this is a known issue with git on Windows without Developer " + f"Mode enabled). Try pointing your package at the real file " + f"path instead." + ) esphome_config = new_yaml.get(CONF_ESPHOME) or {} min_version = esphome_config.get(CONF_MIN_VERSION) if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse( diff --git a/esphome/git.py b/esphome/git.py index f36bd559ef..094a6dae19 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -6,6 +6,7 @@ import logging from pathlib import Path import re import subprocess +import sys import urllib.parse import esphome.config_validation as cv @@ -94,6 +95,92 @@ def _compute_destination_path(key: str, domain: str) -> Path: return base_dir / h.hexdigest()[:8] +def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None: + """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub. + + On Windows, when ``core.symlinks=false`` (the default unless the user has + SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated), + git materializes files with tree mode ``120000`` as plain text files + whose content is the literal symlink target path. Opening such a file + yields the target path string instead of the target's content. + + If ``file_path`` is one of those stubs, return the resolved target Path + inside ``repo_dir``. Otherwise return ``None`` and the caller should use + ``file_path`` as-is. + + Designed to be called *only* when normal access has already produced an + unexpected result (e.g. YAML parsed as a top-level scalar), so the + per-file ``git ls-files`` subprocess cost is paid only on the failure + path. Returns ``None`` on any error or check failure — it's purely a + best-effort recovery, never raises. + """ + # On non-Windows, git creates real symlinks; ordinary file access already + # transparently follows them. + if sys.platform != "win32": + return None + if file_path.is_symlink(): + return None + if not file_path.is_file(): + return None + + try: + rel = file_path.relative_to(repo_dir) + except ValueError: + return None + + try: + # ``git ls-files -s `` prints " \t" + # for that single entry, or empty if untracked. + out = run_git_command( + ["git", "ls-files", "-s", "--", rel.as_posix()], + git_dir=repo_dir, + ) + except GitException: + return None + + parts = out.split() + if not parts or parts[0] != "120000": + return None + + # Stubs are short ASCII relative paths. Decode defensively, and only + # strip the trailing newline git's checkout may append — preserving any + # whitespace that could be part of a valid target name. + try: + raw = file_path.read_bytes() + except OSError: + return None + try: + target_str = raw.decode("utf-8").rstrip("\r\n") + except UnicodeDecodeError: + return None + + # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g. + # embedded NUL bytes from a hostile symlink blob, paths too long for the + # OS, or temporary I/O errors). Catch broadly — this helper is purely a + # best-effort recovery and must never raise. + try: + target_path = (file_path.parent / target_str).resolve() + repo_root_resolved = repo_dir.resolve() + except (OSError, ValueError, RuntimeError): + return None + + # ``Path.resolve()`` follows ``..``; re-verify containment afterwards. + try: + target_path.relative_to(repo_root_resolved) + except ValueError: + _LOGGER.warning( + "Refusing to follow symlink %s -> %s (escapes repository)", + file_path, + target_str, + ) + return None + + if not target_path.is_file(): + return None + + return target_path + + def clone_or_update( *, url: str, diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index eab6bfc2cb..690c47c183 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import os from pathlib import Path from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits( "--hard", "old_sha", ] + + +def test_resolve_symlink_stub_returns_none_on_non_windows( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """On non-Windows, resolve_symlink_stub returns None without calling git.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + stub = repo_dir / "file.yaml" + stub.write_text("static/file.yaml") + + with patch("esphome.git.sys.platform", "linux"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_target_for_mode_120000( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A mode-120000 file is recognised as a stub; its target Path is returned.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "real.yaml" + target.write_text("esphome:\n name: real\n") + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\treal.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + # Stub file itself was not modified — only inspected. + assert stub.read_text() == "static/real.yaml" + + +def test_resolve_symlink_stub_resolves_relative_parent_paths( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Symlink targets with ``..`` segments resolve correctly within the repo.""" + repo_dir = tmp_path / "repo" + (repo_dir / "subdir").mkdir(parents=True) + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "shared.yaml" + target.write_text("shared content") + + stub = repo_dir / "subdir" / "shared.yaml" + stub.write_text("../static/shared.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_refuses_escape_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing outside the repository is not followed.""" + outside = tmp_path / "outside.yaml" + outside.write_text("sensitive") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "escape.yaml" + stub.write_text("../outside.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_real_symlink( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A real symlink already opens transparently, so the helper short-circuits. + + Skipped on Windows where symlink creation requires + SeCreateSymbolicLinkPrivilege. + """ + if os.name == "nt": + pytest.skip("Requires symlink-creation privilege on Windows") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target = repo_dir / "real.yaml" + target.write_text("real content") + + real_link = repo_dir / "link.yaml" + real_link.symlink_to("real.yaml") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, real_link) + + assert result is None + # No git call needed for real symlinks. + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_for_regular_file( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A regular file (mode 100644) whose content looks path-shaped is not + followed.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + regular = repo_dir / "looks_like_path.txt" + regular.write_text("static/something.yaml") + + mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, regular) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_git_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """If ``git ls-files`` fails (e.g. not a repo), the helper returns None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.side_effect = GitCommandError("ls-files exploded") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_non_utf8_content( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file whose bytes are not valid UTF-8 must not raise — return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "binary.bin" + stub.write_bytes(b"\xff\xfe\x00\xff") + + mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_preserves_whitespace_in_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Only trailing CR/LF is stripped — internal whitespace is preserved.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target_dir = repo_dir / "dir with spaces" + target_dir.mkdir() + target = target_dir / "real.yaml" + target.write_text("hello") + + stub = repo_dir / "link.yaml" + # Trailing newline (as git's checkout may append) is stripped, but + # whitespace inside the target path itself must survive. + stub.write_bytes(b"dir with spaces/real.yaml\n") + + mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_returns_none_for_directory_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing at a directory has no file content to load.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "dir_target").mkdir() + + stub = repo_dir / "link_to_dir" + stub.write_text("dir_target") + + mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_resolve_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Path.resolve() raising (e.g. on a malformed target) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "broken.yaml" + stub.write_text("ignored") + + mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "resolve", side_effect=OSError("bad path")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_file_missing( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that doesn't exist is rejected before git is consulted.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + missing = repo_dir / "ghost.yaml" # not created + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, missing) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_path_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that isn't under repo_dir is rejected (ValueError from relative_to).""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + outside = tmp_path / "stray.yaml" + outside.write_text("something") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, outside) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_untracked( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Empty `git ls-files` output (untracked file) makes the helper return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "untracked.yaml" + stub.write_text("static/foo.yaml") + + mock_run_git_command.return_value = "" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_read_bytes_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "racy.yaml" + stub.write_text("static/racy.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "read_bytes", side_effect=OSError("vanished")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index c71be2fbab..b5816f742e 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -837,3 +837,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: assert isinstance(result["value"], Lambda) assert result["value"].value == 'return "bar";' + + +@patch("esphome.git.resolve_symlink_stub") +@patch("esphome.git.clone_or_update") +def test_remote_package_symlink_stub_is_followed( + mock_clone_or_update: MagicMock, + mock_resolve_symlink_stub: MagicMock, + tmp_path: Path, +) -> None: + """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub + returns a target, the loader follows the target and uses its content.""" + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + # Stub file: content is the target path string (simulating Windows behavior). + stub = repo_dir / "file1.yaml" + stub.write_text("static/file1.yaml") + + # Real target with valid YAML mapping. + target = repo_dir / "static" / "file1.yaml" + target.write_text("substitutions:\n hello: world\n") + + mock_clone_or_update.return_value = (repo_dir, None) + mock_resolve_symlink_stub.return_value = target + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + # Must succeed (does not raise the helpful cv.Invalid) because the stub + # was followed and a valid mapping was loaded from the target. + do_packages_pass(config) + assert mock_resolve_symlink_stub.called + + +@patch("esphome.git.clone_or_update") +def test_remote_package_scalar_yaml_raises_helpful_error( + mock_clone_or_update: MagicMock, tmp_path: Path +) -> None: + """A remote package YAML that is a top-level scalar (e.g. an unmaterialized + git symlink on Windows) raises a clear cv.Invalid, not AttributeError. + + Regression test for the case where a repo containing a YAML symlink, + checked out on Windows without symlink privilege, lands as a short text + file containing the symlink target path. PyYAML parses that as a bare + string scalar; the package loader must reject it with a human-readable + error instead of dying inside ``.get()``. + """ + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + # Simulate the broken-symlink state: a YAML file whose entire content is + # the symlink target string. PyYAML parses this as a top-level scalar. + (repo_dir / "file1.yaml").write_text("static/file1.yaml") + + mock_clone_or_update.return_value = (repo_dir, None) + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + with pytest.raises(cv.Invalid) as exc_info: + do_packages_pass(config) + + msg = str(exc_info.value) + assert "mapping at the top level" in msg + assert "file1.yaml" in msg