[packages] Resolve git symlinks on Windows when materialized as text (#16657)

This commit is contained in:
Jesse Hills
2026-05-26 19:56:44 +12:00
committed by GitHub
parent ae74920b81
commit 423b60c90c
4 changed files with 507 additions and 8 deletions

View File

@@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
If loading fails after cloning, attempts a revert and retry in case
a prior cached checkout is stale.
"""
repo_dir, revert = git.clone_or_update(
repo_root, revert = git.clone_or_update(
url=config[CONF_URL],
ref=config.get(CONF_REF),
refresh=config[CONF_REFRESH],
@@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
)
files: list[dict[str, Any]] = []
# ``repo_root`` is the directory containing ``.git`` and must be passed
# to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a
# subdirectory via the user's CONF_PATH and is used for file lookups.
repo_dir = repo_root
if base_path := config.get(CONF_PATH):
repo_dir = repo_dir / base_path
@@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]:
def _load_package_yaml(yaml_file: Path, filename: str) -> dict:
"""Load a YAML file from a remote package, validating min_version."""
try:
new_yaml = yaml_util.load_yaml(yaml_file)
except EsphomeError as e:
def _load(path: Path) -> dict | str | None:
try:
return yaml_util.load_yaml(path)
except EsphomeError as e:
raise cv.Invalid(
f"{filename} is not a valid YAML file."
f" Please check the file contents.\n{e}"
) from e
new_yaml = _load(yaml_file)
if not isinstance(new_yaml, dict):
# On Windows, git defaults to core.symlinks=false unless the user
# has Developer Mode enabled or is running elevated. Files stored
# in the repo as symlinks (tree mode 120000) are then checked out
# as plain text files containing the symlink target path, so
# parsing them as YAML yields a bare scalar instead of a mapping.
# Best-effort: follow the symlink target ourselves and re-load.
target = git.resolve_symlink_stub(repo_root, yaml_file)
if target is not None:
new_yaml = _load(target)
if not isinstance(new_yaml, dict):
raise cv.Invalid(
f"{filename} is not a valid YAML file."
f" Please check the file contents.\n{e}"
) from e
f"{filename} does not contain a YAML mapping at the top level "
f"(got {type(new_yaml).__name__}). "
f"If this file is a git symlink in the source repository, it "
f"may not have been materialized correctly on your platform "
f"(this is a known issue with git on Windows without Developer "
f"Mode enabled). Try pointing your package at the real file "
f"path instead."
)
esphome_config = new_yaml.get(CONF_ESPHOME) or {}
min_version = esphome_config.get(CONF_MIN_VERSION)
if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse(

View File

@@ -6,6 +6,7 @@ import logging
from pathlib import Path
import re
import subprocess
import sys
import urllib.parse
import esphome.config_validation as cv
@@ -94,6 +95,92 @@ def _compute_destination_path(key: str, domain: str) -> Path:
return base_dir / h.hexdigest()[:8]
def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None:
"""Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub.
On Windows, when ``core.symlinks=false`` (the default unless the user has
SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated),
git materializes files with tree mode ``120000`` as plain text files
whose content is the literal symlink target path. Opening such a file
yields the target path string instead of the target's content.
If ``file_path`` is one of those stubs, return the resolved target Path
inside ``repo_dir``. Otherwise return ``None`` and the caller should use
``file_path`` as-is.
Designed to be called *only* when normal access has already produced an
unexpected result (e.g. YAML parsed as a top-level scalar), so the
per-file ``git ls-files`` subprocess cost is paid only on the failure
path. Returns ``None`` on any error or check failure — it's purely a
best-effort recovery, never raises.
"""
# On non-Windows, git creates real symlinks; ordinary file access already
# transparently follows them.
if sys.platform != "win32":
return None
if file_path.is_symlink():
return None
if not file_path.is_file():
return None
try:
rel = file_path.relative_to(repo_dir)
except ValueError:
return None
try:
# ``git ls-files -s <path>`` prints "<mode> <sha> <stage>\t<path>"
# for that single entry, or empty if untracked.
out = run_git_command(
["git", "ls-files", "-s", "--", rel.as_posix()],
git_dir=repo_dir,
)
except GitException:
return None
parts = out.split()
if not parts or parts[0] != "120000":
return None
# Stubs are short ASCII relative paths. Decode defensively, and only
# strip the trailing newline git's checkout may append — preserving any
# whitespace that could be part of a valid target name.
try:
raw = file_path.read_bytes()
except OSError:
return None
try:
target_str = raw.decode("utf-8").rstrip("\r\n")
except UnicodeDecodeError:
return None
# ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g.
# embedded NUL bytes from a hostile symlink blob, paths too long for the
# OS, or temporary I/O errors). Catch broadly — this helper is purely a
# best-effort recovery and must never raise.
try:
target_path = (file_path.parent / target_str).resolve()
repo_root_resolved = repo_dir.resolve()
except (OSError, ValueError, RuntimeError):
return None
# ``Path.resolve()`` follows ``..``; re-verify containment afterwards.
try:
target_path.relative_to(repo_root_resolved)
except ValueError:
_LOGGER.warning(
"Refusing to follow symlink %s -> %s (escapes repository)",
file_path,
target_str,
)
return None
if not target_path.is_file():
return None
return target_path
def clone_or_update(
*,
url: str,

View File

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
import os
from pathlib import Path
from typing import Any
from unittest.mock import Mock
from unittest.mock import Mock, patch
import pytest
@@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits(
"--hard",
"old_sha",
]
def test_resolve_symlink_stub_returns_none_on_non_windows(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""On non-Windows, resolve_symlink_stub returns None without calling git."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "file.yaml"
stub.write_text("static/file.yaml")
with patch("esphome.git.sys.platform", "linux"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
mock_run_git_command.assert_not_called()
def test_resolve_symlink_stub_returns_target_for_mode_120000(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A mode-120000 file is recognised as a stub; its target Path is returned."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
(repo_dir / "static").mkdir()
target = repo_dir / "static" / "real.yaml"
target.write_text("esphome:\n name: real\n")
stub = repo_dir / "real.yaml"
stub.write_text("static/real.yaml")
mock_run_git_command.return_value = "120000 abc123 0\treal.yaml"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result == target.resolve()
# Stub file itself was not modified — only inspected.
assert stub.read_text() == "static/real.yaml"
def test_resolve_symlink_stub_resolves_relative_parent_paths(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""Symlink targets with ``..`` segments resolve correctly within the repo."""
repo_dir = tmp_path / "repo"
(repo_dir / "subdir").mkdir(parents=True)
(repo_dir / "static").mkdir()
target = repo_dir / "static" / "shared.yaml"
target.write_text("shared content")
stub = repo_dir / "subdir" / "shared.yaml"
stub.write_text("../static/shared.yaml")
mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result == target.resolve()
def test_resolve_symlink_stub_refuses_escape_outside_repo(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A symlink pointing outside the repository is not followed."""
outside = tmp_path / "outside.yaml"
outside.write_text("sensitive")
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "escape.yaml"
stub.write_text("../outside.yaml")
mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_returns_none_for_real_symlink(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A real symlink already opens transparently, so the helper short-circuits.
Skipped on Windows where symlink creation requires
SeCreateSymbolicLinkPrivilege.
"""
if os.name == "nt":
pytest.skip("Requires symlink-creation privilege on Windows")
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
target = repo_dir / "real.yaml"
target.write_text("real content")
real_link = repo_dir / "link.yaml"
real_link.symlink_to("real.yaml")
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, real_link)
assert result is None
# No git call needed for real symlinks.
mock_run_git_command.assert_not_called()
def test_resolve_symlink_stub_returns_none_for_regular_file(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A regular file (mode 100644) whose content looks path-shaped is not
followed."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
regular = repo_dir / "looks_like_path.txt"
regular.write_text("static/something.yaml")
mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, regular)
assert result is None
def test_resolve_symlink_stub_returns_none_when_git_fails(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""If ``git ls-files`` fails (e.g. not a repo), the helper returns None."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "real.yaml"
stub.write_text("static/real.yaml")
mock_run_git_command.side_effect = GitCommandError("ls-files exploded")
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_returns_none_for_non_utf8_content(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A file whose bytes are not valid UTF-8 must not raise — return None."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "binary.bin"
stub.write_bytes(b"\xff\xfe\x00\xff")
mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_preserves_whitespace_in_target(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""Only trailing CR/LF is stripped — internal whitespace is preserved."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
target_dir = repo_dir / "dir with spaces"
target_dir.mkdir()
target = target_dir / "real.yaml"
target.write_text("hello")
stub = repo_dir / "link.yaml"
# Trailing newline (as git's checkout may append) is stripped, but
# whitespace inside the target path itself must survive.
stub.write_bytes(b"dir with spaces/real.yaml\n")
mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result == target.resolve()
def test_resolve_symlink_stub_returns_none_for_directory_target(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A symlink pointing at a directory has no file content to load."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
(repo_dir / "dir_target").mkdir()
stub = repo_dir / "link_to_dir"
stub.write_text("dir_target")
mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir"
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_returns_none_when_resolve_raises(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""Path.resolve() raising (e.g. on a malformed target) must not propagate."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "broken.yaml"
stub.write_text("ignored")
mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml"
with (
patch("esphome.git.sys.platform", "win32"),
patch.object(Path, "resolve", side_effect=OSError("bad path")),
):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_returns_none_when_file_missing(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A file path that doesn't exist is rejected before git is consulted."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
missing = repo_dir / "ghost.yaml" # not created
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, missing)
assert result is None
mock_run_git_command.assert_not_called()
def test_resolve_symlink_stub_returns_none_when_path_outside_repo(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""A file path that isn't under repo_dir is rejected (ValueError from relative_to)."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
outside = tmp_path / "stray.yaml"
outside.write_text("something")
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, outside)
assert result is None
mock_run_git_command.assert_not_called()
def test_resolve_symlink_stub_returns_none_when_untracked(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""Empty `git ls-files` output (untracked file) makes the helper return None."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "untracked.yaml"
stub.write_text("static/foo.yaml")
mock_run_git_command.return_value = ""
with patch("esphome.git.sys.platform", "win32"):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None
def test_resolve_symlink_stub_returns_none_when_read_bytes_raises(
tmp_path: Path, mock_run_git_command: Mock
) -> None:
"""An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate."""
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
stub = repo_dir / "racy.yaml"
stub.write_text("static/racy.yaml")
mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml"
with (
patch("esphome.git.sys.platform", "win32"),
patch.object(Path, "read_bytes", side_effect=OSError("vanished")),
):
result = git.resolve_symlink_stub(repo_dir, stub)
assert result is None

View File

@@ -837,3 +837,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None:
assert isinstance(result["value"], Lambda)
assert result["value"].value == 'return "bar";'
@patch("esphome.git.resolve_symlink_stub")
@patch("esphome.git.clone_or_update")
def test_remote_package_symlink_stub_is_followed(
mock_clone_or_update: MagicMock,
mock_resolve_symlink_stub: MagicMock,
tmp_path: Path,
) -> None:
"""When a package YAML is a scalar (symlink stub) and resolve_symlink_stub
returns a target, the loader follows the target and uses its content."""
CORE.config_path = tmp_path / "test.yaml"
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
(repo_dir / "static").mkdir()
# Stub file: content is the target path string (simulating Windows behavior).
stub = repo_dir / "file1.yaml"
stub.write_text("static/file1.yaml")
# Real target with valid YAML mapping.
target = repo_dir / "static" / "file1.yaml"
target.write_text("substitutions:\n hello: world\n")
mock_clone_or_update.return_value = (repo_dir, None)
mock_resolve_symlink_stub.return_value = target
config: dict[str, Any] = {
"packages": {
"test_package": {
"url": "https://github.com/esphome/repo1",
"ref": "main",
"files": ["file1.yaml"],
}
}
}
# Must succeed (does not raise the helpful cv.Invalid) because the stub
# was followed and a valid mapping was loaded from the target.
do_packages_pass(config)
assert mock_resolve_symlink_stub.called
@patch("esphome.git.clone_or_update")
def test_remote_package_scalar_yaml_raises_helpful_error(
mock_clone_or_update: MagicMock, tmp_path: Path
) -> None:
"""A remote package YAML that is a top-level scalar (e.g. an unmaterialized
git symlink on Windows) raises a clear cv.Invalid, not AttributeError.
Regression test for the case where a repo containing a YAML symlink,
checked out on Windows without symlink privilege, lands as a short text
file containing the symlink target path. PyYAML parses that as a bare
string scalar; the package loader must reject it with a human-readable
error instead of dying inside ``.get()``.
"""
CORE.config_path = tmp_path / "test.yaml"
repo_dir = tmp_path / "repo"
repo_dir.mkdir()
# Simulate the broken-symlink state: a YAML file whose entire content is
# the symlink target string. PyYAML parses this as a top-level scalar.
(repo_dir / "file1.yaml").write_text("static/file1.yaml")
mock_clone_or_update.return_value = (repo_dir, None)
config: dict[str, Any] = {
"packages": {
"test_package": {
"url": "https://github.com/esphome/repo1",
"ref": "main",
"files": ["file1.yaml"],
}
}
}
with pytest.raises(cv.Invalid) as exc_info:
do_packages_pass(config)
msg = str(exc_info.value)
assert "mapping at the top level" in msg
assert "file1.yaml" in msg