[platformio] Give the ccache wrapper a cmd.exe safe path (#18495)

esphome/platformio/ccache.py.script

@@ -1,5 +1,4 @@
 import os
-import shutil
 
 # pylint: disable=E0602
 Import("env")  # noqa
@@ -9,15 +8,17 @@ Import("env")  # noqa
 # esphome/platformio/toolchain.py); this script only supplies the SCons-level
 # mechanism.
 #
+# The binary comes pre-resolved in ESPHOME_CCACHE_PATH; _ccache_env() has
+# already stripped the Windows \\?\ prefix that cmd.exe cannot run.
+#
 # This is a "pre" script, so the platform's builder (which sets CC/CXX and
 # clones the construction environment for framework and library builds) runs
 # after it. Replacing CC/CXX here would be overwritten, and replacing them in
 # a "post" script would miss the already-cloned library environments. Wrapping
 # SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler
 # invocation from every environment funnels through it at execution time.
-if (
-    os.environ.get("ESPHOME_CCACHE_ENABLE") == "1"
-    and (ccache_path := shutil.which("ccache")) is not None
+if os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" and (
+    ccache_path := os.environ.get("ESPHOME_CCACHE_PATH")
 ):
     original_spawn = env["SPAWN"]
 

esphome/platformio/toolchain.py

@@ -60,6 +60,9 @@ def _strip_win_long_path_prefix(path: str) -> str:
     "The system cannot find the path specified." Stripping the prefix early
     keeps the path shell-quotable.
 
+    Also applied to the ccache path exported by ``_ccache_env()``, which
+    ``shutil.which`` can return with the same prefix.
+
     No-op on non-Windows platforms.
     """
     if sys.platform != "win32":
@@ -235,8 +238,8 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None:
         _write_pio_stamp_python(stamp_file, current)
 
 
-def _ccache_usable() -> bool:
-    """Return True when the ``ccache`` on PATH actually runs.
+def _ccache_runs(ccache: str) -> bool:
+    """Return True when the ``ccache`` found on PATH actually runs.
 
     ``shutil.which`` proves existence, not runnability: on Windows it also
     matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
@@ -244,9 +247,6 @@ def _ccache_usable() -> bool:
     step with an opaque OS error, so probe once and fall back to compiling
     without ccache when the probe fails.
     """
-    ccache = shutil.which("ccache")
-    if ccache is None:
-        return False
     try:
         subprocess.run(
             [ccache, "--version"],
@@ -265,14 +265,29 @@ def _ccache_usable() -> bool:
 
 
 def _ccache_env() -> dict[str, str]:
-    """Return ccache settings for PlatformIO builds.
+    r"""Return ccache settings for PlatformIO builds.
 
     Enabled by default whenever the ``ccache`` binary is on PATH; set
     ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to
-    force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE``
-    so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script,
-    which wraps compiler invocations inside SCons) only have to check for
-    ``"1"`` instead of re-implementing the policy.
+    force it on without the runnability probe; a binary is still needed).
+    The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` and the
+    binary's location into ``ESPHOME_CCACHE_PATH`` so platform build scripts
+    (the shared ``ccache.py`` extra script, which wraps compiler invocations
+    inside SCons) only have to check for ``"1"`` and use the path as given
+    instead of re-implementing the policy.
+
+    The path is exported rather than looked up again inside SCons because
+    ``shutil.which`` can return a Windows extended-length ``\\?\`` path
+    (ESPHome Desktop puts its bundled ccache on PATH that way). Such a path
+    runs fine through ``CreateProcess``, which is how ESP-IDF invokes it,
+    but SCons runs every compile through ``cmd.exe``, which fails on it with
+    "The system cannot find the path specified." (#18399), so the prefix is
+    stripped here with ``_strip_win_long_path_prefix()`` before the
+    runnability probe, which therefore validates the exact string the build
+    will execute.
+    ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the
+    script only honours it together with ``ESPHOME_CCACHE_ENABLE=1``, and this
+    function always sets both or neither.
 
     The returned values are merged into the environment of the PlatformIO
     subprocess only, never into ``os.environ``: a long-running process
@@ -293,13 +308,27 @@ def _ccache_env() -> dict[str, str]:
     build dir. The other ``CCACHE_*`` values the user already set in the
     environment are respected.
     """
-    if "ESPHOME_CCACHE_ENABLE" in os.environ:
-        enabled = get_bool_env("ESPHOME_CCACHE_ENABLE")
-    else:
-        enabled = _ccache_usable()
-    env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"}
-    if not enabled:
-        return env
+    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
+    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
+        return {"ESPHOME_CCACHE_ENABLE": "0"}
+    ccache_path = shutil.which("ccache")
+    if ccache_path is None:
+        if explicit:
+            _LOGGER.warning(
+                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
+                "compiling without ccache"
+            )
+        return {"ESPHOME_CCACHE_ENABLE": "0"}
+    # Strip before probing so the probe validates (and the failure warning
+    # names) the exact string the build will execute through cmd.exe.
+    ccache_path = _strip_win_long_path_prefix(ccache_path)
+    # An explicit opt-in skips the runnability probe.
+    if not explicit and not _ccache_runs(ccache_path):
+        return {"ESPHOME_CCACHE_ENABLE": "0"}
+    env = {
+        "ESPHOME_CCACHE_ENABLE": "1",
+        "ESPHOME_CCACHE_PATH": ccache_path,
+    }
     # build_path is set during preload for every config-loading command, so it
     # being unset means a caller built the environment too early; fail loudly
     # rather than with an opaque TypeError from Path(None).

tests/unit_tests/test_platformio_toolchain.py

@@ -2,7 +2,7 @@
 
 # pylint: disable=protected-access
 
-from collections.abc import Generator
+from collections.abc import Callable, Generator
 from contextlib import contextmanager
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 import json
@@ -437,6 +437,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None:
         env = toolchain._ccache_env()
 
     assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+    assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache"
     assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve())
     assert env["CCACHE_DIR"].endswith("platformio-ccache")
     assert env["CCACHE_NOHASHDIR"] == "true"
@@ -446,17 +447,35 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None:
     assert "ESPHOME_CCACHE_ENABLE" not in os.environ
 
 
-def test_ccache_env_disabled_without_binary(setup_core: Path) -> None:
-    """Ccache stays off when the binary is not on PATH."""
+@pytest.mark.parametrize(
+    ("env_vars", "expect_warning"),
+    [
+        pytest.param({}, False, id="default"),
+        pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, True, id="forced-on"),
+    ],
+)
+def test_ccache_env_disabled_without_binary(
+    setup_core: Path,
+    caplog: pytest.LogCaptureFixture,
+    env_vars: dict[str, str],
+    expect_warning: bool,
+) -> None:
+    """Ccache stays off when the binary is not on PATH, even when forced on.
+
+    A deliberate opt-in that finds no binary is downgraded with a warning so
+    the user can tell why it had no effect; the default path stays quiet.
+    """
     CORE.build_path = setup_core / "build" / "test"
 
     with (
-        patch.dict(os.environ, {}, clear=True),
+        patch.dict(os.environ, env_vars, clear=True),
         patch.object(toolchain.shutil, "which", return_value=None),
+        caplog.at_level("WARNING"),
     ):
         env = toolchain._ccache_env()
 
     assert env == {"ESPHOME_CCACHE_ENABLE": "0"}
+    assert ("no ccache binary is on PATH" in caplog.text) is expect_warning
 
 
 @pytest.mark.parametrize(
@@ -489,14 +508,47 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None:
 
     with (
         patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
         patch.object(toolchain.subprocess, "run") as mock_probe,
     ):
         env = toolchain._ccache_env()
 
     assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+    # The binary's location is still handed to the build script.
+    assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache"
     mock_probe.assert_not_called()
 
 
+def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None:
+    r"""A ``\\?\`` ccache path from PATH is exported without the prefix.
+
+    That is the shape ESPHome Desktop puts on PATH (#18399); see ``_ccache_env``.
+    """
+    CORE.build_path = setup_core / "build" / "test"
+    prefixed = (
+        "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder"
+        "\\ccache\\ccache.exe"
+    )
+    stripped = (
+        "C:\\Users\\jesse\\AppData\\Local\\ESPHome Device Builder\\ccache\\ccache.exe"
+    )
+
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        # shutil.which is patched, so the win32 code path of the real
+        # implementation (which crashes on a POSIX host) is never reached.
+        patch("esphome.platformio.toolchain.sys.platform", "win32"),
+        patch.object(toolchain.shutil, "which", return_value=prefixed),
+        patch.object(toolchain.subprocess, "run") as mock_probe,
+    ):
+        env = toolchain._ccache_env()
+
+    assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+    assert env["ESPHOME_CCACHE_PATH"] == stripped
+    # The probe validates the exact string the build will execute.
+    assert mock_probe.call_args[0][0] == [stripped, "--version"]
+
+
 def test_ccache_env_opt_out(setup_core: Path) -> None:
     """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present."""
     CORE.build_path = setup_core / "build" / "test"
@@ -516,7 +568,7 @@ def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None:
 
     with (
         patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True),
-        patch.object(toolchain.shutil, "which", return_value=None),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
     ):
         env = toolchain._ccache_env()
 
@@ -563,8 +615,10 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only(
 
         env = mock_run_external_process.call_args[1]["env"]
         assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+        assert env["ESPHOME_CCACHE_PATH"] == "/usr/bin/ccache"
         assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve())
         assert "ESPHOME_CCACHE_ENABLE" not in os.environ
+        assert "ESPHOME_CCACHE_PATH" not in os.environ
         assert "CCACHE_BASEDIR" not in os.environ
 
 
@@ -613,6 +667,182 @@ def test_copy_ccache_script(setup_core: Path) -> None:
     assert dest.read_text() == source.read_text()
 
 
+class _FakeSConsEnv(dict):
+    """Just enough of a SCons construction environment for ccache.py."""
+
+    def Replace(self, **kwargs: object) -> None:  # noqa: N802
+        self.update(kwargs)
+
+
+def _load_ccache_script(
+    env_vars: dict[str, str], original_spawn: Callable[..., int] | None = None
+) -> tuple[_FakeSConsEnv, Callable[..., int]]:
+    """Run ccache.py.script against a fake SCons env and return (env, original SPAWN)."""
+    if original_spawn is None:
+        original_spawn = Mock(name="original_spawn", return_value=0)
+    scons_env = _FakeSConsEnv(SPAWN=original_spawn)
+    source = (Path(toolchain.__file__).parent / "ccache.py.script").read_text()
+    with patch.dict(os.environ, env_vars, clear=True):
+        exec(  # noqa: S102
+            compile(source, "ccache.py", "exec"),
+            {"Import": lambda *_names: None, "env": scons_env},
+        )
+    return scons_env, original_spawn
+
+
+def _scons_win32_escape(x: str) -> str:
+    """Copy of ``SCons.Platform.win32.escape``: quote, guarding a trailing backslash."""
+    if x[-1] == "\\":
+        x = x + "\\"
+    return '"' + x + '"'
+
+
+def test_ccache_script_wraps_compiles_with_exported_path() -> None:
+    """The SCons script uses ESPHOME_CCACHE_PATH as given, without a PATH lookup."""
+    ccache_path = "C:\\Users\\jesse\\ESPHome Device Builder\\ccache\\ccache.exe"
+    scons_env, original_spawn = _load_ccache_script(
+        {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": ccache_path}
+    )
+    spawn = scons_env["SPAWN"]
+    assert spawn is not original_spawn
+
+    # A compile step is routed through ccache, with the same path used for
+    # the program and (escaped) as the first argument.
+    compile_args = ["xtensa-lx106-elf-g++", "-o", "main.o", "-c", "main.cpp"]
+    spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", compile_args, {})
+    original_spawn.assert_called_once_with(
+        "cmd.exe",
+        _scons_win32_escape,
+        ccache_path,
+        [_scons_win32_escape(ccache_path), *compile_args],
+        {},
+    )
+
+    # Link steps pass through untouched.
+    original_spawn.reset_mock()
+    link_args = ["xtensa-lx106-elf-g++", "-o", "firmware.elf", "main.o"]
+    spawn("cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {})
+    original_spawn.assert_called_once_with(
+        "cmd.exe", _scons_win32_escape, "xtensa-lx106-elf-g++", link_args, {}
+    )
+
+
+@pytest.mark.parametrize(
+    "env_vars",
+    [
+        pytest.param({"ESPHOME_CCACHE_ENABLE": "0"}, id="disabled"),
+        pytest.param({"ESPHOME_CCACHE_ENABLE": "1"}, id="enabled-without-path"),
+        pytest.param({}, id="unset"),
+    ],
+)
+def test_ccache_script_leaves_spawn_alone_without_path(
+    env_vars: dict[str, str],
+) -> None:
+    """Without both the enable flag and a path, SPAWN is not replaced."""
+    scons_env, original_spawn = _load_ccache_script(env_vars)
+    assert scons_env["SPAWN"] is original_spawn
+
+
+def _scons_win32_spawn(
+    sh: str, escape: Callable[[str], str], cmd: str, args: list[str], env: dict
+) -> int:
+    r"""Mirror of ``SCons.Platform.win32.spawn``: every command runs via ``cmd.exe /C``.
+
+    SCons is not importable in the test environment (PlatformIO fetches it at
+    build time), so the lines that matter are mirrored here. The command line
+    SCons hands ``os.spawnve`` goes to ``CreateProcess`` via ``subprocess``
+    instead (identical on Windows, where a string passes through untouched);
+    ``spawnve`` itself crashes inside pytest.
+    """
+    return subprocess.run(
+        " ".join([sh, "/C", escape(" ".join(args))]), env=env, check=False
+    ).returncode
+
+
+_MARKER_ENV = "ESPHOME_TEST_CCACHE_MARKER"
+# Stands in for a compile: the "ccache" is really the Python interpreter, and
+# the compile "flags" make it write a marker file so the test can tell whether
+# the wrapped command actually ran to completion.
+_FAKE_COMPILE_ARGS = [
+    "-c",
+    f"import os, pathlib; pathlib.Path(os.environ['{_MARKER_ENV}']).write_text('compiled')",
+]
+
+
+def _spawn_fake_compile_via_cmd_exe(scons_env: _FakeSConsEnv, marker: Path) -> int:
+    """Run one wrapped compile step the way SCons does on Windows."""
+    child_env = {**os.environ, _MARKER_ENV: str(marker)}
+    return scons_env["SPAWN"](
+        os.environ.get("COMSPEC", "cmd.exe"),
+        _scons_win32_escape,
+        "xtensa-lx106-elf-gcc",
+        [_scons_win32_escape(arg) if " " in arg else arg for arg in _FAKE_COMPILE_ARGS],
+        child_env,
+    )
+
+
+_WINDOWS_ONLY = pytest.mark.skipif(
+    sys.platform != "win32", reason="drives cmd.exe, which SCons uses only on Windows"
+)
+
+
+@_WINDOWS_ONLY
+def test_ccache_env_real_probe_runs_stripped_path(setup_core: Path) -> None:
+    r"""With a ``\\?\`` which result, the real probe runs the stripped binary.
+
+    The probe therefore validates the exact string the build will execute
+    through ``cmd.exe``; probing the verbatim path instead would pass even
+    when the stripped path is unusable (``CreateProcess`` accepts
+    extended-length paths, ``cmd.exe`` does not).
+    """
+    CORE.build_path = setup_core / "build" / "test"
+    assert not sys.executable.startswith("\\\\?\\")
+
+    with (
+        patch.dict(os.environ, {}, clear=False),
+        patch.object(
+            toolchain.shutil, "which", return_value="\\\\?\\" + sys.executable
+        ),
+    ):
+        os.environ.pop("ESPHOME_CCACHE_ENABLE", None)
+        env = toolchain._ccache_env()
+
+    assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+    assert env["ESPHOME_CCACHE_PATH"] == sys.executable
+
+
+@_WINDOWS_ONLY
+@pytest.mark.parametrize(
+    ("prefix", "expect_ok"),
+    [
+        pytest.param("", True, id="stripped-path-compiles"),
+        pytest.param("\\\\?\\", False, id="verbatim-path-fails"),
+    ],
+)
+def test_ccache_wrapper_through_cmd_exe(
+    tmp_path: Path, prefix: str, expect_ok: bool
+) -> None:
+    r"""End to end through ``cmd.exe``: the exported path works, a ``\\?\`` one does not.
+
+    The interpreter stands in for ccache; the spawn mirrors SCons on Windows.
+    The failing case is the mechanism behind #18399 ("The system cannot find
+    the path specified." on every compile step); should it ever start passing,
+    ``cmd.exe`` learned extended-length paths and the strip is no longer needed.
+    """
+    marker = tmp_path / "compiled.txt"
+    scons_env, _ = _load_ccache_script(
+        {"ESPHOME_CCACHE_ENABLE": "1", "ESPHOME_CCACHE_PATH": prefix + sys.executable},
+        original_spawn=_scons_win32_spawn,
+    )
+    assert scons_env["SPAWN"] is not _scons_win32_spawn
+
+    rc = _spawn_fake_compile_via_cmd_exe(scons_env, marker)
+    assert (rc == 0) is expect_ok
+    assert marker.exists() is expect_ok
+    if expect_ok:
+        assert marker.read_text() == "compiled"
+
+
 @pytest.mark.parametrize(
     ("platform", "input_path", "expected"),
     [