Share the ccache and tools-path helpers across the native backends

esphome/arduino8266/framework.py

@@ -25,19 +25,17 @@ from pathlib import Path
 import platform
 import shutil
 
-import platformdirs
-
-import esphome.config_validation as cv
-from esphome.core import CORE, EsphomeError
+from esphome.core import EsphomeError, Version
 from esphome.framework_helpers import (
     archive_extract_all,
+    ccache_defaults_env,
     download_from_mirrors,
     download_with_resume,
+    resolve_ccache_path,
     rmdir,
     str_to_lst_of_str,
+    tools_cache_path,
 )
-from esphome.helpers import get_bool_env, get_str_env
-from esphome.platformio.library import ensure_list
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -61,25 +59,16 @@ ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str(
 
 
 def get_arduino8266_tools_path() -> Path:
-    # Treat an empty/whitespace prefix as unset: Path("") resolves to the CWD,
-    # which clean-all would then delete.
-    if prefix := get_str_env("ESPHOME_ARDUINO8266_PREFIX", "").strip():
-        path = Path(prefix).expanduser()
-    else:
-        # Machine-global so all projects share one install; see
-        # espidf.framework.get_idf_tools_path for the location rationale.
-        path = (
-            Path(platformdirs.user_cache_dir("esphome", appauthor=False))
-            / "arduino8266"
-        )
-    return path.resolve()
+    # Machine-global so all projects share one install; see
+    # espidf.framework.get_idf_tools_path for the location rationale.
+    return tools_cache_path("ESPHOME_ARDUINO8266_PREFIX", "arduino8266")
 
 
 # 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0
-MIN_FRAMEWORK_VERSION = cv.Version(3, 1, 1)
+MIN_FRAMEWORK_VERSION = Version(3, 1, 1)
 
 
-def framework_package_version(ver: cv.Version) -> str:
+def framework_package_version(ver: Version) -> str:
     """Map an Arduino core version (e.g. 3.1.2) to its package version.
 
     Same encoding as the PlatformIO package registry uses for core 3.x
@@ -158,8 +147,10 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None
         if ver.get("name") != version:
             continue
         for file in ver.get("files", []):
-            # ensure_list: a bare string would make ``in`` a substring test
-            systems = ensure_list(file.get("system") or "*")
+            # A bare string would make ``in`` a substring test
+            systems = file.get("system") or "*"
+            if isinstance(systems, str):
+                systems = [systems]
             if "*" in systems or system in systems:
                 sha256 = (file.get("checksum") or {}).get("sha256")
                 if not sha256:
@@ -241,22 +232,21 @@ def _find_ninja() -> Path:
         return Path(binary)
     try:
         import ninja
-    except ImportError as err:
-        raise EsphomeError(
-            "ninja not found on PATH or in the ninja package; reinstall the "
-            "esphome Python environment"
-        ) from err
-
-    binary = Path(ninja.BIN_DIR) / ("ninja.exe" if os.name == "nt" else "ninja")
-    if not binary.is_file():
+    except ImportError:
+        wheel_binary = None
+    else:
+        wheel_binary = Path(ninja.BIN_DIR) / (
+            "ninja.exe" if os.name == "nt" else "ninja"
+        )
+    if wheel_binary is None or not wheel_binary.is_file():
         raise EsphomeError(
             "ninja not found on PATH or in the ninja package; reinstall the "
             "esphome Python environment"
         )
-    return binary
+    return wheel_binary
 
 
-def check_and_install(framework_version: cv.Version) -> dict[str, Path]:
+def check_and_install(framework_version: Version) -> dict[str, Path]:
     """Ensure framework, toolchain, and ninja are installed; return their paths."""
     package_version = framework_package_version(framework_version)
     framework_path = get_framework_path(package_version)
@@ -291,29 +281,8 @@ def get_build_env(toolchain_path: Path) -> dict[str, str]:
 
 @functools.cache
 def ccache_path() -> str | None:
-    """The ccache binary to prefix compiles with, or None when disabled.
-
-    Same convention as the PlatformIO path: on by default when the binary is
-    on PATH, ``ESPHOME_CCACHE_ENABLE=0`` disables it, and an explicit ``=1``
-    warns when no binary is found and skips the runnability probe.
-    """
-    from esphome.platformio.toolchain import _ccache_runs, _strip_win_long_path_prefix
-
-    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
-    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
-        return None
-    ccache = shutil.which("ccache")
-    if ccache is None:
-        if explicit:
-            _LOGGER.warning(
-                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
-                "compiling without ccache"
-            )
-        return None
-    ccache = _strip_win_long_path_prefix(ccache)
-    if not explicit and not _ccache_runs(ccache):
-        return None
-    return ccache
+    """The ccache binary to prefix compiles with, or None when disabled."""
+    return resolve_ccache_path()
 
 
 def ccache_env() -> dict[str, str]:
@@ -326,10 +295,4 @@ def ccache_env() -> dict[str, str]:
     """
     if ccache_path() is None:
         return {}
-    defaults = {
-        "CCACHE_DIR": str(get_arduino8266_tools_path() / "ccache"),
-        "CCACHE_NOHASHDIR": "true",
-        "CCACHE_DEPEND": "1",
-        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
-    }
-    return {k: v for k, v in defaults.items() if k not in os.environ}
+    return ccache_defaults_env(get_arduino8266_tools_path() / "ccache")

esphome/components/esp8266/__init__.py

@@ -134,7 +134,11 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
         return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
     if ver <= cv.Version(2, 6, 2):
         return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-    return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+    # Same encoding the native toolchain uses for its package download, so a
+    # version bump cannot drift between the two paths.
+    from esphome.arduino8266.framework import framework_package_version
+
+    return f"~{framework_package_version(ver)}"
 
 
 # NOTE: Keep this in mind when updating the recommended version:

esphome/components/nrf52/framework.py

@@ -7,8 +7,6 @@ import shutil
 import sys
 import tempfile
 
-import platformdirs
-
 import esphome.config_validation as cv
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
@@ -20,8 +18,8 @@ from esphome.framework_helpers import (
     rmdir,
     run_command_ok,
     str_to_lst_of_str,
+    tools_cache_path,
 )
-from esphome.helpers import get_str_env
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -51,15 +49,9 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str(
 
 
 def get_sdk_nrf_tools_path() -> Path:
-    # A blank ESPHOME_SDK_NRF_PREFIX must be treated as unset: Path("")
-    # resolves to the CWD, which clean-all would then delete.
-    if prefix := get_str_env("ESPHOME_SDK_NRF_PREFIX", "").strip():
-        path = Path(prefix).expanduser()
-    else:
-        # Machine-global (OS user cache dir) so all projects share one install;
-        # see espidf.framework.get_idf_tools_path for the location rationale.
-        path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf"
-    return path.resolve()
+    # Machine-global (OS user cache dir) so all projects share one install;
+    # see espidf.framework.get_idf_tools_path for the location rationale.
+    return tools_cache_path("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf")
 
 
 def _needs_venv_rebuild(

esphome/espidf/framework.py

@@ -11,8 +11,6 @@ import re
 import shutil
 from typing import Any, NoReturn
 
-import platformdirs
-
 from esphome.core import CORE, Version
 from esphome.framework_helpers import (
     PathType,
@@ -26,8 +24,9 @@ from esphome.framework_helpers import (
     run_command,
     run_command_ok,
     str_to_lst_of_str,
+    tools_cache_path,
 )
-from esphome.helpers import get_bool_env, get_str_env, write_file_if_changed
+from esphome.helpers import get_bool_env, write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -88,22 +87,10 @@ def get_idf_tools_path() -> Path:
     Returns:
         Path object pointing to the ESP-IDF tools directory
     """
-    # Treat an empty/whitespace ESPHOME_ESP_IDF_PREFIX as unset: Path("")
-    # resolves to the CWD, which would install into (and let clean-all delete)
-    # the working directory by accident.
-    if prefix := get_str_env("ESPHOME_ESP_IDF_PREFIX", "").strip():
-        path = Path(prefix).expanduser()
-    else:
-        # Machine-global so all projects share the multi-GB install instead of
-        # a per-config-directory copy. The user cache dir (not ~/.esphome)
-        # avoids colliding with data_dir when configs live in the home dir.
-        # appauthor=False drops the redundant <author>\ segment on Windows
-        # (which otherwise repeats "esphome\esphome\") to keep the path short.
-        path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
-    # Resolve so an unnormalized config path (e.g. compiling ``../config/x.yaml``)
-    # doesn't leave ``..`` segments in the IDF_TOOLS_PATH handed to idf.py, which
-    # otherwise warns that the venv interpreter path doesn't match the install.
-    return path.resolve()
+    # Machine-global so all projects share the multi-GB install instead of
+    # a per-config-directory copy; see framework_helpers.tools_cache_path
+    # for the env-override and normalization rules.
+    return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf")
 
 
 # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply

esphome/framework_helpers.py

@@ -1169,3 +1169,135 @@ def download_from_mirrors(
             f"No mirror URL template matched the provided substitutions:{details}"
         )
     raise ValueError("download_from_mirrors called with an empty mirrors list")
+
+
+def tools_cache_path(env_var: str, subdir: str) -> Path:
+    """A backend's machine-global tools directory, with an env override.
+
+    A blank/whitespace override is treated as unset: ``Path("")`` resolves
+    to the CWD, which ``clean-all`` would then delete.
+    """
+    import platformdirs
+
+    from esphome.helpers import get_str_env
+
+    if prefix := get_str_env(env_var, "").strip():
+        return Path(prefix).expanduser().resolve()
+    return (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir
+    ).resolve()
+
+
+def strip_win_long_path_prefix(path: str) -> str:
+    r"""Strip the Windows extended-length path prefix from ``path``.
+
+    Handles both forms documented at
+    https://learn.microsoft.com/windows/win32/fileio/naming-a-file:
+
+    * ``\\?\C:\path\to\file`` -> ``C:\path\to\file``
+    * ``\\?\UNC\server\share\path`` -> ``\\server\share\path``
+
+    The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with
+    ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates
+    into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from
+    the environment, falling back to ``os.path.normpath(sys.executable)``)
+    and ends up baked into SCons-emitted command lines for build steps such
+    as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand
+    the ``\\?\`` prefix, so the build fails with
+    "The system cannot find the path specified." Stripping the prefix early
+    keeps the path shell-quotable.
+
+    Also applied to the ccache path exported by the ccache helpers, which
+    ``shutil.which`` can return with the same prefix.
+
+    No-op on non-Windows platforms.
+    """
+    if sys.platform != "win32":
+        return path
+    if path.startswith("\\\\?\\UNC\\"):
+        # \\?\UNC\server\share\... -> \\server\share\...
+        return "\\\\" + path[len("\\\\?\\UNC\\") :]
+    if path.startswith("\\\\?\\"):
+        return path[len("\\\\?\\") :]
+    return path
+
+
+def _ccache_runs(ccache: str) -> bool:
+    """Return True when the ``ccache`` found on PATH actually runs.
+
+    ``shutil.which`` proves existence, not runnability: on Windows it also
+    matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
+    target is gone. Wrapping compiles around such a find fails every compile
+    step with an opaque OS error, so probe once and fall back to compiling
+    without ccache when the probe fails.
+    """
+    try:
+        subprocess.run(
+            [ccache, "--version"],
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            timeout=15,
+        )
+    except (OSError, subprocess.SubprocessError):
+        _LOGGER.warning(
+            "Ignoring ccache at %s because it failed to run; compiling without ccache",
+            ccache,
+        )
+        return False
+    return True
+
+
+def resolve_ccache_path() -> str | None:
+    """The ccache binary to wrap compiles with, or None when disabled.
+
+    Shared policy for every backend: on by default when a runnable ccache is
+    on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1``
+    warns when no binary is found and skips the runnability probe. The
+    Windows extended-length prefix is stripped before probing so the probe
+    validates the exact string the build will execute (#18399).
+    """
+    import shutil
+
+    from esphome.helpers import get_bool_env
+
+    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
+    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
+        return None
+    ccache = shutil.which("ccache")
+    if ccache is None:
+        if explicit:
+            _LOGGER.warning(
+                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
+                "compiling without ccache"
+            )
+        return None
+    ccache = strip_win_long_path_prefix(ccache)
+    if not explicit and not _ccache_runs(ccache):
+        return None
+    return ccache
+
+
+def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
+    """Default ``CCACHE_*`` values for a build subprocess (not os.environ).
+
+    Values the user already set in the environment are respected. Depend
+    mode is on: both native backends emit depfiles (-MMD / CMake), which
+    keeps cache-miss overhead low.
+    """
+    from esphome.core import CORE
+
+    # build_path is set during preload for every config-loading command; unset
+    # means the caller built the environment too early. Fail loudly rather
+    # than silently drop CCACHE_BASEDIR (losing cross-device cache hits).
+    if CORE.build_path is None:
+        raise ValueError(
+            "CORE.build_path must be set before constructing the build environment"
+        )
+    defaults = {
+        "CCACHE_DIR": str(cache_dir),
+        "CCACHE_NOHASHDIR": "true",
+        "CCACHE_DEPEND": "1",
+        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
+    }
+    return {k: v for k, v in defaults.items() if k not in os.environ}

esphome/platformio/toolchain.py

@@ -4,8 +4,6 @@ import logging
 import os
 from pathlib import Path
 import re
-import shutil
-import subprocess
 import sys
 from typing import TYPE_CHECKING, Any
 
@@ -13,10 +11,10 @@ import platformdirs
 
 from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE
 from esphome.core import CORE, EsphomeError
+from esphome.framework_helpers import resolve_ccache_path, strip_win_long_path_prefix
 from esphome.helpers import (
     add_git_ceiling_directory,
     copy_file_if_changed,
-    get_bool_env,
     rmtree,
     write_file,
 )
@@ -41,40 +39,6 @@ _PIO_PYTHON_STAMP_LOCK = ".esphome.pio.stamp.lock"
 _PIO_PYTHON_STAMP_SCHEMA = "0"
 
 
-def _strip_win_long_path_prefix(path: str) -> str:
-    r"""Strip the Windows extended-length path prefix from ``path``.
-
-    Handles both forms documented at
-    https://learn.microsoft.com/windows/win32/fileio/naming-a-file:
-
-    * ``\\?\C:\path\to\file`` -> ``C:\path\to\file``
-    * ``\\?\UNC\server\share\path`` -> ``\\server\share\path``
-
-    The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with
-    ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates
-    into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from
-    the environment, falling back to ``os.path.normpath(sys.executable)``)
-    and ends up baked into SCons-emitted command lines for build steps such
-    as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand
-    the ``\\?\`` prefix, so the build fails with
-    "The system cannot find the path specified." Stripping the prefix early
-    keeps the path shell-quotable.
-
-    Also applied to the ccache path exported by ``_ccache_env()``, which
-    ``shutil.which`` can return with the same prefix.
-
-    No-op on non-Windows platforms.
-    """
-    if sys.platform != "win32":
-        return path
-    if path.startswith("\\\\?\\UNC\\"):
-        # \\?\UNC\server\share\... -> \\server\share\...
-        return "\\\\" + path[len("\\\\?\\UNC\\") :]
-    if path.startswith("\\\\?\\"):
-        return path[len("\\\\?\\") :]
-    return path
-
-
 def get_platformio_config() -> "ProjectConfig | None":
     """Return PlatformIO's ``ProjectConfig``, or None when PlatformIO is absent."""
     try:
@@ -238,32 +202,6 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None:
         _write_pio_stamp_python(stamp_file, current)
 
 
-def _ccache_runs(ccache: str) -> bool:
-    """Return True when the ``ccache`` found on PATH actually runs.
-
-    ``shutil.which`` proves existence, not runnability: on Windows it also
-    matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
-    target is gone. Wrapping compiles around such a find fails every compile
-    step with an opaque OS error, so probe once and fall back to compiling
-    without ccache when the probe fails.
-    """
-    try:
-        subprocess.run(
-            [ccache, "--version"],
-            check=True,
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-            timeout=15,
-        )
-    except (OSError, subprocess.SubprocessError):
-        _LOGGER.warning(
-            "Ignoring ccache at %s because it failed to run; compiling without ccache",
-            ccache,
-        )
-        return False
-    return True
-
-
 def _ccache_env() -> dict[str, str]:
     r"""Return ccache settings for PlatformIO builds.
 
@@ -282,7 +220,7 @@ def _ccache_env() -> dict[str, str]:
     runs fine through ``CreateProcess``, which is how ESP-IDF invokes it,
     but SCons runs every compile through ``cmd.exe``, which fails on it with
     "The system cannot find the path specified." (#18399), so the prefix is
-    stripped here with ``_strip_win_long_path_prefix()`` before the
+    stripped here with ``strip_win_long_path_prefix()`` before the
     runnability probe, which therefore validates the exact string the build
     will execute.
     ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the
@@ -308,22 +246,8 @@ def _ccache_env() -> dict[str, str]:
     build dir. The other ``CCACHE_*`` values the user already set in the
     environment are respected.
     """
-    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
-    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
-        return {"ESPHOME_CCACHE_ENABLE": "0"}
-    ccache_path = shutil.which("ccache")
+    ccache_path = resolve_ccache_path()
     if ccache_path is None:
-        if explicit:
-            _LOGGER.warning(
-                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
-                "compiling without ccache"
-            )
-        return {"ESPHOME_CCACHE_ENABLE": "0"}
-    # Strip before probing so the probe validates (and the failure warning
-    # names) the exact string the build will execute through cmd.exe.
-    ccache_path = _strip_win_long_path_prefix(ccache_path)
-    # An explicit opt-in skips the runnability probe.
-    if not explicit and not _ccache_runs(ccache_path):
         return {"ESPHOME_CCACHE_ENABLE": "0"}
     env = {
         "ESPHOME_CCACHE_ENABLE": "1",
@@ -385,7 +309,7 @@ def run_platformio_cli(*args, **kwargs) -> str | int:
     # Strip the Windows extended-length path prefix from sys.executable so it
     # doesn't propagate into PlatformIO's $PYTHONEXE and break SCons-emitted
     # command lines run through cmd.exe.
-    python_exe = _strip_win_long_path_prefix(sys.executable)
+    python_exe = strip_win_long_path_prefix(sys.executable)
     if python_exe != sys.executable:
         # Only override PYTHONEXEPATH when we actually stripped a prefix.
         # PlatformIO's get_pythonexe_path() reads this and falls back to

tests/unit_tests/test_arduino8266_framework.py

@@ -202,7 +202,7 @@ def test_registry_download_no_system_match() -> None:
 
 
 def test_registry_download_version_not_found() -> None:
-    resp = _registry_response([])
+    resp = MagicMock()
     resp.json.return_value = {"versions": [{"name": "2.0.0", "files": []}]}
     with (
         patch("requests.get", return_value=resp),
@@ -364,7 +364,7 @@ def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> No
     monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1")
     with (
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.platformio.toolchain._ccache_runs", side_effect=AssertionError),
+        patch("esphome.framework_helpers._ccache_runs", side_effect=AssertionError),
     ):
         assert framework.ccache_path() == "/usr/bin/ccache"
 
@@ -429,3 +429,13 @@ def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None:
         framework._install_package("pkg", "1.0.0", dest, ["http://m"])
     mock_download.assert_not_called()
     mock_rmdir.assert_not_called()
+
+
+def test_ccache_env_requires_build_path() -> None:
+    """Building the env before preload set build_path fails loudly."""
+    CORE.build_path = None
+    with (
+        patch.object(framework, "ccache_path", return_value="/cc/ccache"),
+        pytest.raises(ValueError, match="build_path"),
+    ):
+        framework.ccache_env()

tests/unit_tests/test_platformio_toolchain.py

@@ -874,7 +874,7 @@ def test_strip_win_long_path_prefix(
 ) -> None:
     r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32."""
     with patch("esphome.platformio.toolchain.sys.platform", platform):
-        assert toolchain._strip_win_long_path_prefix(input_path) == expected
+        assert toolchain.strip_win_long_path_prefix(input_path) == expected
 
 
 def test_run_platformio_cli_strips_win_long_path_prefix(