From 0fe9d871247b66bdc4279cca2c7ba8af6ca05181 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 14:04:38 -0500 Subject: [PATCH] Share the ccache and tools-path helpers across the native backends --- esphome/arduino8266/framework.py | 87 ++++-------- esphome/components/esp8266/__init__.py | 6 +- esphome/components/nrf52/framework.py | 16 +-- esphome/espidf/framework.py | 25 +--- esphome/framework_helpers.py | 132 ++++++++++++++++++ esphome/platformio/toolchain.py | 84 +---------- .../unit_tests/test_arduino8266_framework.py | 14 +- tests/unit_tests/test_platformio_toolchain.py | 2 +- 8 files changed, 189 insertions(+), 177 deletions(-) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 066667806c..8d828dc936 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -25,19 +25,17 @@ from pathlib import Path import platform import shutil -import platformdirs - -import esphome.config_validation as cv -from esphome.core import CORE, EsphomeError +from esphome.core import EsphomeError, Version from esphome.framework_helpers import ( archive_extract_all, + ccache_defaults_env, download_from_mirrors, download_with_resume, + resolve_ccache_path, rmdir, str_to_lst_of_str, + tools_cache_path, ) -from esphome.helpers import get_bool_env, get_str_env -from esphome.platformio.library import ensure_list _LOGGER = logging.getLogger(__name__) @@ -61,25 +59,16 @@ ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( def get_arduino8266_tools_path() -> Path: - # Treat an empty/whitespace prefix as unset: Path("") resolves to the CWD, - # which clean-all would then delete. - if prefix := get_str_env("ESPHOME_ARDUINO8266_PREFIX", "").strip(): - path = Path(prefix).expanduser() - else: - # Machine-global so all projects share one install; see - # espidf.framework.get_idf_tools_path for the location rationale. - path = ( - Path(platformdirs.user_cache_dir("esphome", appauthor=False)) - / "arduino8266" - ) - return path.resolve() + # Machine-global so all projects share one install; see + # espidf.framework.get_idf_tools_path for the location rationale. + return tools_cache_path("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") # 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0 -MIN_FRAMEWORK_VERSION = cv.Version(3, 1, 1) +MIN_FRAMEWORK_VERSION = Version(3, 1, 1) -def framework_package_version(ver: cv.Version) -> str: +def framework_package_version(ver: Version) -> str: """Map an Arduino core version (e.g. 3.1.2) to its package version. Same encoding as the PlatformIO package registry uses for core 3.x @@ -158,8 +147,10 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None if ver.get("name") != version: continue for file in ver.get("files", []): - # ensure_list: a bare string would make ``in`` a substring test - systems = ensure_list(file.get("system") or "*") + # A bare string would make ``in`` a substring test + systems = file.get("system") or "*" + if isinstance(systems, str): + systems = [systems] if "*" in systems or system in systems: sha256 = (file.get("checksum") or {}).get("sha256") if not sha256: @@ -241,22 +232,21 @@ def _find_ninja() -> Path: return Path(binary) try: import ninja - except ImportError as err: - raise EsphomeError( - "ninja not found on PATH or in the ninja package; reinstall the " - "esphome Python environment" - ) from err - - binary = Path(ninja.BIN_DIR) / ("ninja.exe" if os.name == "nt" else "ninja") - if not binary.is_file(): + except ImportError: + wheel_binary = None + else: + wheel_binary = Path(ninja.BIN_DIR) / ( + "ninja.exe" if os.name == "nt" else "ninja" + ) + if wheel_binary is None or not wheel_binary.is_file(): raise EsphomeError( "ninja not found on PATH or in the ninja package; reinstall the " "esphome Python environment" ) - return binary + return wheel_binary -def check_and_install(framework_version: cv.Version) -> dict[str, Path]: +def check_and_install(framework_version: Version) -> dict[str, Path]: """Ensure framework, toolchain, and ninja are installed; return their paths.""" package_version = framework_package_version(framework_version) framework_path = get_framework_path(package_version) @@ -291,29 +281,8 @@ def get_build_env(toolchain_path: Path) -> dict[str, str]: @functools.cache def ccache_path() -> str | None: - """The ccache binary to prefix compiles with, or None when disabled. - - Same convention as the PlatformIO path: on by default when the binary is - on PATH, ``ESPHOME_CCACHE_ENABLE=0`` disables it, and an explicit ``=1`` - warns when no binary is found and skips the runnability probe. - """ - from esphome.platformio.toolchain import _ccache_runs, _strip_win_long_path_prefix - - explicit = "ESPHOME_CCACHE_ENABLE" in os.environ - if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): - return None - ccache = shutil.which("ccache") - if ccache is None: - if explicit: - _LOGGER.warning( - "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " - "compiling without ccache" - ) - return None - ccache = _strip_win_long_path_prefix(ccache) - if not explicit and not _ccache_runs(ccache): - return None - return ccache + """The ccache binary to prefix compiles with, or None when disabled.""" + return resolve_ccache_path() def ccache_env() -> dict[str, str]: @@ -326,10 +295,4 @@ def ccache_env() -> dict[str, str]: """ if ccache_path() is None: return {} - defaults = { - "CCACHE_DIR": str(get_arduino8266_tools_path() / "ccache"), - "CCACHE_NOHASHDIR": "true", - "CCACHE_DEPEND": "1", - "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), - } - return {k: v for k, v in defaults.items() if k not in os.environ} + return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index abc2f5dc91..4084893e7c 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -134,7 +134,11 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" if ver <= cv.Version(2, 6, 2): return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + # Same encoding the native toolchain uses for its package download, so a + # version bump cannot drift between the two paths. + from esphome.arduino8266.framework import framework_package_version + + return f"~{framework_package_version(ver)}" # NOTE: Keep this in mind when updating the recommended version: diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index d487820440..48963fa89f 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -7,8 +7,6 @@ import shutil import sys import tempfile -import platformdirs - import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -20,8 +18,8 @@ from esphome.framework_helpers import ( rmdir, run_command_ok, str_to_lst_of_str, + tools_cache_path, ) -from esphome.helpers import get_str_env _LOGGER = logging.getLogger(__name__) @@ -51,15 +49,9 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str( def get_sdk_nrf_tools_path() -> Path: - # A blank ESPHOME_SDK_NRF_PREFIX must be treated as unset: Path("") - # resolves to the CWD, which clean-all would then delete. - if prefix := get_str_env("ESPHOME_SDK_NRF_PREFIX", "").strip(): - path = Path(prefix).expanduser() - else: - # Machine-global (OS user cache dir) so all projects share one install; - # see espidf.framework.get_idf_tools_path for the location rationale. - path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf" - return path.resolve() + # Machine-global (OS user cache dir) so all projects share one install; + # see espidf.framework.get_idf_tools_path for the location rationale. + return tools_cache_path("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") def _needs_venv_rebuild( diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 0f6ef873b8..6c01058044 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -11,8 +11,6 @@ import re import shutil from typing import Any, NoReturn -import platformdirs - from esphome.core import CORE, Version from esphome.framework_helpers import ( PathType, @@ -26,8 +24,9 @@ from esphome.framework_helpers import ( run_command, run_command_ok, str_to_lst_of_str, + tools_cache_path, ) -from esphome.helpers import get_bool_env, get_str_env, write_file_if_changed +from esphome.helpers import get_bool_env, write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -88,22 +87,10 @@ def get_idf_tools_path() -> Path: Returns: Path object pointing to the ESP-IDF tools directory """ - # Treat an empty/whitespace ESPHOME_ESP_IDF_PREFIX as unset: Path("") - # resolves to the CWD, which would install into (and let clean-all delete) - # the working directory by accident. - if prefix := get_str_env("ESPHOME_ESP_IDF_PREFIX", "").strip(): - path = Path(prefix).expanduser() - else: - # Machine-global so all projects share the multi-GB install instead of - # a per-config-directory copy. The user cache dir (not ~/.esphome) - # avoids colliding with data_dir when configs live in the home dir. - # appauthor=False drops the redundant \ segment on Windows - # (which otherwise repeats "esphome\esphome\") to keep the path short. - path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf" - # Resolve so an unnormalized config path (e.g. compiling ``../config/x.yaml``) - # doesn't leave ``..`` segments in the IDF_TOOLS_PATH handed to idf.py, which - # otherwise warns that the venv interpreter path doesn't match the install. - return path.resolve() + # Machine-global so all projects share the multi-GB install instead of + # a per-config-directory copy; see framework_helpers.tools_cache_path + # for the env-override and normalization rules. + return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf") # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index b8a43220ff..224223f05d 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -1169,3 +1169,135 @@ def download_from_mirrors( f"No mirror URL template matched the provided substitutions:{details}" ) raise ValueError("download_from_mirrors called with an empty mirrors list") + + +def tools_cache_path(env_var: str, subdir: str) -> Path: + """A backend's machine-global tools directory, with an env override. + + A blank/whitespace override is treated as unset: ``Path("")`` resolves + to the CWD, which ``clean-all`` would then delete. + """ + import platformdirs + + from esphome.helpers import get_str_env + + if prefix := get_str_env(env_var, "").strip(): + return Path(prefix).expanduser().resolve() + return ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir + ).resolve() + + +def strip_win_long_path_prefix(path: str) -> str: + r"""Strip the Windows extended-length path prefix from ``path``. + + Handles both forms documented at + https://learn.microsoft.com/windows/win32/fileio/naming-a-file: + + * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` + * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` + + The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with + ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates + into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from + the environment, falling back to ``os.path.normpath(sys.executable)``) + and ends up baked into SCons-emitted command lines for build steps such + as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand + the ``\\?\`` prefix, so the build fails with + "The system cannot find the path specified." Stripping the prefix early + keeps the path shell-quotable. + + Also applied to the ccache path exported by the ccache helpers, which + ``shutil.which`` can return with the same prefix. + + No-op on non-Windows platforms. + """ + if sys.platform != "win32": + return path + if path.startswith("\\\\?\\UNC\\"): + # \\?\UNC\server\share\... -> \\server\share\... + return "\\\\" + path[len("\\\\?\\UNC\\") :] + if path.startswith("\\\\?\\"): + return path[len("\\\\?\\") :] + return path + + +def _ccache_runs(ccache: str) -> bool: + """Return True when the ``ccache`` found on PATH actually runs. + + ``shutil.which`` proves existence, not runnability: on Windows it also + matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose + target is gone. Wrapping compiles around such a find fails every compile + step with an opaque OS error, so probe once and fall back to compiling + without ccache when the probe fails. + """ + try: + subprocess.run( + [ccache, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + _LOGGER.warning( + "Ignoring ccache at %s because it failed to run; compiling without ccache", + ccache, + ) + return False + return True + + +def resolve_ccache_path() -> str | None: + """The ccache binary to wrap compiles with, or None when disabled. + + Shared policy for every backend: on by default when a runnable ccache is + on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1`` + warns when no binary is found and skips the runnability probe. The + Windows extended-length prefix is stripped before probing so the probe + validates the exact string the build will execute (#18399). + """ + import shutil + + from esphome.helpers import get_bool_env + + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return None + ccache = shutil.which("ccache") + if ccache is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return None + ccache = strip_win_long_path_prefix(ccache) + if not explicit and not _ccache_runs(ccache): + return None + return ccache + + +def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: + """Default ``CCACHE_*`` values for a build subprocess (not os.environ). + + Values the user already set in the environment are respected. Depend + mode is on: both native backends emit depfiles (-MMD / CMake), which + keeps cache-miss overhead low. + """ + from esphome.core import CORE + + # build_path is set during preload for every config-loading command; unset + # means the caller built the environment too early. Fail loudly rather + # than silently drop CCACHE_BASEDIR (losing cross-device cache hits). + if CORE.build_path is None: + raise ValueError( + "CORE.build_path must be set before constructing the build environment" + ) + defaults = { + "CCACHE_DIR": str(cache_dir), + "CCACHE_NOHASHDIR": "true", + "CCACHE_DEPEND": "1", + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + } + return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index d76581d032..3eeaecf7e4 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -4,8 +4,6 @@ import logging import os from pathlib import Path import re -import shutil -import subprocess import sys from typing import TYPE_CHECKING, Any @@ -13,10 +11,10 @@ import platformdirs from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import resolve_ccache_path, strip_win_long_path_prefix from esphome.helpers import ( add_git_ceiling_directory, copy_file_if_changed, - get_bool_env, rmtree, write_file, ) @@ -41,40 +39,6 @@ _PIO_PYTHON_STAMP_LOCK = ".esphome.pio.stamp.lock" _PIO_PYTHON_STAMP_SCHEMA = "0" -def _strip_win_long_path_prefix(path: str) -> str: - r"""Strip the Windows extended-length path prefix from ``path``. - - Handles both forms documented at - https://learn.microsoft.com/windows/win32/fileio/naming-a-file: - - * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` - * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` - - The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with - ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates - into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from - the environment, falling back to ``os.path.normpath(sys.executable)``) - and ends up baked into SCons-emitted command lines for build steps such - as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand - the ``\\?\`` prefix, so the build fails with - "The system cannot find the path specified." Stripping the prefix early - keeps the path shell-quotable. - - Also applied to the ccache path exported by ``_ccache_env()``, which - ``shutil.which`` can return with the same prefix. - - No-op on non-Windows platforms. - """ - if sys.platform != "win32": - return path - if path.startswith("\\\\?\\UNC\\"): - # \\?\UNC\server\share\... -> \\server\share\... - return "\\\\" + path[len("\\\\?\\UNC\\") :] - if path.startswith("\\\\?\\"): - return path[len("\\\\?\\") :] - return path - - def get_platformio_config() -> "ProjectConfig | None": """Return PlatformIO's ``ProjectConfig``, or None when PlatformIO is absent.""" try: @@ -238,32 +202,6 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) -def _ccache_runs(ccache: str) -> bool: - """Return True when the ``ccache`` found on PATH actually runs. - - ``shutil.which`` proves existence, not runnability: on Windows it also - matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose - target is gone. Wrapping compiles around such a find fails every compile - step with an opaque OS error, so probe once and fall back to compiling - without ccache when the probe fails. - """ - try: - subprocess.run( - [ccache, "--version"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=15, - ) - except (OSError, subprocess.SubprocessError): - _LOGGER.warning( - "Ignoring ccache at %s because it failed to run; compiling without ccache", - ccache, - ) - return False - return True - - def _ccache_env() -> dict[str, str]: r"""Return ccache settings for PlatformIO builds. @@ -282,7 +220,7 @@ def _ccache_env() -> dict[str, str]: runs fine through ``CreateProcess``, which is how ESP-IDF invokes it, but SCons runs every compile through ``cmd.exe``, which fails on it with "The system cannot find the path specified." (#18399), so the prefix is - stripped here with ``_strip_win_long_path_prefix()`` before the + stripped here with ``strip_win_long_path_prefix()`` before the runnability probe, which therefore validates the exact string the build will execute. ``ESPHOME_CCACHE_PATH`` is an internal channel, not a user setting: the @@ -308,22 +246,8 @@ def _ccache_env() -> dict[str, str]: build dir. The other ``CCACHE_*`` values the user already set in the environment are respected. """ - explicit = "ESPHOME_CCACHE_ENABLE" in os.environ - if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): - return {"ESPHOME_CCACHE_ENABLE": "0"} - ccache_path = shutil.which("ccache") + ccache_path = resolve_ccache_path() if ccache_path is None: - if explicit: - _LOGGER.warning( - "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " - "compiling without ccache" - ) - return {"ESPHOME_CCACHE_ENABLE": "0"} - # Strip before probing so the probe validates (and the failure warning - # names) the exact string the build will execute through cmd.exe. - ccache_path = _strip_win_long_path_prefix(ccache_path) - # An explicit opt-in skips the runnability probe. - if not explicit and not _ccache_runs(ccache_path): return {"ESPHOME_CCACHE_ENABLE": "0"} env = { "ESPHOME_CCACHE_ENABLE": "1", @@ -385,7 +309,7 @@ def run_platformio_cli(*args, **kwargs) -> str | int: # Strip the Windows extended-length path prefix from sys.executable so it # doesn't propagate into PlatformIO's $PYTHONEXE and break SCons-emitted # command lines run through cmd.exe. - python_exe = _strip_win_long_path_prefix(sys.executable) + python_exe = strip_win_long_path_prefix(sys.executable) if python_exe != sys.executable: # Only override PYTHONEXEPATH when we actually stripped a prefix. # PlatformIO's get_pythonexe_path() reads this and falls back to diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 2e72dfb26a..7602674c16 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -202,7 +202,7 @@ def test_registry_download_no_system_match() -> None: def test_registry_download_version_not_found() -> None: - resp = _registry_response([]) + resp = MagicMock() resp.json.return_value = {"versions": [{"name": "2.0.0", "files": []}]} with ( patch("requests.get", return_value=resp), @@ -364,7 +364,7 @@ def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> No monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1") with ( patch("shutil.which", return_value="/usr/bin/ccache"), - patch("esphome.platformio.toolchain._ccache_runs", side_effect=AssertionError), + patch("esphome.framework_helpers._ccache_runs", side_effect=AssertionError), ): assert framework.ccache_path() == "/usr/bin/ccache" @@ -429,3 +429,13 @@ def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None: framework._install_package("pkg", "1.0.0", dest, ["http://m"]) mock_download.assert_not_called() mock_rmdir.assert_not_called() + + +def test_ccache_env_requires_build_path() -> None: + """Building the env before preload set build_path fails loudly.""" + CORE.build_path = None + with ( + patch.object(framework, "ccache_path", return_value="/cc/ccache"), + pytest.raises(ValueError, match="build_path"), + ): + framework.ccache_env() diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 172b288c25..cb421c6d01 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -874,7 +874,7 @@ def test_strip_win_long_path_prefix( ) -> None: r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32.""" with patch("esphome.platformio.toolchain.sys.platform", platform): - assert toolchain._strip_win_long_path_prefix(input_path) == expected + assert toolchain.strip_win_long_path_prefix(input_path) == expected def test_run_platformio_cli_strips_win_long_path_prefix(