From b79cbcbde77dce3582c5d4ae9dd0045b08a596e2 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:56:18 -0400 Subject: [PATCH] [espidf] Install native ESP-IDF into a machine-global cache dir (#17306) --- docker/docker_entrypoint.sh | 4 ++ .../etc/s6-overlay/s6-rc.d/esphome/run | 4 ++ esphome/__main__.py | 5 +- esphome/espidf/clang_tidy.py | 6 +-- esphome/espidf/framework.py | 35 +++++++++----- esphome/writer.py | 9 ++++ requirements.txt | 1 + tests/unit_tests/test_espidf_framework.py | 47 +++++++++++++++++++ tests/unit_tests/test_writer.py | 38 +++++++++++++-- 9 files changed, 129 insertions(+), 20 deletions(-) diff --git a/docker/docker_entrypoint.sh b/docker/docker_entrypoint.sh index 18baf40c29..598b553c08 100755 --- a/docker/docker_entrypoint.sh +++ b/docker/docker_entrypoint.sh @@ -21,6 +21,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms" export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages" export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache" +# Keep the native ESP-IDF install on the persistent cache root, not the +# container's ephemeral user cache dir (re-downloaded on every restart). +export ESPHOME_ESP_IDF_PREFIX="$(dirname "${pio_cache_base}")/idf" + # If /build is mounted, use that as the build path # otherwise use path in /config (so that builds aren't lost on container restart) if [[ -d /build ]]; then diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run index dff61fd2f3..f50de659b9 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run @@ -15,6 +15,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms" export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages" export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache" +# Keep the native ESP-IDF install on the persistent /data volume, not the +# container's ephemeral user cache dir (wiped on every add-on update/restart). +export ESPHOME_ESP_IDF_PREFIX=/data/cache/idf + if bashio::config.true 'leave_front_door_open'; then export DISABLE_HA_AUTHENTICATION=true fi diff --git a/esphome/__main__.py b/esphome/__main__.py index 1062df7167..1767d3b7ca 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2386,7 +2386,10 @@ def parse_args(argv): ) parser_clean_all = subparsers.add_parser( - "clean-all", help="Clean all build and platform files." + "clean-all", + help="Clean all build and platform files, including machine-global " + "toolchain caches shared by all configurations, so other projects will " + "re-download them on next build.", ) parser_clean_all.add_argument( "configuration", help="Your YAML file or configuration directory.", nargs="*" diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index d3f4d151c2..88ecda60b9 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -147,9 +147,9 @@ def _setup_core(work_dir: Path, settings: _Settings) -> None: from esphome.core import CORE CORE.name = TIDY_PROJECT_NAME - # config_path's parent is the data dir root: the IDF install lives at - # ``/.esphome/idf`` -- keep it beside (not inside) the per-run - # project dir so clearing the project doesn't force an IDF re-download. + # config_path's parent is the data dir root for per-run artifacts (idedata, + # converted pio_components). The IDF install is in the global cache dir, + # independent of this path. CORE.config_path = work_dir.parent / "tidy.yaml" CORE.build_path = work_dir esp32 = CORE.data.setdefault(KEY_ESP32, {}) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index c994ce2410..25283e3c99 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -9,6 +9,8 @@ import re import shutil import tempfile +import platformdirs + from esphome.config_validation import Version from esphome.core import CORE from esphome.framework_helpers import ( @@ -80,10 +82,18 @@ def _get_idf_tools_path() -> Path: Returns: Path object pointing to the ESP-IDF tools directory """ - if "ESPHOME_ESP_IDF_PREFIX" in os.environ: - path = Path(get_str_env("ESPHOME_ESP_IDF_PREFIX", None)).expanduser() + # Treat an empty/whitespace ESPHOME_ESP_IDF_PREFIX as unset: Path("") + # resolves to the CWD, which would install into (and let clean-all delete) + # the working directory by accident. + if prefix := get_str_env("ESPHOME_ESP_IDF_PREFIX", "").strip(): + path = Path(prefix).expanduser() else: - path = CORE.data_dir / "idf" + # Machine-global so all projects share the multi-GB install instead of + # a per-config-directory copy. The user cache dir (not ~/.esphome) + # avoids colliding with data_dir when configs live in the home dir. + # appauthor=False drops the redundant \ segment on Windows + # (which otherwise repeats "esphome\esphome\") to keep the path short. + path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf" # Resolve so an unnormalized config path (e.g. compiling ``../config/x.yaml``) # doesn't leave ``..`` segments in the IDF_TOOLS_PATH handed to idf.py, which # otherwise warns that the venv interpreter path doesn't match the install. @@ -145,10 +155,11 @@ def _check_windows_path_length() -> None: " fatal error: bits/c++config.h: No such file or directory\n" " cannot execute 'as': CreateProcess: No such file or directory\n" "To fix, either:\n" - " - Enable Windows long path support: set\n" - " HKLM\\SYSTEM\\CurrentControlSet\\Control\\FileSystem\\LongPathsEnabled\n" - " to 1 and reboot, or\n" - " - Move your ESPHome project to a shorter path\n" + " - Enable Windows long path support, then reboot. In an elevated\n" + " PowerShell run:\n" + " Set-ItemProperty 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' LongPathsEnabled 1\n" + " Details: https://learn.microsoft.com/windows/win32/fileio/maximum-file-path-limitation\n" + " - Or set ESPHOME_ESP_IDF_PREFIX to a shorter path (e.g. C:\\ESPHome\\idf)\n" "Then delete the ESP-IDF tools directory above so the toolchain " "reinstalls cleanly.", tools_path, @@ -553,7 +564,7 @@ def _check_esphome_idf_framework_install( # Logged every invocation (not just on install) so the user can verify the # override. A changed URL needs ``esphome clean-all`` to force a re-download # (``esphome clean`` only wipes the build dir, not the extracted framework - # under /idf/frameworks/). + # under the global install dir's ``frameworks/``). if source_url: _LOGGER.info("Using framework source override: %s", source_url) @@ -822,11 +833,9 @@ def _ccache_env() -> dict[str, str]: Enabled by default whenever the ``ccache`` binary is on PATH; set ``IDF_CCACHE_ENABLE=0`` in the environment to opt out. The cache lives under - the IDF tools path. How widely it is shared depends on where that resolves: - across projects (and surviving ``clean-all``) when it is a common location - (``ESPHOME_ESP_IDF_PREFIX`` or the add-on ``/data``), but per-project under - ``.esphome/idf`` for a default pip install, where ``clean-all`` clears it - along with the framework. + the IDF tools path (the machine-global cache dir, or + ``ESPHOME_ESP_IDF_PREFIX``), so it is shared across all projects and removed + by ``esphome clean-all`` along with the framework. Depend mode keeps cache-miss overhead low (hashes the compiler's depfiles instead of preprocessing). ``CCACHE_BASEDIR`` rewrites the per-build diff --git a/esphome/writer.py b/esphome/writer.py index a9c072f156..52f2d169b3 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -653,6 +653,15 @@ def clean_all(configuration: list[str]): elif item.is_dir() and item.name != "storage": rmtree(item) + # The native ESP-IDF install lives in a machine-global cache dir, outside + # any .esphome data dir, so the per-config loop above won't reach it. + from esphome.espidf.framework import _get_idf_tools_path + + idf_install_path = _get_idf_tools_path() + if idf_install_path.is_dir(): + _LOGGER.info("Deleting %s", idf_install_path) + rmtree(idf_install_path) + # Clean PlatformIO project files try: from platformio.project.config import ProjectConfig diff --git a/requirements.txt b/requirements.txt index 85f4b56c07..3832045cfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ bleak==2.1.1 smpclient==6.0.0 requests==2.34.2 py7zr==1.1.3 +platformdirs==4.9.4 # native esp-idf toolchain global cache dir # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index fe888ac8b9..f3e160925a 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -36,6 +36,19 @@ from esphome.espidf.framework import ( from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path +@pytest.fixture(autouse=True) +def _isolate_idf_install_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pin the ESP-IDF install root to a tmp dir for every test. + + The default location is the OS user cache dir, so without this any test + that builds framework paths or pre-creates the framework dir would touch + the real ``~/.cache/esphome`` on the developer's machine. Tests that need + to exercise the override or default-resolution logic clear/override the env + themselves. + """ + monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(tmp_path / "idf_install")) + + @pytest.mark.parametrize( ("source", "expected"), [ @@ -791,6 +804,38 @@ def test_get_idf_tools_path_env_override(tmp_path: Path) -> None: assert _get_idf_tools_path() == Path(override) +@pytest.mark.parametrize("value", ["", " "]) +def test_get_idf_tools_path_blank_env_falls_back_to_default( + value: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A blank ESPHOME_ESP_IDF_PREFIX is treated as unset, not as CWD. + + Path("") would resolve to the working directory, which clean-all could then + delete by accident. + """ + import platformdirs + + monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", value) + expected = ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf" + ).resolve() + assert _get_idf_tools_path() == expected + + +def test_get_idf_tools_path_default_uses_user_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Without the env override the install root is the machine-global OS user + cache dir, not the per-config ``/idf``.""" + import platformdirs + + monkeypatch.delenv("ESPHOME_ESP_IDF_PREFIX", raising=False) + expected = ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf" + ).resolve() + assert _get_idf_tools_path() == expected + + def test_write_idf_version_txt_warns_on_write_error(tmp_path: Path) -> None: with patch("pathlib.Path.write_text", side_effect=OSError("denied")): # write failure is caught and warned, not raised @@ -908,3 +953,5 @@ def test_check_windows_path_length_long_path_warns( message = caplog.records[0].getMessage() assert _LONG_IDF_PATH in message assert "long path support" in message + # The install is global now; the remedy is the prefix env, not moving the project. + assert "ESPHOME_ESP_IDF_PREFIX" in message diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index c8cf68ff3e..18d08e7cb1 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -67,15 +67,23 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any: want to verify the PIO-cleanup branch (e.g. test_clean_all, test_clean_all_partial_exists) install their own inner patch which stacks on top of this one and wins for the duration of their block. + + Also pin ``ESPHOME_ESP_IDF_PREFIX`` to a nonexistent tmp dir for the + same reason: ``clean_all`` removes the now machine-global ESP-IDF + install, which otherwise defaults to the real ``~/.cache/esphome``. """ pio_root = tmp_path_factory.mktemp("isolated_pio") / "nonexistent" + idf_root = tmp_path_factory.mktemp("isolated_idf") / "nonexistent" mock_cfg = MagicMock() mock_cfg.get.side_effect = lambda section, option: ( str(pio_root / option) if section == "platformio" else "" ) - with patch( - "platformio.project.config.ProjectConfig.get_instance", - return_value=mock_cfg, + with ( + patch( + "platformio.project.config.ProjectConfig.get_instance", + return_value=mock_cfg, + ), + patch.dict("os.environ", {"ESPHOME_ESP_IDF_PREFIX": str(idf_root)}), ): yield @@ -990,6 +998,30 @@ def test_clean_all_with_yaml_file( assert str(build_dir) in caplog.text +@patch("esphome.writer.CORE") +def test_clean_all_removes_global_idf_install( + mock_core: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """clean_all removes the machine-global native ESP-IDF install dir.""" + idf_install = tmp_path / "idf_install" + (idf_install / "frameworks").mkdir(parents=True) + monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(idf_install)) + + config_dir = tmp_path / "config" + config_dir.mkdir() + + from esphome.writer import clean_all + + with caplog.at_level("INFO"): + clean_all([str(config_dir)]) + + assert not idf_install.exists() + assert str(idf_install.resolve()) in caplog.text + + @patch("esphome.writer.CORE") def test_clean_all_with_yaml_build_path( mock_core: MagicMock,