[espidf] Enable ccache by default for ESP-IDF builds (#17163)

esphome/espidf/framework.py

@@ -23,7 +23,7 @@ from esphome.framework_helpers import (
     run_command_ok,
     str_to_lst_of_str,
 )
-from esphome.helpers import get_str_env, write_file_if_changed
+from esphome.helpers import get_bool_env, get_str_env, write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -814,6 +814,56 @@ def check_esp_idf_install(
     return framework_path, python_env_path
 
 
+def _ccache_env() -> dict[str, str]:
+    """Return ccache settings for ESP-IDF compiles.
+
+    Enabled by default whenever the ``ccache`` binary is on PATH; set
+    ``IDF_CCACHE_ENABLE=0`` in the environment to opt out. The cache lives under
+    the IDF tools path. How widely it is shared depends on where that resolves:
+    across projects (and surviving ``clean-all``) when it is a common location
+    (``ESPHOME_ESP_IDF_PREFIX`` or the add-on ``/data``), but per-project under
+    ``.esphome/idf`` for a default pip install, where ``clean-all`` clears it
+    along with the framework.
+
+    Depend mode keeps cache-miss overhead low (hashes the compiler's depfiles
+    instead of preprocessing). ``CCACHE_BASEDIR`` rewrites the per-build
+    absolute paths (generated ``sdkconfig`` include, etc.) so different devices
+    share framework cache entries; it is scoped to the build dir on purpose --
+    a broader base would also rewrite the shared IDF path under the cache dir
+    and lose those hits.
+
+    Only values the user has not already set in the environment are returned, so
+    a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected.
+    """
+    # Honor an explicit choice already in the environment (opt-out or opt-in).
+    if "IDF_CCACHE_ENABLE" in os.environ:
+        if not get_bool_env("IDF_CCACHE_ENABLE"):
+            return {}
+    elif shutil.which("ccache") is None:
+        # ESP-IDF silently skips ccache without the binary; don't enable it.
+        return {}
+
+    # ccache is enabled past here. build_path is set during preload for every
+    # config-loading command, so it being unset means a caller built the IDF env
+    # too early -- fail loudly rather than silently drop CCACHE_BASEDIR (which
+    # would quietly cost cross-device cache hits).
+    if CORE.build_path is None:
+        raise ValueError(
+            "CORE.build_path must be set before constructing the ESP-IDF build "
+            "environment"
+        )
+
+    defaults = {
+        "IDF_CCACHE_ENABLE": "1",
+        "CCACHE_DIR": str(_get_idf_tools_path() / "ccache"),
+        "CCACHE_NOHASHDIR": "true",
+        "CCACHE_DEPEND": "1",
+        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
+    }
+    # Don't override CCACHE_* values the user already set in their environment.
+    return {k: v for k, v in defaults.items() if k not in os.environ}
+
+
 def get_framework_env(
     framework_path: PathType,
     python_env_path: PathType | None = None,
@@ -856,4 +906,7 @@ def get_framework_env(
     env.update(export_vars)
     env["PATH"] = os.pathsep.join(paths_to_export + path_list)
 
+    # 6. Enable ccache for the compile toolchain (default on when available).
+    env.update(_ccache_env())
+
     return env

tests/unit_tests/test_espidf_framework.py

@@ -15,6 +15,7 @@ from unittest.mock import patch
 import pytest
 
 from esphome.espidf.framework import (
+    _ccache_env,
     _check_stamp,
     _check_windows_path_length,
     _clone_idf_with_submodules,
@@ -620,6 +621,8 @@ def test_get_framework_env_with_python_env(tmp_path: Path) -> None:
             "esphome.espidf.framework._get_idf_tool_paths",
             return_value=(["/tool/bin"], {"IDF_X": "1"}),
         ),
+        # ccache env is covered separately; keep this test host-independent.
+        patch("esphome.espidf.framework._ccache_env", return_value={}),
     ):
         env = get_framework_env(
             tmp_path / "fw", tmp_path / "penv", {"PATH": "/usr/bin"}
@@ -640,6 +643,8 @@ def test_get_framework_env_without_python_env_uses_os_path(tmp_path: Path) -> No
         ),
         patch("esphome.espidf.framework._get_idf_version", return_value="5.1.2"),
         patch("esphome.espidf.framework._get_idf_tool_paths", return_value=([], {})),
+        # ccache env is covered separately; keep this test host-independent.
+        patch("esphome.espidf.framework._ccache_env", return_value={}),
     ):
         env = get_framework_env(tmp_path / "fw")
 
@@ -647,6 +652,88 @@ def test_get_framework_env_without_python_env_uses_os_path(tmp_path: Path) -> No
     assert env["PATH"]  # taken from os.environ
 
 
+# ---------------------------------------------------------------------------
+# _ccache_env
+# ---------------------------------------------------------------------------
+
+
+def _ccache_patches(tmp_path: Path, which: str | None, build_path: Path | None):
+    return (
+        patch("esphome.espidf.framework.shutil.which", return_value=which),
+        patch(
+            "esphome.espidf.framework._get_idf_tools_path",
+            return_value=tmp_path / "tools",
+        ),
+        patch(
+            "esphome.espidf.framework.CORE",
+            SimpleNamespace(build_path=build_path),
+        ),
+    )
+
+
+def test_ccache_env_default_enabled_when_available(tmp_path: Path) -> None:
+    p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build")
+    with patch.dict("os.environ", {}, clear=True), p1, p2, p3:
+        env = _ccache_env()
+    assert env["IDF_CCACHE_ENABLE"] == "1"
+    assert env["CCACHE_DIR"] == str(tmp_path / "tools" / "ccache")
+    assert env["CCACHE_NOHASHDIR"] == "true"
+    assert env["CCACHE_DEPEND"] == "1"
+    assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve())
+
+
+def test_ccache_env_disabled_when_binary_missing(tmp_path: Path) -> None:
+    # build_path is None here too: a disabled cache must not require it.
+    p1, p2, p3 = _ccache_patches(tmp_path, None, None)
+    with patch.dict("os.environ", {}, clear=True), p1, p2, p3:
+        assert _ccache_env() == {}
+
+
+def test_ccache_env_opt_out_via_env(tmp_path: Path) -> None:
+    # Explicit IDF_CCACHE_ENABLE=0 wins even when the binary is present, and
+    # short-circuits before build_path is needed.
+    p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", None)
+    with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "0"}, clear=True), p1, p2, p3:
+        assert _ccache_env() == {}
+
+
+def test_ccache_env_opt_in_without_binary(tmp_path: Path) -> None:
+    # Explicit IDF_CCACHE_ENABLE=1 forces it on without probing PATH. It's
+    # already in the environment, so it isn't re-emitted, but the rest is.
+    p1, p2, p3 = _ccache_patches(tmp_path, None, tmp_path / "build")
+    with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "1"}, clear=True), p1, p2, p3:
+        env = _ccache_env()
+    assert "IDF_CCACHE_ENABLE" not in env
+    assert env["CCACHE_DIR"] == str(tmp_path / "tools" / "ccache")
+    assert env["CCACHE_DEPEND"] == "1"
+
+
+def test_ccache_env_preserves_user_overrides(tmp_path: Path) -> None:
+    # User-set CCACHE_* values must not be clobbered; unset ones still default.
+    p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build")
+    user_env = {"CCACHE_DIR": "/my/cache", "CCACHE_MAXSIZE": "9G"}
+    with patch.dict("os.environ", user_env, clear=True), p1, p2, p3:
+        env = _ccache_env()
+    assert "CCACHE_DIR" not in env
+    assert "CCACHE_MAXSIZE" not in env
+    assert env["IDF_CCACHE_ENABLE"] == "1"
+    assert env["CCACHE_DEPEND"] == "1"
+
+
+def test_ccache_env_raises_without_build_path(tmp_path: Path) -> None:
+    # Enabled but no build_path means the IDF env was built too early -- fail
+    # loudly instead of silently dropping CCACHE_BASEDIR.
+    p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", None)
+    with (
+        patch.dict("os.environ", {}, clear=True),
+        p1,
+        p2,
+        p3,
+        pytest.raises(ValueError, match="build_path"),
+    ):
+        _ccache_env()
+
+
 # ---------------------------------------------------------------------------
 # _check_stamp / _write_idf_version_txt / _get_idf_tools_path
 # ---------------------------------------------------------------------------