[esp8266] Speed up builds with ccache when available (#17722)

esphome/components/esp8266/__init__.py

@@ -30,6 +30,7 @@ from esphome.core import (
 )
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.helpers import IS_MACOS, copy_file_if_changed
+from esphome.platformio.toolchain import copy_ccache_script
 from esphome.types import ConfigType
 
 from .boards import BOARDS, ESP8266_LD_SCRIPTS
@@ -294,6 +295,7 @@ async def to_code(config):
         )
 
     extra_scripts = [
+        "pre:ccache.py",
         "pre:testing_mode.py",
         "pre:exclude_updater.py",
         "pre:exclude_waveform.py",
@@ -443,31 +445,18 @@ async def finalize_serial_config() -> None:
 # Called by writer.py
 def copy_files() -> None:
     dir = Path(__file__).parent
-    post_build_file = dir / "post_build.py.script"
-    copy_file_if_changed(
-        post_build_file,
-        CORE.relative_build_path("post_build.py"),
-    )
-    testing_mode_file = dir / "testing_mode.py.script"
-    copy_file_if_changed(
-        testing_mode_file,
-        CORE.relative_build_path("testing_mode.py"),
-    )
-    exclude_updater_file = dir / "exclude_updater.py.script"
-    copy_file_if_changed(
-        exclude_updater_file,
-        CORE.relative_build_path("exclude_updater.py"),
-    )
-    exclude_waveform_file = dir / "exclude_waveform.py.script"
-    copy_file_if_changed(
-        exclude_waveform_file,
-        CORE.relative_build_path("exclude_waveform.py"),
-    )
-    remove_float_scanf_file = dir / "remove_float_scanf.py.script"
-    copy_file_if_changed(
-        remove_float_scanf_file,
-        CORE.relative_build_path("remove_float_scanf.py"),
-    )
+    for script in (
+        "post_build",
+        "testing_mode",
+        "exclude_updater",
+        "exclude_waveform",
+        "remove_float_scanf",
+    ):
+        copy_file_if_changed(
+            dir / f"{script}.py.script",
+            CORE.relative_build_path(f"{script}.py"),
+        )
+    copy_ccache_script()
 
 
 # ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder

esphome/platformio/ccache.py.script

@@ -0,0 +1,34 @@
+import os
+import shutil
+
+# pylint: disable=E0602
+Import("env")  # noqa
+
+# ESPHome decides whether ccache is used and exports the CCACHE_* settings
+# into the environment before PlatformIO starts (_ccache_env() in
+# esphome/platformio/toolchain.py); this script only supplies the SCons-level
+# mechanism.
+#
+# This is a "pre" script, so the platform's builder (which sets CC/CXX and
+# clones the construction environment for framework and library builds) runs
+# after it. Replacing CC/CXX here would be overwritten, and replacing them in
+# a "post" script would miss the already-cloned library environments. Wrapping
+# SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler
+# invocation from every environment funnels through it at execution time.
+if (
+    os.environ.get("ESPHOME_CCACHE_ENABLE") == "1"
+    and (ccache_path := shutil.which("ccache")) is not None
+):
+    original_spawn = env["SPAWN"]
+
+    def ccache_spawn(sh, escape, cmd, args, child_env):
+        # Only wrap compile steps (gcc/g++ with -c); linking, archiving and
+        # the other tools gain nothing from ccache.
+        prog = os.path.basename(cmd).removesuffix(".exe")
+        if prog.endswith(("gcc", "g++")) and "-c" in args:
+            cmd = ccache_path
+            args = [escape(ccache_path), *args]
+        return original_spawn(sh, escape, cmd, args, child_env)
+
+    env.Replace(SPAWN=ccache_spawn)
+    print("ESPHome: Compiling with ccache")

esphome/platformio/toolchain.py

@@ -4,12 +4,21 @@ import logging
 import os
 from pathlib import Path
 import re
+import shutil
 import sys
 from typing import TYPE_CHECKING
 
+import platformdirs
+
 from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE
 from esphome.core import CORE, EsphomeError
-from esphome.helpers import add_git_ceiling_directory, rmtree, write_file
+from esphome.helpers import (
+    add_git_ceiling_directory,
+    copy_file_if_changed,
+    get_bool_env,
+    rmtree,
+    write_file,
+)
 from esphome.util import FlashImage, run_external_process
 
 if TYPE_CHECKING:
@@ -225,6 +234,77 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None:
         _write_pio_stamp_python(stamp_file, current)
 
 
+def _ccache_env() -> dict[str, str]:
+    """Return ccache settings for PlatformIO builds.
+
+    Enabled by default whenever the ``ccache`` binary is on PATH; set
+    ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to
+    force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE``
+    so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script,
+    which wraps compiler invocations inside SCons) only have to check for
+    ``"1"`` instead of re-implementing the policy.
+
+    The returned values are merged into the environment of the PlatformIO
+    subprocess only, never into ``os.environ``: a long-running process
+    (e.g. the dashboard) also runs ESP-IDF builds, whose own ccache setup
+    skips defaults for ``CCACHE_*`` keys it finds already set, so leaking
+    these values would hand it the wrong cache dir and a stale basedir.
+
+    This mirrors ``_ccache_env()`` in ``esphome/espidf/framework.py``. The
+    cache lives under the machine-global ESPHome cache dir, so it is shared
+    across all projects and removed by ``esphome clean-all``. Unlike the
+    ESP-IDF path, ``CCACHE_DEPEND`` is not set: SCons compiles don't emit
+    the depfiles depend mode needs, so ccache's default preprocessor mode
+    is used.
+
+    ``CCACHE_BASEDIR`` rewrites the per-device absolute paths (the generated
+    sources under src/, the .pioenvs build dir) so different devices with
+    identical source share cache entries; it is always set to the current
+    build dir. The other ``CCACHE_*`` values the user already set in the
+    environment are respected.
+    """
+    if "ESPHOME_CCACHE_ENABLE" in os.environ:
+        enabled = get_bool_env("ESPHOME_CCACHE_ENABLE")
+    else:
+        enabled = shutil.which("ccache") is not None
+    env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"}
+    if not enabled:
+        return env
+    # build_path is set during preload for every config-loading command, so it
+    # being unset means a caller built the environment too early; fail loudly
+    # rather than with an opaque TypeError from Path(None).
+    if CORE.build_path is None:
+        raise ValueError(
+            "CORE.build_path must be set before constructing the PlatformIO "
+            "build environment"
+        )
+    env["CCACHE_BASEDIR"] = str(Path(CORE.build_path).resolve())
+    defaults = {
+        "CCACHE_DIR": str(
+            Path(platformdirs.user_cache_dir("esphome", appauthor=False))
+            / "platformio-ccache"
+        ),
+        "CCACHE_NOHASHDIR": "true",
+    }
+    env.update({k: v for k, v in defaults.items() if k not in os.environ})
+    return env
+
+
+def copy_ccache_script() -> None:
+    """Copy the shared ccache SCons pre-script into the build dir.
+
+    Platform components call this from their ``copy_files()`` and add
+    ``pre:ccache.py`` to their ``extra_scripts``. The script wraps compiler
+    invocations inside SCons with ccache; it is platform-agnostic, so it
+    lives here next to ``_ccache_env()`` rather than being duplicated per
+    component.
+    """
+    copy_file_if_changed(
+        Path(__file__).parent / "ccache.py.script",
+        CORE.relative_build_path("ccache.py"),
+    )
+
+
 def run_platformio_cli(*args, **kwargs) -> str | int:
     # Re-provision the PlatformIO cache if the interpreter's major.minor changed
     # since it was last built; a stale platform otherwise rejects the new Python
@@ -256,7 +336,14 @@ def run_platformio_cli(*args, **kwargs) -> str | int:
         os.environ["PYTHONEXEPATH"] = python_exe
     cmd = [python_exe, "-m", "esphome.platformio.runner"] + list(args)
 
-    return run_external_process(*cmd, **kwargs)
+    # ccache settings go into the subprocess environment only (see
+    # _ccache_env() for why they must not leak into os.environ). A caller
+    # supplied env is used as the base when present.
+    base_env = kwargs.pop("env", None)
+    env = dict(os.environ if base_env is None else base_env)
+    env.update(_ccache_env())
+
+    return run_external_process(*cmd, env=env, **kwargs)
 
 
 def run_platformio_cli_run(config, verbose, *args, **kwargs) -> str | int:

esphome/util.py

@@ -314,6 +314,7 @@ def run_external_process(*cmd: str, **kwargs: Any) -> int | str:
             encoding="utf-8",
             check=False,
             close_fds=False,
+            env=kwargs.get("env"),
         )
         return proc.stdout if capture_stdout else proc.returncode
     except KeyboardInterrupt:  # pylint: disable=try-except-raise

tests/unit_tests/test_platformio_toolchain.py

@@ -322,6 +322,149 @@ def test_run_platformio_cli_sets_environment_variables(
         assert "arg" in args
 
 
+def test_ccache_env_enabled_by_default(setup_core: Path) -> None:
+    """Ccache is enabled when the binary is on PATH and no override is set."""
+    CORE.build_path = setup_core / "build" / "test"
+
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
+    ):
+        env = toolchain._ccache_env()
+
+    assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+    assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve())
+    assert env["CCACHE_DIR"].endswith("platformio-ccache")
+    assert env["CCACHE_NOHASHDIR"] == "true"
+    # Nothing may leak into os.environ: a later ESP-IDF build in the same
+    # process would otherwise skip its own ccache defaults.
+    assert "CCACHE_BASEDIR" not in os.environ
+    assert "ESPHOME_CCACHE_ENABLE" not in os.environ
+
+
+def test_ccache_env_disabled_without_binary(setup_core: Path) -> None:
+    """Ccache stays off when the binary is not on PATH."""
+    CORE.build_path = setup_core / "build" / "test"
+
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value=None),
+    ):
+        env = toolchain._ccache_env()
+
+    assert env == {"ESPHOME_CCACHE_ENABLE": "0"}
+
+
+def test_ccache_env_opt_out(setup_core: Path) -> None:
+    """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present."""
+    CORE.build_path = setup_core / "build" / "test"
+
+    with (
+        patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
+    ):
+        env = toolchain._ccache_env()
+
+    assert env == {"ESPHOME_CCACHE_ENABLE": "0"}
+
+
+def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None:
+    """A truthy override value is normalized to "1" for the build scripts."""
+    CORE.build_path = setup_core / "build" / "test"
+
+    with (
+        patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value=None),
+    ):
+        env = toolchain._ccache_env()
+
+    assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+
+
+def test_ccache_env_respects_user_values_and_refreshes_basedir(
+    setup_core: Path,
+) -> None:
+    """User CCACHE_* values win, but CCACHE_BASEDIR follows the build dir."""
+    user_env = {
+        "CCACHE_DIR": "/custom/cache",
+        "CCACHE_BASEDIR": "/stale/other-device",
+    }
+    CORE.build_path = setup_core / "build" / "test"
+
+    with (
+        patch.dict(os.environ, user_env, clear=True),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
+    ):
+        env = toolchain._ccache_env()
+
+    # CCACHE_DIR is not returned, so the user's os.environ value applies in
+    # the subprocess; CCACHE_BASEDIR is always refreshed to the build dir.
+    assert "CCACHE_DIR" not in env
+    assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve())
+
+
+def test_run_platformio_cli_passes_ccache_env_to_subprocess_only(
+    setup_core: Path, mock_run_external_process: Mock
+) -> None:
+    """The ccache settings reach the subprocess env without touching os.environ."""
+    CORE.build_path = str(setup_core / "build" / "test")
+
+    with (
+        patch.dict(os.environ, {}, clear=False),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
+    ):
+        os.environ.pop("ESPHOME_CCACHE_ENABLE", None)
+        mock_run_external_process.return_value = 0
+        toolchain.run_platformio_cli("test", "arg")
+
+        env = mock_run_external_process.call_args[1]["env"]
+        assert env["ESPHOME_CCACHE_ENABLE"] == "1"
+        assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve())
+        assert "ESPHOME_CCACHE_ENABLE" not in os.environ
+        assert "CCACHE_BASEDIR" not in os.environ
+
+
+def test_ccache_env_requires_build_path(setup_core: Path) -> None:
+    """Enabling ccache without a build path fails loudly."""
+    CORE.build_path = None
+
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"),
+        pytest.raises(ValueError, match="CORE.build_path must be set"),
+    ):
+        toolchain._ccache_env()
+
+
+def test_run_platformio_cli_merges_caller_env(
+    setup_core: Path, mock_run_external_process: Mock
+) -> None:
+    """A caller-supplied env is the base and gains the ccache settings."""
+    CORE.build_path = str(setup_core / "build" / "test")
+
+    with patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"):
+        mock_run_external_process.return_value = 0
+        toolchain.run_platformio_cli(
+            "test", env={"CUSTOM_VAR": "1", "ESPHOME_CCACHE_ENABLE": "0"}
+        )
+
+    env = mock_run_external_process.call_args[1]["env"]
+    assert env["CUSTOM_VAR"] == "1"
+    # The normalized enable flag still lands in the subprocess env.
+    assert "ESPHOME_CCACHE_ENABLE" in env
+
+
+def test_copy_ccache_script(setup_core: Path) -> None:
+    """The shared ccache pre-script is copied into the build dir."""
+    CORE.build_path = setup_core / "build" / "test"
+
+    toolchain.copy_ccache_script()
+
+    dest = setup_core / "build" / "test" / "ccache.py"
+    source = Path(toolchain.__file__).parent / "ccache.py.script"
+    assert dest.read_text() == source.read_text()
+
+
 @pytest.mark.parametrize(
     ("platform", "input_path", "expected"),
     [
@@ -375,7 +518,10 @@ def test_run_platformio_cli_strips_win_long_path_prefix(
     )
 
     with (
-        patch.dict(os.environ, {}, clear=False),
+        # Pin ccache off: patching sys.platform to win32 (sys is a singleton,
+        # so the stdlib sees it too) would send shutil.which down the Windows
+        # code path, which crashes on a POSIX host.
+        patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=False),
         patch("esphome.platformio.toolchain.sys.platform", "win32"),
         patch("esphome.platformio.toolchain.sys.executable", prefixed_exe),
     ):