diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 7ce10d465d..618bf775a0 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -30,6 +30,7 @@ from esphome.core import ( ) from esphome.core.config import BOARD_MAX_LENGTH from esphome.helpers import IS_MACOS, copy_file_if_changed +from esphome.platformio.toolchain import copy_ccache_script from esphome.types import ConfigType from .boards import BOARDS, ESP8266_LD_SCRIPTS @@ -294,6 +295,7 @@ async def to_code(config): ) extra_scripts = [ + "pre:ccache.py", "pre:testing_mode.py", "pre:exclude_updater.py", "pre:exclude_waveform.py", @@ -443,31 +445,18 @@ async def finalize_serial_config() -> None: # Called by writer.py def copy_files() -> None: dir = Path(__file__).parent - post_build_file = dir / "post_build.py.script" - copy_file_if_changed( - post_build_file, - CORE.relative_build_path("post_build.py"), - ) - testing_mode_file = dir / "testing_mode.py.script" - copy_file_if_changed( - testing_mode_file, - CORE.relative_build_path("testing_mode.py"), - ) - exclude_updater_file = dir / "exclude_updater.py.script" - copy_file_if_changed( - exclude_updater_file, - CORE.relative_build_path("exclude_updater.py"), - ) - exclude_waveform_file = dir / "exclude_waveform.py.script" - copy_file_if_changed( - exclude_waveform_file, - CORE.relative_build_path("exclude_waveform.py"), - ) - remove_float_scanf_file = dir / "remove_float_scanf.py.script" - copy_file_if_changed( - remove_float_scanf_file, - CORE.relative_build_path("remove_float_scanf.py"), - ) + for script in ( + "post_build", + "testing_mode", + "exclude_updater", + "exclude_waveform", + "remove_float_scanf", + ): + copy_file_if_changed( + dir / f"{script}.py.script", + CORE.relative_build_path(f"{script}.py"), + ) + copy_ccache_script() # ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder diff --git a/esphome/platformio/ccache.py.script b/esphome/platformio/ccache.py.script new file mode 100644 index 0000000000..cc08a8c044 --- /dev/null +++ b/esphome/platformio/ccache.py.script @@ -0,0 +1,34 @@ +import os +import shutil + +# pylint: disable=E0602 +Import("env") # noqa + +# ESPHome decides whether ccache is used and exports the CCACHE_* settings +# into the environment before PlatformIO starts (_ccache_env() in +# esphome/platformio/toolchain.py); this script only supplies the SCons-level +# mechanism. +# +# This is a "pre" script, so the platform's builder (which sets CC/CXX and +# clones the construction environment for framework and library builds) runs +# after it. Replacing CC/CXX here would be overwritten, and replacing them in +# a "post" script would miss the already-cloned library environments. Wrapping +# SPAWN instead is ordering-proof: clones copy the wrapper, and every compiler +# invocation from every environment funnels through it at execution time. +if ( + os.environ.get("ESPHOME_CCACHE_ENABLE") == "1" + and (ccache_path := shutil.which("ccache")) is not None +): + original_spawn = env["SPAWN"] + + def ccache_spawn(sh, escape, cmd, args, child_env): + # Only wrap compile steps (gcc/g++ with -c); linking, archiving and + # the other tools gain nothing from ccache. + prog = os.path.basename(cmd).removesuffix(".exe") + if prog.endswith(("gcc", "g++")) and "-c" in args: + cmd = ccache_path + args = [escape(ccache_path), *args] + return original_spawn(sh, escape, cmd, args, child_env) + + env.Replace(SPAWN=ccache_spawn) + print("ESPHome: Compiling with ccache") diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 105d4a8283..0959cbfffb 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -4,12 +4,21 @@ import logging import os from pathlib import Path import re +import shutil import sys from typing import TYPE_CHECKING +import platformdirs + from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError -from esphome.helpers import add_git_ceiling_directory, rmtree, write_file +from esphome.helpers import ( + add_git_ceiling_directory, + copy_file_if_changed, + get_bool_env, + rmtree, + write_file, +) from esphome.util import FlashImage, run_external_process if TYPE_CHECKING: @@ -225,6 +234,77 @@ def _check_platformio_python_stamp(config: "ProjectConfig") -> None: _write_pio_stamp_python(stamp_file, current) +def _ccache_env() -> dict[str, str]: + """Return ccache settings for PlatformIO builds. + + Enabled by default whenever the ``ccache`` binary is on PATH; set + ``ESPHOME_CCACHE_ENABLE=0`` in the environment to opt out (or ``1`` to + force it on). The decision is normalized into ``ESPHOME_CCACHE_ENABLE`` + so platform build scripts (e.g. the esp8266 ``ccache.py`` extra script, + which wraps compiler invocations inside SCons) only have to check for + ``"1"`` instead of re-implementing the policy. + + The returned values are merged into the environment of the PlatformIO + subprocess only, never into ``os.environ``: a long-running process + (e.g. the dashboard) also runs ESP-IDF builds, whose own ccache setup + skips defaults for ``CCACHE_*`` keys it finds already set, so leaking + these values would hand it the wrong cache dir and a stale basedir. + + This mirrors ``_ccache_env()`` in ``esphome/espidf/framework.py``. The + cache lives under the machine-global ESPHome cache dir, so it is shared + across all projects and removed by ``esphome clean-all``. Unlike the + ESP-IDF path, ``CCACHE_DEPEND`` is not set: SCons compiles don't emit + the depfiles depend mode needs, so ccache's default preprocessor mode + is used. + + ``CCACHE_BASEDIR`` rewrites the per-device absolute paths (the generated + sources under src/, the .pioenvs build dir) so different devices with + identical source share cache entries; it is always set to the current + build dir. The other ``CCACHE_*`` values the user already set in the + environment are respected. + """ + if "ESPHOME_CCACHE_ENABLE" in os.environ: + enabled = get_bool_env("ESPHOME_CCACHE_ENABLE") + else: + enabled = shutil.which("ccache") is not None + env = {"ESPHOME_CCACHE_ENABLE": "1" if enabled else "0"} + if not enabled: + return env + # build_path is set during preload for every config-loading command, so it + # being unset means a caller built the environment too early; fail loudly + # rather than with an opaque TypeError from Path(None). + if CORE.build_path is None: + raise ValueError( + "CORE.build_path must be set before constructing the PlatformIO " + "build environment" + ) + env["CCACHE_BASEDIR"] = str(Path(CORE.build_path).resolve()) + defaults = { + "CCACHE_DIR": str( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) + / "platformio-ccache" + ), + "CCACHE_NOHASHDIR": "true", + } + env.update({k: v for k, v in defaults.items() if k not in os.environ}) + return env + + +def copy_ccache_script() -> None: + """Copy the shared ccache SCons pre-script into the build dir. + + Platform components call this from their ``copy_files()`` and add + ``pre:ccache.py`` to their ``extra_scripts``. The script wraps compiler + invocations inside SCons with ccache; it is platform-agnostic, so it + lives here next to ``_ccache_env()`` rather than being duplicated per + component. + """ + copy_file_if_changed( + Path(__file__).parent / "ccache.py.script", + CORE.relative_build_path("ccache.py"), + ) + + def run_platformio_cli(*args, **kwargs) -> str | int: # Re-provision the PlatformIO cache if the interpreter's major.minor changed # since it was last built; a stale platform otherwise rejects the new Python @@ -256,7 +336,14 @@ def run_platformio_cli(*args, **kwargs) -> str | int: os.environ["PYTHONEXEPATH"] = python_exe cmd = [python_exe, "-m", "esphome.platformio.runner"] + list(args) - return run_external_process(*cmd, **kwargs) + # ccache settings go into the subprocess environment only (see + # _ccache_env() for why they must not leak into os.environ). A caller + # supplied env is used as the base when present. + base_env = kwargs.pop("env", None) + env = dict(os.environ if base_env is None else base_env) + env.update(_ccache_env()) + + return run_external_process(*cmd, env=env, **kwargs) def run_platformio_cli_run(config, verbose, *args, **kwargs) -> str | int: diff --git a/esphome/util.py b/esphome/util.py index b597b4b42e..f7d33bd2a9 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -314,6 +314,7 @@ def run_external_process(*cmd: str, **kwargs: Any) -> int | str: encoding="utf-8", check=False, close_fds=False, + env=kwargs.get("env"), ) return proc.stdout if capture_stdout else proc.returncode except KeyboardInterrupt: # pylint: disable=try-except-raise diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 013030d38f..723646fbd6 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -322,6 +322,149 @@ def test_run_platformio_cli_sets_environment_variables( assert "arg" in args +def test_ccache_env_enabled_by_default(setup_core: Path) -> None: + """Ccache is enabled when the binary is on PATH and no override is set.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) + assert env["CCACHE_DIR"].endswith("platformio-ccache") + assert env["CCACHE_NOHASHDIR"] == "true" + # Nothing may leak into os.environ: a later ESP-IDF build in the same + # process would otherwise skip its own ccache defaults. + assert "CCACHE_BASEDIR" not in os.environ + assert "ESPHOME_CCACHE_ENABLE" not in os.environ + + +def test_ccache_env_disabled_without_binary(setup_core: Path) -> None: + """Ccache stays off when the binary is not on PATH.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(toolchain.shutil, "which", return_value=None), + ): + env = toolchain._ccache_env() + + assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + + +def test_ccache_env_opt_out(setup_core: Path) -> None: + """ESPHOME_CCACHE_ENABLE=0 disables ccache even with the binary present.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + ): + env = toolchain._ccache_env() + + assert env == {"ESPHOME_CCACHE_ENABLE": "0"} + + +def test_ccache_env_normalizes_enable_value(setup_core: Path) -> None: + """A truthy override value is normalized to "1" for the build scripts.""" + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "yes"}, clear=True), + patch.object(toolchain.shutil, "which", return_value=None), + ): + env = toolchain._ccache_env() + + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + + +def test_ccache_env_respects_user_values_and_refreshes_basedir( + setup_core: Path, +) -> None: + """User CCACHE_* values win, but CCACHE_BASEDIR follows the build dir.""" + user_env = { + "CCACHE_DIR": "/custom/cache", + "CCACHE_BASEDIR": "/stale/other-device", + } + CORE.build_path = setup_core / "build" / "test" + + with ( + patch.dict(os.environ, user_env, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + ): + env = toolchain._ccache_env() + + # CCACHE_DIR is not returned, so the user's os.environ value applies in + # the subprocess; CCACHE_BASEDIR is always refreshed to the build dir. + assert "CCACHE_DIR" not in env + assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) + + +def test_run_platformio_cli_passes_ccache_env_to_subprocess_only( + setup_core: Path, mock_run_external_process: Mock +) -> None: + """The ccache settings reach the subprocess env without touching os.environ.""" + CORE.build_path = str(setup_core / "build" / "test") + + with ( + patch.dict(os.environ, {}, clear=False), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + ): + os.environ.pop("ESPHOME_CCACHE_ENABLE", None) + mock_run_external_process.return_value = 0 + toolchain.run_platformio_cli("test", "arg") + + env = mock_run_external_process.call_args[1]["env"] + assert env["ESPHOME_CCACHE_ENABLE"] == "1" + assert env["CCACHE_BASEDIR"] == str((setup_core / "build" / "test").resolve()) + assert "ESPHOME_CCACHE_ENABLE" not in os.environ + assert "CCACHE_BASEDIR" not in os.environ + + +def test_ccache_env_requires_build_path(setup_core: Path) -> None: + """Enabling ccache without a build path fails loudly.""" + CORE.build_path = None + + with ( + patch.dict(os.environ, {}, clear=True), + patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"), + pytest.raises(ValueError, match="CORE.build_path must be set"), + ): + toolchain._ccache_env() + + +def test_run_platformio_cli_merges_caller_env( + setup_core: Path, mock_run_external_process: Mock +) -> None: + """A caller-supplied env is the base and gains the ccache settings.""" + CORE.build_path = str(setup_core / "build" / "test") + + with patch.object(toolchain.shutil, "which", return_value="/usr/bin/ccache"): + mock_run_external_process.return_value = 0 + toolchain.run_platformio_cli( + "test", env={"CUSTOM_VAR": "1", "ESPHOME_CCACHE_ENABLE": "0"} + ) + + env = mock_run_external_process.call_args[1]["env"] + assert env["CUSTOM_VAR"] == "1" + # The normalized enable flag still lands in the subprocess env. + assert "ESPHOME_CCACHE_ENABLE" in env + + +def test_copy_ccache_script(setup_core: Path) -> None: + """The shared ccache pre-script is copied into the build dir.""" + CORE.build_path = setup_core / "build" / "test" + + toolchain.copy_ccache_script() + + dest = setup_core / "build" / "test" / "ccache.py" + source = Path(toolchain.__file__).parent / "ccache.py.script" + assert dest.read_text() == source.read_text() + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ @@ -375,7 +518,10 @@ def test_run_platformio_cli_strips_win_long_path_prefix( ) with ( - patch.dict(os.environ, {}, clear=False), + # Pin ccache off: patching sys.platform to win32 (sys is a singleton, + # so the stdlib sees it too) would send shutil.which down the Windows + # code path, which crashes on a POSIX host. + patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=False), patch("esphome.platformio.toolchain.sys.platform", "win32"), patch("esphome.platformio.toolchain.sys.executable", prefixed_exe), ):