From c27f97b8f91565227e88e84d2bdfa53e0d7127f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 12:51:32 -0500 Subject: [PATCH] [esp32] Precompile the shared core headers for src compiles --- esphome/build_gen/espidf.py | 152 ++++++++++++++++++ esphome/build_helpers/idedata.py | 14 +- esphome/espidf/framework.py | 42 +++-- esphome/espidf/toolchain.py | 5 + tests/unit_tests/build_gen/test_espidf.py | 98 +++++++++++ .../unit_tests/build_helpers/test_idedata.py | 10 +- tests/unit_tests/test_espidf_toolchain.py | 7 + 7 files changed, 299 insertions(+), 29 deletions(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 2ef89cf595..1c21d138e7 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -3,7 +3,16 @@ import json import logging from pathlib import Path +import subprocess +from esphome.build_helpers.idedata import is_launcher, split_command +from esphome.build_helpers.pch import ( + PCH_CORE_HEADER, + PCH_HEADER_NAME, + pch_checksum, + pch_enabled, + pch_header_text, +) from esphome.components.esp32 import ( get_esp32_variant, get_excluded_builtin_components, @@ -22,6 +31,24 @@ from esphome.helpers import mkdir_p, write_file_if_changed _LOGGER = logging.getLogger(__name__) +# Prefix-header contents, defines.h first so USE_* macros exist for the +# rest. Deliberately hard-coded: frequency-derived sets measured no better +# and kept selecting headers that cannot compile standalone (X-macro, +# platform-variant). Every entry must be safe to include first in an +# empty TU. +_PCH_HEADERS = ( + PCH_CORE_HEADER, + "esphome/core/component.h", + "esphome/core/helpers.h", + "esphome/core/log.h", + "esphome/core/application.h", + "esphome/core/automation.h", +) + +# Header and .gch/.sum sidecars, relative to the device dir; see +# _pch_cmake() and prepare_pch() for the layout rationale +_PCH_BUILD_HEADER = f"build/{PCH_HEADER_NAME}" + # Replaces the IDF default C++ standard (-std=gnu++2b appended to # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(), @@ -279,9 +306,129 @@ idf_component_register( target_link_options(${{COMPONENT_LIB}} PUBLIC {link_opts_str} ) +{_pch_cmake()}""" + + +def _pch_cmake() -> str: + """The src component's precompiled-header block (C++ TUs only). + + The -include stays relative (resolved from the compilers' cwd, the + build dir, where prepare_pch() puts the header and .gch); an absolute + path would poison ccache keys with the per-device build path. + """ + if not pch_enabled(): + return "" + return f""" +# ESPHome precompiled header (see esphome/build_helpers/pch.py) +target_compile_options(${{COMPONENT_LIB}} PRIVATE + "$<$:-include>" + "$<$:{PCH_HEADER_NAME}>" +) """ +def _pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None: + """The exact src C++ flags from compile_commands.json, retargeted at + the header; None when no configured C++ TU is available yet.""" + try: + entries = json.loads( + (build_dir / "compile_commands.json").read_text(encoding="utf-8") + ) + except (OSError, json.JSONDecodeError): + return None + entry = next( + ( + e + for e in entries + if "__idf_src.dir" in e.get("command", "") + and e.get("file", "").endswith((".cpp", ".cc", ".cxx")) + ), + None, + ) + if entry is None: + return None + tokens = split_command(entry["command"]) + # A DB recorded with ccache enabled prefixes the compiler with the + # launcher; the .gch must be compiled directly + if tokens and is_launcher(tokens[0]): + tokens = tokens[1:] + args: list[str] = [] + arg_it = iter(tokens) + for tok in arg_it: + # Strip the source/output and any -include (the header holds it) + if tok in ("-include", "-o", "-c"): + next(arg_it, None) + continue + args.append(tok) + return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)] + + +def prepare_pch() -> None: + """Compile the prefix header's .gch and write its ccache .sum. + + Runs right before ninja, after every reconfigure, so the flags in + compile_commands.json and the sdkconfig are the settled ones. The .sum + doubles as the freshness stamp; a failed .gch compile falls back to + the plain header include. + """ + if not pch_enabled(): + return + build_dir = CORE.relative_build_path("build") + header = CORE.relative_build_path(_PCH_BUILD_HEADER) + gch = Path(f"{header}.gch") + sum_path = Path(f"{gch}.sum") + try: + sdkconfig = CORE.relative_build_path(f"sdkconfig.{CORE.name}").read_text( + encoding="utf-8" + ) + except OSError: + sdkconfig = "no-sdkconfig" + checksum = pch_checksum( + CORE.relative_src_path(), + _PCH_HEADERS, + ( + str(idf_version()), + CORE.cpp_standard or "", + sdkconfig, + *get_project_compile_flags(), + *get_project_cxx_compile_flags(), + ), + ) + if ( + gch.is_file() + and sum_path.is_file() + and sum_path.read_text(encoding="utf-8").strip() == checksum + ): + return + failed_marker = Path(f"{gch}.failed") + if ( + failed_marker.is_file() + and failed_marker.read_text(encoding="utf-8").strip() == checksum + ): + return + cmd = _pch_compile_command(build_dir, header, gch) + if cmd is None: + return + try: + result = subprocess.run( + cmd, cwd=build_dir, capture_output=True, text=True, check=False + ) + error = result.stderr.strip() if result.returncode != 0 else None + except OSError as err: + error = str(err) + if error is not None: + _LOGGER.warning( + "Precompiled header failed; compiling without it: %s", error[:400] + ) + gch.unlink(missing_ok=True) + sum_path.unlink(missing_ok=True) + # Skip retries until a flag/header/sdkconfig change alters the checksum + failed_marker.write_text(checksum + "\n", encoding="utf-8") + return + failed_marker.unlink(missing_ok=True) + sum_path.write_text(checksum + "\n", encoding="utf-8") + + def write_project( minimal: bool = False, builtin_components: list[str] | None = None ) -> None: @@ -301,6 +448,11 @@ def write_project( get_component_cmakelists(), ) + if pch_enabled(): + write_file_if_changed( + CORE.relative_build_path(_PCH_BUILD_HEADER), pch_header_text(_PCH_HEADERS) + ) + # Snapshot the exclusion set so has_outdated_files() can trigger a # discovery reconfigure when it changes. Excluded components never # register in project_description.json, so re-including one (e.g. a diff --git a/esphome/build_helpers/idedata.py b/esphome/build_helpers/idedata.py index 4b57b5e983..8c5aeffdac 100644 --- a/esphome/build_helpers/idedata.py +++ b/esphome/build_helpers/idedata.py @@ -76,7 +76,7 @@ def _is_esphome_src(file: str) -> bool: ) -def _split_command(command: str) -> list[str]: +def split_command(command: str) -> list[str]: r"""Tokenize a compile_commands.json / response-file command string. On Windows, tokenize per Windows ``argv`` rules via ``CommandLineToArgvW``. @@ -128,7 +128,7 @@ def _expand_response_files(tokens: list[str], directory: Path) -> list[str]: try: out.extend( _expand_response_files( - _split_command(rf.read_text(encoding="utf-8")), directory + split_command(rf.read_text(encoding="utf-8")), directory ) ) continue @@ -157,7 +157,7 @@ def _pick_entry(entries: list[dict]) -> dict: _LAUNCHER_STEMS = frozenset({"ccache", "sccache", "distcc", "icecc", "buildcache"}) -def _is_launcher(token: str) -> bool: +def is_launcher(token: str) -> bool: return Path(token).stem.lower() in _LAUNCHER_STEMS @@ -166,7 +166,7 @@ def parse_entry( ) -> tuple[str, list[str], list[str], list[str]]: """Parse one compile_commands entry -> (cxx_path, defines, includes, cxx_flags).""" directory = Path(entry["directory"]) - tokens = _expand_response_files(_split_command(entry["command"]), directory) + tokens = _expand_response_files(split_command(entry["command"]), directory) def _include(raw: str) -> str: # Resolve against the entry's ``directory`` so cached idedata works @@ -182,7 +182,7 @@ def parse_entry( if not tokens: # An empty command, or one that was only the launcher; fail by name raise ValueError(f"empty compile command for {entry.get('file')}") - if _is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"): + if is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"): # Stale DB built with a launcher this run no longer configures; the # real compiler is the next token _LOGGER.warning("Stripping unconfigured launcher %s", tokens[0]) @@ -282,7 +282,7 @@ def _cache_usable(cached: object) -> bool: if not isinstance(cached, dict) or "cc_path" not in cached: return False cxx_path = cached.get("cxx_path") - if not isinstance(cxx_path, str) or _is_launcher(cxx_path): + if not isinstance(cxx_path, str) or is_launcher(cxx_path): return False includes = cached.get("includes") return isinstance(includes, dict) and isinstance(includes.get("build"), list) @@ -330,7 +330,7 @@ def load_or_build_idedata( def reject_launcher_compiler(cxx_path: str) -> None: """Reject a compile DB naming a launcher (ccache) as the compiler; it must never be probed, cached, or consumed.""" - if _is_launcher(cxx_path): + if is_launcher(cxx_path): raise EsphomeError( f"compile_commands.json names the launcher {cxx_path} as the " "compiler; the compile database is unusable" diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 82445fbacf..55a493def8 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -17,6 +17,7 @@ from esphome.build_helpers.ccache import ( parse_enable_env, resolve_ccache_path, ) +from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path from esphome.core import Version from esphome.framework_helpers import ( @@ -1200,14 +1201,30 @@ def _ccache_env() -> dict[str, str]: Only values the user has not already set in the environment are returned, so a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected. """ - # IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared - # ESPHOME_CCACHE_ENABLE. + if not _ccache_enabled(): + # The raw knob value (e.g. "disable") is still inherited by idf.py + # via os.environ, where a non-false-constant string reads as + # truthy; export the canonical off spelling instead + return {"IDF_CCACHE_ENABLE": "0"} + + env = ccache_defaults_env(get_idf_tools_path() / "ccache") + env.update(ccache_pch_env()) + # Exactly one canonical spelling ever reaches idf.py, whatever the + # accepted input spelling was ("enable", "yes", ...) + env["IDF_CCACHE_ENABLE"] = "1" + return env + + +def _ccache_enabled() -> bool: + """Whether ESP-IDF compiles run under ccache. + + IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared + ESPHOME_CCACHE_ENABLE; when unset, enabled iff a runnable binary is + on PATH. + """ idf_knob = parse_enable_env("IDF_CCACHE_ENABLE") if idf_knob is False: - # The raw value (e.g. "disable") is still inherited by idf.py via - # os.environ, where a non-false-constant string reads as truthy; - # export the canonical off spelling instead - return {"IDF_CCACHE_ENABLE": "0"} + return False if idf_knob is True: # Forced on ignores the runnability verdict, but the outcome is # worth saying out loud. Only the truly-missing case means idf.py @@ -1226,17 +1243,8 @@ def _ccache_env() -> dict[str, str]: "was rejected here (probe failure or " "ESPHOME_CCACHE_ENABLE=0); idf.py will use it anyway" ) - elif resolve_ccache_path() is None: - # ESP-IDF silently skips ccache without the binary; export the - # canonical off spelling so an unparsable inherited value (or a - # probe-rejected ccache idf.py would still find) cannot enable it - return {"IDF_CCACHE_ENABLE": "0"} - - env = ccache_defaults_env(get_idf_tools_path() / "ccache") - # Exactly one canonical spelling ever reaches idf.py, whatever the - # accepted input spelling was ("enable", "yes", ...) - env["IDF_CCACHE_ENABLE"] = "1" - return env + return True + return resolve_ccache_path() is not None def get_framework_env( diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 986f9dfb8b..a5d145e3c1 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -528,6 +528,11 @@ def run_compile(config, verbose: bool) -> int: return result.returncode _patch_memory_segments() + # After every reconfigure so compile_commands and sdkconfig are settled + from esphome.build_gen.espidf import prepare_pch + + prepare_pch() + # Build args = [] diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 079f10ddb9..fd97b9b074 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import logging from pathlib import Path +import subprocess from unittest.mock import patch import pytest @@ -488,3 +489,100 @@ def test_get_component_cmakelists_no_compile_features() -> None: content = get_component_cmakelists() assert "target_compile_features" not in content + + +def _make_pch_device(tmp_path: Path, name: str) -> Path: + """A device dir with the pch source headers and a stub compile_commands.""" + from esphome.build_gen.espidf import _PCH_HEADERS + + dev = tmp_path / name + for header in _PCH_HEADERS: + path = dev / "src" / header + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + build = dev / "build" + build.mkdir(exist_ok=True) + from esphome.build_helpers.pch import pch_header_text + + (build / "esphome_pch.h").write_text(pch_header_text(_PCH_HEADERS)) + (build / "compile_commands.json").write_text( + json.dumps( + [ + { + "directory": str(build), + "command": ( + "g++ -DX=1 -include esphome_pch.h " + "-o esp-idf/src/CMakeFiles/__idf_src.dir/a.cpp.obj " + f"-c {dev}/src/a.cpp" + ), + "file": f"{dev}/src/a.cpp", + } + ] + ) + ) + return dev + + +def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None: + from esphome.build_gen.espidf import prepare_pch + + dev = _make_pch_device(tmp_path, "dev_a") + CORE.build_path = dev + gch = dev / "build" / "esphome_pch.h.gch" + + def fake_compile(cmd, **kwargs): + # The compile must target the header, not the stub TU + assert cmd[-5:-3] == ["c++-header", "-c"] + gch.write_bytes(b"gch") + return subprocess.CompletedProcess(cmd, 0, "", "") + + with ( + patch.object(CORE, "name", "test"), + patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + ): + prepare_pch() + assert (dev / "build" / "esphome_pch.h").read_text().startswith("#include") + checksum = (dev / "build" / "esphome_pch.h.gch.sum").read_text().strip() + assert len(checksum) == 64 + # Unchanged inputs: the second call must not recompile + with ( + patch.object(CORE, "name", "test"), + patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError), + ): + prepare_pch() + + +def test_pch_no_device_path_poison(tmp_path: Path) -> None: + """Regression: neither the injected -include nor the .sum may carry the + per-device build path, or cross-device ccache sharing breaks.""" + from esphome.build_gen.espidf import get_component_cmakelists, prepare_pch + + sums = [] + for name in ("dev_a", "dev_b"): + dev = _make_pch_device(tmp_path, name) + CORE.build_path = dev + gch = dev / "build" / "esphome_pch.h.gch" + + def fake_compile(cmd, _gch=gch, **kwargs): + _gch.write_bytes(b"gch") + return subprocess.CompletedProcess(cmd, 0, "", "") + + with ( + patch.object(CORE, "name", name), + patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + ): + prepare_pch() + content = get_component_cmakelists() + assert str(dev) not in content + sums.append((dev / "build" / "esphome_pch.h.gch.sum").read_text()) + assert sums[0] == sums[1] + + +def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None: + from esphome.build_gen.espidf import get_component_cmakelists + + content = get_component_cmakelists() + assert '"$<$:-include>"' in content + assert '"$<$:esphome_pch.h>"' in content + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + assert "-include" not in get_component_cmakelists() diff --git a/tests/unit_tests/build_helpers/test_idedata.py b/tests/unit_tests/build_helpers/test_idedata.py index fcf9c67086..615d3dfe37 100644 --- a/tests/unit_tests/build_helpers/test_idedata.py +++ b/tests/unit_tests/build_helpers/test_idedata.py @@ -327,7 +327,7 @@ def test_split_command_preserves_paths_and_unescapes_quotes() -> None: r"""Backslash paths survive while ``\"`` define-quoting is unescaped.""" command = r"C:\esp\bin\riscv32-esp-elf-g++.exe -DVER=\"1.2.3\" -IC:/inc/a -c x.cpp" - tokens = idedata._split_command(command) + tokens = idedata.split_command(command) assert tokens[0] == r"C:\esp\bin\riscv32-esp-elf-g++.exe" assert '-DVER="1.2.3"' in tokens @@ -341,8 +341,8 @@ def test_split_command_empty_returns_empty() -> None: Guards against ``CommandLineToArgvW("")`` returning the current process name instead of an empty list. """ - assert idedata._split_command("") == [] - assert idedata._split_command(" ") == [] + assert idedata.split_command("") == [] + assert idedata.split_command(" ") == [] @pytest.mark.skipif(os.name != "nt", reason="Windows argv tokenization") @@ -503,9 +503,9 @@ def test_load_or_build_idedata_rebuilds_non_dict_cache(tmp_path: Path) -> None: def test_is_launcher_matches_only_known_launchers() -> None: """Compilers of any shape pass; only the closed launcher set matches.""" for token in ("/t/g++-13", "gcc-8.4.0", "clang++-17", "armcc", "icx", "cc"): - assert not idedata._is_launcher(token) + assert not idedata.is_launcher(token) for token in ("/opt/homebrew/bin/ccache", "CCACHE.EXE", "distcc", "sccache"): - assert idedata._is_launcher(token) + assert idedata.is_launcher(token) def test_load_or_build_idedata_corrupted_cache_is_logged( diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 9deb27d83c..2cd5d4e1cb 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -93,6 +93,13 @@ def test_get_configured_targets_ci_installs_all(monkeypatch: pytest.MonkeyPatch) assert toolchain._get_configured_targets() is None +@pytest.fixture(autouse=True) +def _no_ccache(monkeypatch: pytest.MonkeyPatch) -> None: + """Deterministic run_compile: no host ccache probe, no pch work.""" + monkeypatch.setenv("IDF_CCACHE_ENABLE", "0") + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + + def _setup_build(setup_core: Path) -> tuple[Path, Path]: """Point CORE at a build dir; return (compile_commands, idedata cache) paths.""" CORE.name = "test"