[esp32] Precompile the shared core headers for src compiles

esphome/build_gen/espidf.py

@@ -3,7 +3,16 @@
 import json
 import logging
 from pathlib import Path
+import subprocess
 
+from esphome.build_helpers.idedata import is_launcher, split_command
+from esphome.build_helpers.pch import (
+    PCH_CORE_HEADER,
+    PCH_HEADER_NAME,
+    pch_checksum,
+    pch_enabled,
+    pch_header_text,
+)
 from esphome.components.esp32 import (
     get_esp32_variant,
     get_excluded_builtin_components,
@@ -22,6 +31,24 @@ from esphome.helpers import mkdir_p, write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
 
+# Prefix-header contents, defines.h first so USE_* macros exist for the
+# rest. Deliberately hard-coded: frequency-derived sets measured no better
+# and kept selecting headers that cannot compile standalone (X-macro,
+# platform-variant). Every entry must be safe to include first in an
+# empty TU.
+_PCH_HEADERS = (
+    PCH_CORE_HEADER,
+    "esphome/core/component.h",
+    "esphome/core/helpers.h",
+    "esphome/core/log.h",
+    "esphome/core/application.h",
+    "esphome/core/automation.h",
+)
+
+# Header and .gch/.sum sidecars, relative to the device dir; see
+# _pch_cmake() and prepare_pch() for the layout rationale
+_PCH_BUILD_HEADER = f"build/{PCH_HEADER_NAME}"
+
 # Replaces the IDF default C++ standard (-std=gnu++2b appended to
 # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via
 # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(),
@@ -279,9 +306,129 @@ idf_component_register(
 target_link_options(${{COMPONENT_LIB}} PUBLIC
     {link_opts_str}
 )
+{_pch_cmake()}"""
+
+
+def _pch_cmake() -> str:
+    """The src component's precompiled-header block (C++ TUs only).
+
+    The -include stays relative (resolved from the compilers' cwd, the
+    build dir, where prepare_pch() puts the header and .gch); an absolute
+    path would poison ccache keys with the per-device build path.
+    """
+    if not pch_enabled():
+        return ""
+    return f"""
+# ESPHome precompiled header (see esphome/build_helpers/pch.py)
+target_compile_options(${{COMPONENT_LIB}} PRIVATE
+    "$<$<COMPILE_LANGUAGE:CXX>:-include>"
+    "$<$<COMPILE_LANGUAGE:CXX>:{PCH_HEADER_NAME}>"
+)
 """
 
 
+def _pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None:
+    """The exact src C++ flags from compile_commands.json, retargeted at
+    the header; None when no configured C++ TU is available yet."""
+    try:
+        entries = json.loads(
+            (build_dir / "compile_commands.json").read_text(encoding="utf-8")
+        )
+    except (OSError, json.JSONDecodeError):
+        return None
+    entry = next(
+        (
+            e
+            for e in entries
+            if "__idf_src.dir" in e.get("command", "")
+            and e.get("file", "").endswith((".cpp", ".cc", ".cxx"))
+        ),
+        None,
+    )
+    if entry is None:
+        return None
+    tokens = split_command(entry["command"])
+    # A DB recorded with ccache enabled prefixes the compiler with the
+    # launcher; the .gch must be compiled directly
+    if tokens and is_launcher(tokens[0]):
+        tokens = tokens[1:]
+    args: list[str] = []
+    arg_it = iter(tokens)
+    for tok in arg_it:
+        # Strip the source/output and any -include (the header holds it)
+        if tok in ("-include", "-o", "-c"):
+            next(arg_it, None)
+            continue
+        args.append(tok)
+    return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)]
+
+
+def prepare_pch() -> None:
+    """Compile the prefix header's .gch and write its ccache .sum.
+
+    Runs right before ninja, after every reconfigure, so the flags in
+    compile_commands.json and the sdkconfig are the settled ones. The .sum
+    doubles as the freshness stamp; a failed .gch compile falls back to
+    the plain header include.
+    """
+    if not pch_enabled():
+        return
+    build_dir = CORE.relative_build_path("build")
+    header = CORE.relative_build_path(_PCH_BUILD_HEADER)
+    gch = Path(f"{header}.gch")
+    sum_path = Path(f"{gch}.sum")
+    try:
+        sdkconfig = CORE.relative_build_path(f"sdkconfig.{CORE.name}").read_text(
+            encoding="utf-8"
+        )
+    except OSError:
+        sdkconfig = "no-sdkconfig"
+    checksum = pch_checksum(
+        CORE.relative_src_path(),
+        _PCH_HEADERS,
+        (
+            str(idf_version()),
+            CORE.cpp_standard or "",
+            sdkconfig,
+            *get_project_compile_flags(),
+            *get_project_cxx_compile_flags(),
+        ),
+    )
+    if (
+        gch.is_file()
+        and sum_path.is_file()
+        and sum_path.read_text(encoding="utf-8").strip() == checksum
+    ):
+        return
+    failed_marker = Path(f"{gch}.failed")
+    if (
+        failed_marker.is_file()
+        and failed_marker.read_text(encoding="utf-8").strip() == checksum
+    ):
+        return
+    cmd = _pch_compile_command(build_dir, header, gch)
+    if cmd is None:
+        return
+    try:
+        result = subprocess.run(
+            cmd, cwd=build_dir, capture_output=True, text=True, check=False
+        )
+        error = result.stderr.strip() if result.returncode != 0 else None
+    except OSError as err:
+        error = str(err)
+    if error is not None:
+        _LOGGER.warning(
+            "Precompiled header failed; compiling without it: %s", error[:400]
+        )
+        gch.unlink(missing_ok=True)
+        sum_path.unlink(missing_ok=True)
+        # Skip retries until a flag/header/sdkconfig change alters the checksum
+        failed_marker.write_text(checksum + "\n", encoding="utf-8")
+        return
+    failed_marker.unlink(missing_ok=True)
+    sum_path.write_text(checksum + "\n", encoding="utf-8")
+
+
 def write_project(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> None:
@@ -301,6 +448,11 @@ def write_project(
         get_component_cmakelists(),
     )
 
+    if pch_enabled():
+        write_file_if_changed(
+            CORE.relative_build_path(_PCH_BUILD_HEADER), pch_header_text(_PCH_HEADERS)
+        )
+
     # Snapshot the exclusion set so has_outdated_files() can trigger a
     # discovery reconfigure when it changes. Excluded components never
     # register in project_description.json, so re-including one (e.g. a

esphome/build_helpers/idedata.py

@@ -76,7 +76,7 @@ def _is_esphome_src(file: str) -> bool:
     )
 
 
-def _split_command(command: str) -> list[str]:
+def split_command(command: str) -> list[str]:
     r"""Tokenize a compile_commands.json / response-file command string.
 
     On Windows, tokenize per Windows ``argv`` rules via ``CommandLineToArgvW``.
@@ -128,7 +128,7 @@ def _expand_response_files(tokens: list[str], directory: Path) -> list[str]:
             try:
                 out.extend(
                     _expand_response_files(
-                        _split_command(rf.read_text(encoding="utf-8")), directory
+                        split_command(rf.read_text(encoding="utf-8")), directory
                     )
                 )
                 continue
@@ -157,7 +157,7 @@ def _pick_entry(entries: list[dict]) -> dict:
 _LAUNCHER_STEMS = frozenset({"ccache", "sccache", "distcc", "icecc", "buildcache"})
 
 
-def _is_launcher(token: str) -> bool:
+def is_launcher(token: str) -> bool:
     return Path(token).stem.lower() in _LAUNCHER_STEMS
 
 
@@ -166,7 +166,7 @@ def parse_entry(
 ) -> tuple[str, list[str], list[str], list[str]]:
     """Parse one compile_commands entry -> (cxx_path, defines, includes, cxx_flags)."""
     directory = Path(entry["directory"])
-    tokens = _expand_response_files(_split_command(entry["command"]), directory)
+    tokens = _expand_response_files(split_command(entry["command"]), directory)
 
     def _include(raw: str) -> str:
         # Resolve against the entry's ``directory`` so cached idedata works
@@ -182,7 +182,7 @@ def parse_entry(
     if not tokens:
         # An empty command, or one that was only the launcher; fail by name
         raise ValueError(f"empty compile command for {entry.get('file')}")
-    if _is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"):
+    if is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"):
         # Stale DB built with a launcher this run no longer configures; the
         # real compiler is the next token
         _LOGGER.warning("Stripping unconfigured launcher %s", tokens[0])
@@ -282,7 +282,7 @@ def _cache_usable(cached: object) -> bool:
     if not isinstance(cached, dict) or "cc_path" not in cached:
         return False
     cxx_path = cached.get("cxx_path")
-    if not isinstance(cxx_path, str) or _is_launcher(cxx_path):
+    if not isinstance(cxx_path, str) or is_launcher(cxx_path):
         return False
     includes = cached.get("includes")
     return isinstance(includes, dict) and isinstance(includes.get("build"), list)
@@ -330,7 +330,7 @@ def load_or_build_idedata(
 def reject_launcher_compiler(cxx_path: str) -> None:
     """Reject a compile DB naming a launcher (ccache) as the compiler; it
     must never be probed, cached, or consumed."""
-    if _is_launcher(cxx_path):
+    if is_launcher(cxx_path):
         raise EsphomeError(
             f"compile_commands.json names the launcher {cxx_path} as the "
             "compiler; the compile database is unusable"

esphome/espidf/framework.py

@@ -17,6 +17,7 @@ from esphome.build_helpers.ccache import (
     parse_enable_env,
     resolve_ccache_path,
 )
+from esphome.build_helpers.pch import ccache_pch_env
 from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path
 from esphome.core import Version
 from esphome.framework_helpers import (
@@ -1200,14 +1201,30 @@ def _ccache_env() -> dict[str, str]:
     Only values the user has not already set in the environment are returned, so
     a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected.
     """
-    # IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared
-    # ESPHOME_CCACHE_ENABLE.
+    if not _ccache_enabled():
+        # The raw knob value (e.g. "disable") is still inherited by idf.py
+        # via os.environ, where a non-false-constant string reads as
+        # truthy; export the canonical off spelling instead
+        return {"IDF_CCACHE_ENABLE": "0"}
+
+    env = ccache_defaults_env(get_idf_tools_path() / "ccache")
+    env.update(ccache_pch_env())
+    # Exactly one canonical spelling ever reaches idf.py, whatever the
+    # accepted input spelling was ("enable", "yes", ...)
+    env["IDF_CCACHE_ENABLE"] = "1"
+    return env
+
+
+def _ccache_enabled() -> bool:
+    """Whether ESP-IDF compiles run under ccache.
+
+    IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared
+    ESPHOME_CCACHE_ENABLE; when unset, enabled iff a runnable binary is
+    on PATH.
+    """
     idf_knob = parse_enable_env("IDF_CCACHE_ENABLE")
     if idf_knob is False:
-        # The raw value (e.g. "disable") is still inherited by idf.py via
-        # os.environ, where a non-false-constant string reads as truthy;
-        # export the canonical off spelling instead
-        return {"IDF_CCACHE_ENABLE": "0"}
+        return False
     if idf_knob is True:
         # Forced on ignores the runnability verdict, but the outcome is
         # worth saying out loud. Only the truly-missing case means idf.py
@@ -1226,17 +1243,8 @@ def _ccache_env() -> dict[str, str]:
                     "was rejected here (probe failure or "
                     "ESPHOME_CCACHE_ENABLE=0); idf.py will use it anyway"
                 )
-    elif resolve_ccache_path() is None:
-        # ESP-IDF silently skips ccache without the binary; export the
-        # canonical off spelling so an unparsable inherited value (or a
-        # probe-rejected ccache idf.py would still find) cannot enable it
-        return {"IDF_CCACHE_ENABLE": "0"}
-
-    env = ccache_defaults_env(get_idf_tools_path() / "ccache")
-    # Exactly one canonical spelling ever reaches idf.py, whatever the
-    # accepted input spelling was ("enable", "yes", ...)
-    env["IDF_CCACHE_ENABLE"] = "1"
-    return env
+        return True
+    return resolve_ccache_path() is not None
 
 
 def get_framework_env(

esphome/espidf/toolchain.py

@@ -528,6 +528,11 @@ def run_compile(config, verbose: bool) -> int:
             return result.returncode
         _patch_memory_segments()
 
+    # After every reconfigure so compile_commands and sdkconfig are settled
+    from esphome.build_gen.espidf import prepare_pch
+
+    prepare_pch()
+
     # Build
     args = []
 

tests/unit_tests/build_gen/test_espidf.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import logging
 from pathlib import Path
+import subprocess
 from unittest.mock import patch
 
 import pytest
@@ -488,3 +489,100 @@ def test_get_component_cmakelists_no_compile_features() -> None:
         content = get_component_cmakelists()
 
     assert "target_compile_features" not in content
+
+
+def _make_pch_device(tmp_path: Path, name: str) -> Path:
+    """A device dir with the pch source headers and a stub compile_commands."""
+    from esphome.build_gen.espidf import _PCH_HEADERS
+
+    dev = tmp_path / name
+    for header in _PCH_HEADERS:
+        path = dev / "src" / header
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text("")
+    build = dev / "build"
+    build.mkdir(exist_ok=True)
+    from esphome.build_helpers.pch import pch_header_text
+
+    (build / "esphome_pch.h").write_text(pch_header_text(_PCH_HEADERS))
+    (build / "compile_commands.json").write_text(
+        json.dumps(
+            [
+                {
+                    "directory": str(build),
+                    "command": (
+                        "g++ -DX=1 -include esphome_pch.h "
+                        "-o esp-idf/src/CMakeFiles/__idf_src.dir/a.cpp.obj "
+                        f"-c {dev}/src/a.cpp"
+                    ),
+                    "file": f"{dev}/src/a.cpp",
+                }
+            ]
+        )
+    )
+    return dev
+
+
+def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None:
+    from esphome.build_gen.espidf import prepare_pch
+
+    dev = _make_pch_device(tmp_path, "dev_a")
+    CORE.build_path = dev
+    gch = dev / "build" / "esphome_pch.h.gch"
+
+    def fake_compile(cmd, **kwargs):
+        # The compile must target the header, not the stub TU
+        assert cmd[-5:-3] == ["c++-header", "-c"]
+        gch.write_bytes(b"gch")
+        return subprocess.CompletedProcess(cmd, 0, "", "")
+
+    with (
+        patch.object(CORE, "name", "test"),
+        patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
+    ):
+        prepare_pch()
+    assert (dev / "build" / "esphome_pch.h").read_text().startswith("#include")
+    checksum = (dev / "build" / "esphome_pch.h.gch.sum").read_text().strip()
+    assert len(checksum) == 64
+    # Unchanged inputs: the second call must not recompile
+    with (
+        patch.object(CORE, "name", "test"),
+        patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError),
+    ):
+        prepare_pch()
+
+
+def test_pch_no_device_path_poison(tmp_path: Path) -> None:
+    """Regression: neither the injected -include nor the .sum may carry the
+    per-device build path, or cross-device ccache sharing breaks."""
+    from esphome.build_gen.espidf import get_component_cmakelists, prepare_pch
+
+    sums = []
+    for name in ("dev_a", "dev_b"):
+        dev = _make_pch_device(tmp_path, name)
+        CORE.build_path = dev
+        gch = dev / "build" / "esphome_pch.h.gch"
+
+        def fake_compile(cmd, _gch=gch, **kwargs):
+            _gch.write_bytes(b"gch")
+            return subprocess.CompletedProcess(cmd, 0, "", "")
+
+        with (
+            patch.object(CORE, "name", name),
+            patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
+        ):
+            prepare_pch()
+            content = get_component_cmakelists()
+        assert str(dev) not in content
+        sums.append((dev / "build" / "esphome_pch.h.gch.sum").read_text())
+    assert sums[0] == sums[1]
+
+
+def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None:
+    from esphome.build_gen.espidf import get_component_cmakelists
+
+    content = get_component_cmakelists()
+    assert '"$<$<COMPILE_LANGUAGE:CXX>:-include>"' in content
+    assert '"$<$<COMPILE_LANGUAGE:CXX>:esphome_pch.h>"' in content
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    assert "-include" not in get_component_cmakelists()

tests/unit_tests/build_helpers/test_idedata.py

@@ -327,7 +327,7 @@ def test_split_command_preserves_paths_and_unescapes_quotes() -> None:
     r"""Backslash paths survive while ``\"`` define-quoting is unescaped."""
     command = r"C:\esp\bin\riscv32-esp-elf-g++.exe -DVER=\"1.2.3\" -IC:/inc/a -c x.cpp"
 
-    tokens = idedata._split_command(command)
+    tokens = idedata.split_command(command)
 
     assert tokens[0] == r"C:\esp\bin\riscv32-esp-elf-g++.exe"
     assert '-DVER="1.2.3"' in tokens
@@ -341,8 +341,8 @@ def test_split_command_empty_returns_empty() -> None:
     Guards against ``CommandLineToArgvW("")`` returning the current process name
     instead of an empty list.
     """
-    assert idedata._split_command("") == []
-    assert idedata._split_command("   ") == []
+    assert idedata.split_command("") == []
+    assert idedata.split_command("   ") == []
 
 
 @pytest.mark.skipif(os.name != "nt", reason="Windows argv tokenization")
@@ -503,9 +503,9 @@ def test_load_or_build_idedata_rebuilds_non_dict_cache(tmp_path: Path) -> None:
 def test_is_launcher_matches_only_known_launchers() -> None:
     """Compilers of any shape pass; only the closed launcher set matches."""
     for token in ("/t/g++-13", "gcc-8.4.0", "clang++-17", "armcc", "icx", "cc"):
-        assert not idedata._is_launcher(token)
+        assert not idedata.is_launcher(token)
     for token in ("/opt/homebrew/bin/ccache", "CCACHE.EXE", "distcc", "sccache"):
-        assert idedata._is_launcher(token)
+        assert idedata.is_launcher(token)
 
 
 def test_load_or_build_idedata_corrupted_cache_is_logged(

tests/unit_tests/test_espidf_toolchain.py

@@ -93,6 +93,13 @@ def test_get_configured_targets_ci_installs_all(monkeypatch: pytest.MonkeyPatch)
     assert toolchain._get_configured_targets() is None
 
 
+@pytest.fixture(autouse=True)
+def _no_ccache(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Deterministic run_compile: no host ccache probe, no pch work."""
+    monkeypatch.setenv("IDF_CCACHE_ENABLE", "0")
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+
+
 def _setup_build(setup_core: Path) -> tuple[Path, Path]:
     """Point CORE at a build dir; return (compile_commands, idedata cache) paths."""
     CORE.name = "test"