[esp32] Precompile the core headers for ESP-IDF source compiles (#19820)

.github/workflows/ci-docker.yml

@@ -19,6 +19,7 @@ on:
       - "esphome/core/**"
       - "esphome/writer.py"
       - "esphome/build_gen/**"
+      - "esphome/build_helpers/**"
       - "esphome/espidf/**"
       - "esphome/platformio/**"
       - "esphome/components/bk72xx/**"

esphome/build_gen/espidf.py

@@ -4,6 +4,7 @@ import json
 import logging
 from pathlib import Path
 
+from esphome.build_helpers import pch
 from esphome.components.esp32 import (
     get_esp32_variant,
     get_excluded_builtin_components,
@@ -282,9 +283,55 @@ idf_component_register(
 target_link_options(${{COMPONENT_LIB}} PUBLIC
     {link_opts_str}
 )
+{_pch_cmake_block()}"""
+
+
+# Where CMake puts the .gch of the src component; ccache reads the checksum
+# next to it in place of the .gch
+_PCH_SUM_PATH = "build/esp-idf/src/CMakeFiles/__idf_src.dir/cmake_pch.hxx.gch.sum"
+
+
+def _pch_cmake_block() -> str:
+    """The CMake block that precompiles the core headers for the C++ sources
+    of the src component; empty when disabled."""
+    if not pch.pch_enabled():
+        return ""
+    headers = "\n".join(
+        f'    "$<$<COMPILE_LANGUAGE:CXX>:${{CMAKE_CURRENT_SOURCE_DIR}}/{header}>"'
+        for header in pch.PCH_DEFAULT_HEADERS
+    )
+    return f"""
+# ESPHome precompiled header
+target_precompile_headers(${{COMPONENT_LIB}} PRIVATE
+{headers}
+)
 """
 
 
+def _read_if_exists(path: Path) -> str:
+    return path.read_text(encoding="utf-8") if path.is_file() else ""
+
+
+def write_pch_checksum() -> None:
+    """Write the checksum ccache uses in place of the .gch: the core headers,
+    the framework version, the sdkconfig and the managed component versions."""
+    if not pch.pch_enabled():
+        return
+    pch.log_pch_in_use()
+    checksum = pch.pch_checksum(
+        CORE.relative_src_path(),
+        pch.PCH_DEFAULT_HEADERS,
+        (
+            str(idf_version()),
+            _read_if_exists(CORE.relative_build_path(f"sdkconfig.{CORE.name}")),
+            _read_if_exists(CORE.relative_build_path("dependencies.lock")),
+        ),
+    )
+    path = CORE.relative_build_path(_PCH_SUM_PATH)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    write_file_if_changed(path, checksum + "\n")
+
+
 def write_project(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> None:

esphome/build_helpers/pch.py

@@ -0,0 +1,94 @@
+"""Shared precompiled header policy for the build backends."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import hashlib
+import logging
+import os
+from pathlib import Path
+import posixpath
+import re
+
+from esphome.build_helpers.ccache import parse_enable_env
+
+_LOGGER = logging.getLogger(__name__)
+
+# The core headers every backend precompiles
+PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",)
+
+# What ccache needs to cache compiles that load a .gch
+_CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros")
+
+# Both include forms: an angle include resolving under src/ enters the digest
+_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE)
+
+
+def pch_enabled() -> bool:
+    """Precompiled-header knob: default on, ``ESPHOME_PCH_ENABLE=0`` opts out."""
+    return parse_enable_env("ESPHOME_PCH_ENABLE") is not False
+
+
+def ccache_pch_env() -> dict[str, str]:
+    """What ccache needs to cache compiles that load a .gch, added to what
+    the user already set."""
+    if not pch_enabled():
+        return {}
+    sloppiness = [
+        item.strip()
+        for item in os.environ.get("CCACHE_SLOPPINESS", "").split(",")
+        if item.strip()
+    ]
+    sloppiness += [item for item in _CCACHE_PCH_SLOPPINESS if item not in sloppiness]
+    env = {"CCACHE_SLOPPINESS": ",".join(sloppiness)}
+    if "CCACHE_PCH_EXTSUM" not in os.environ:
+        env["CCACHE_PCH_EXTSUM"] = "true"
+    return env
+
+
+def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
+    """Include closure of ``roots``: src-relative name -> contents.
+
+    Resolution mirrors the compiler (includer's dir, then src root). No
+    #ifdef evaluation: including too much is the safe direction. Headers
+    outside ``src_dir`` are covered by the version strings of the caller.
+    """
+    seen: dict[str, bytes] = {}
+    stack: list[tuple[str, str]] = [(name, "") for name in roots]
+    while stack:
+        name, from_dir = stack.pop()
+        for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
+            rel = posixpath.normpath(candidate)
+            if not rel.startswith("..") and (src_dir / rel).is_file():
+                break
+        else:
+            continue
+        if rel in seen:
+            continue
+        data = seen[rel] = (src_dir / rel).read_bytes()
+        parent = posixpath.dirname(rel)
+        stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data))
+    return seen
+
+
+def pch_checksum(
+    src_dir: Path, include_headers: Iterable[str], extra: Iterable[str]
+) -> str:
+    """Digest of the prefix header's include closure plus ``extra``."""
+    digest = hashlib.sha256()
+    closure = _include_closure(src_dir, include_headers)
+    for name in sorted(closure):
+        digest.update(name.encode())
+        digest.update(closure[name])
+        digest.update(b"\0")
+    for item in extra:
+        digest.update(item.encode())
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+_DISABLE_HINT = " (set ESPHOME_PCH_ENABLE=0 to disable)"
+
+
+def log_pch_in_use() -> None:
+    _LOGGER.info("Compiling with a precompiled header%s", _DISABLE_HINT)

esphome/core/pch_prefix.h

@@ -0,0 +1,5 @@
+#pragma once
+// The core headers the precompiled header holds
+#include "esphome/core/defines.h"
+#include "esphome/core/application.h"
+#include "esphome/core/automation.h"

esphome/espidf/framework.py

@@ -16,6 +16,7 @@ from esphome.build_helpers.ccache import (
     parse_enable_env,
     resolve_ccache_path,
 )
+from esphome.build_helpers.pch import ccache_pch_env
 from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path
 from esphome.core import Version
 from esphome.framework_helpers import (
@@ -1240,6 +1241,7 @@ def _ccache_env() -> dict[str, str]:
         return {"IDF_CCACHE_ENABLE": "0"}
 
     env = ccache_defaults_env(get_idf_tools_path() / "ccache")
+    env.update(ccache_pch_env())
     # Exactly one canonical spelling ever reaches idf.py, whatever the
     # accepted input spelling was ("enable", "yes", ...)
     env["IDF_CCACHE_ENABLE"] = "1"

esphome/espidf/toolchain.py

@@ -528,6 +528,10 @@ def run_compile(config, verbose: bool) -> int:
             return result.returncode
         _patch_memory_segments()
 
+    from esphome.build_gen.espidf import write_pch_checksum
+
+    write_pch_checksum()
+
     # Build
     args = []
 

esphome/writer.py

@@ -220,6 +220,8 @@ ESPHOME_H_EXCLUDE = {
     Path("esphome/core/controller_dispatch.h"),
     # moved to components/ring_buffer/, removed in 2026.11.0
     Path("esphome/core/ring_buffer.h"),
+    # build machinery, not user API
+    Path("esphome/core/pch_prefix.h"),
 }
 ESPHOME_README_TXT = """
 THIS DIRECTORY IS AUTO-GENERATED, DO NOT MODIFY

tests/unit_tests/build_gen/test_espidf.py

@@ -500,3 +500,78 @@ def test_get_component_cmakelists_no_compile_features() -> None:
         content = get_component_cmakelists()
 
     assert "target_compile_features" not in content
+
+
+def _make_pch_project(tmp_path: Path) -> Path:
+    """A build path with the core headers, an sdkconfig and a lock file."""
+    from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS
+
+    CORE.build_path = tmp_path
+    for header in PCH_DEFAULT_HEADERS:
+        path = tmp_path / "src" / header
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text('#include "esphome/core/defines.h"\n')
+    (tmp_path / "src" / "esphome" / "core" / "defines.h").write_text("#define M 1\n")
+    (tmp_path / "sdkconfig.test").write_text("CONFIG_X=y\n")
+    (tmp_path / "dependencies.lock").write_text("espressif/mdns: 1.12.0\n")
+    return tmp_path
+
+
+def _pch_checksum() -> str:
+    from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum
+
+    with patch.object(CORE, "name", "test"):
+        write_pch_checksum()
+    return CORE.relative_build_path(_PCH_SUM_PATH).read_text()
+
+
+def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None:
+    from esphome.build_gen.espidf import get_component_cmakelists
+
+    content = get_component_cmakelists()
+    assert (
+        "target_precompile_headers(${COMPONENT_LIB} PRIVATE\n"
+        '    "$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/'
+        'esphome/core/pch_prefix.h>"\n)'
+    ) in content
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    assert "target_precompile_headers" not in get_component_cmakelists()
+
+
+@pytest.mark.parametrize(
+    ("file", "content"),
+    [
+        ("src/esphome/core/defines.h", "#define M 2\n"),
+        ("sdkconfig.test", "CONFIG_X=n\n"),
+        ("dependencies.lock", "espressif/mdns: 1.13.0\n"),
+    ],
+)
+def test_pch_checksum_tracks_its_inputs(
+    tmp_path: Path, file: str, content: str
+) -> None:
+    """The checksum stands in for the .gch in ccache, so it has to change
+    with a core header, the sdkconfig and a managed component version."""
+    project = _make_pch_project(tmp_path)
+    first = _pch_checksum()
+    assert len(first.strip()) == 64
+    (project / file).write_text(content)
+    assert _pch_checksum() != first
+
+
+def test_pch_checksum_is_the_same_for_two_devices(tmp_path: Path) -> None:
+    sums = []
+    for name in ("dev_a", "dev_b"):
+        _make_pch_project(tmp_path / name)
+        sums.append(_pch_checksum())
+    assert sums[0] == sums[1]
+
+
+def test_pch_checksum_disabled_writes_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum
+
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    _make_pch_project(tmp_path)
+    write_pch_checksum()
+    assert not CORE.relative_build_path(_PCH_SUM_PATH).exists()

tests/unit_tests/build_helpers/test_pch.py

@@ -0,0 +1,111 @@
+"""Tests for esphome.build_helpers.pch."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.build_helpers import pch
+
+
+def _write(src_dir: Path, name: str, content: str) -> None:
+    path = src_dir / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        (None, True),
+        ("1", True),
+        ("0", False),
+        ("false", False),
+        ("", False),
+    ],
+)
+def test_pch_enabled(value: str | None, expected: bool) -> None:
+    env = {} if value is None else {"ESPHOME_PCH_ENABLE": value}
+    with patch.dict(os.environ, env, clear=True):
+        assert pch.pch_enabled() is expected
+
+
+def test_ccache_pch_env_enabled() -> None:
+    with patch.dict(os.environ, {}, clear=True):
+        env = pch.ccache_pch_env()
+    assert env == {
+        "CCACHE_SLOPPINESS": "pch_defines,time_macros",
+        "CCACHE_PCH_EXTSUM": "true",
+    }
+
+
+def test_ccache_pch_env_keeps_user_values() -> None:
+    """A user sloppiness list without the pch entries would stop ccache
+    from caching every compile that loads the .gch."""
+    user = {"CCACHE_SLOPPINESS": "locale, time_macros", "CCACHE_PCH_EXTSUM": "false"}
+    with patch.dict(os.environ, user, clear=True):
+        assert pch.ccache_pch_env() == {
+            "CCACHE_SLOPPINESS": "locale,time_macros,pch_defines"
+        }
+
+
+def test_ccache_pch_env_disabled() -> None:
+    with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True):
+        assert pch.ccache_pch_env() == {}
+
+
+def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None:
+    """Sibling includes resolve against the includer's directory first,
+    full paths against the src root; unresolvable names end the walk."""
+    _write(tmp_path, "esphome/components/x/a.h", '#include "b.h"\n')
+    _write(
+        tmp_path,
+        "esphome/components/x/b.h",
+        '#include "esphome/core/deep.h"\n#include <system.h>\n#include "missing.h"\n',
+    )
+    _write(tmp_path, "esphome/core/deep.h", "")
+    closure = pch._include_closure(tmp_path, ["esphome/components/x/a.h"])
+    assert sorted(closure) == [
+        "esphome/components/x/a.h",
+        "esphome/components/x/b.h",
+        "esphome/core/deep.h",
+    ]
+
+
+def test_include_closure_handles_cycles(tmp_path: Path) -> None:
+    _write(tmp_path, "a.h", '#include "b.h"\n')
+    _write(tmp_path, "b.h", '#include "a.h"\n')
+    assert sorted(pch._include_closure(tmp_path, ["a.h"])) == ["a.h", "b.h"]
+
+
+def test_include_closure_blocks_parent_escape(tmp_path: Path) -> None:
+    _write(tmp_path / "src", "a.h", '#include "../outside.h"\n')
+    (tmp_path / "outside.h").write_text("")
+    assert sorted(pch._include_closure(tmp_path / "src", ["a.h"])) == ["a.h"]
+
+
+def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None:
+    """A transitive header edit or an extra-identity change must change the
+    digest; unrelated files must not."""
+    _write(tmp_path, "root.h", '#include "nested.h"\n')
+    _write(tmp_path, "nested.h", "int a;\n")
+    _write(tmp_path, "unrelated.h", "int u;\n")
+    base = pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    assert base != pch.pch_checksum(tmp_path, ["root.h"], ["other-id"])
+    _write(tmp_path, "unrelated.h", "int changed;\n")
+    assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    _write(tmp_path, "nested.h", "int b;\n")
+    assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+
+
+def test_include_closure_walks_angle_includes_under_src(tmp_path: Path) -> None:
+    """An angle include resolving under src/ must enter the digest; one
+    that does not simply ends the walk."""
+    _write(tmp_path, "a.h", "#include <local.h>\n#include <Arduino.h>\n")
+    (tmp_path / "local.h").write_text("")
+    closure = pch._include_closure(tmp_path, ["a.h"])
+    assert set(closure) == {"a.h", "local.h"}

tests/unit_tests/conftest.py

@@ -174,3 +174,10 @@ def held_lock() -> Callable[..., Callable[..., None]]:
         return acquire
 
     return make
+
+
+@pytest.fixture(autouse=True)
+def _default_pch_knobs(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The precompiled header knobs change what the generators emit; a value
+    exported in the developer's or CI's shell must not reach the tests."""
+    monkeypatch.delenv("ESPHOME_PCH_ENABLE", raising=False)

tests/unit_tests/test_espidf_framework.py

@@ -1609,6 +1609,9 @@ def test_ccache_env_default_enabled_when_available(tmp_path: Path) -> None:
     assert env["CCACHE_NOHASHDIR"] == "true"
     assert env["CCACHE_DEPEND"] == "1"
     assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve())
+    # The pch cannot cache under ccache without these
+    assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
+    assert env["CCACHE_PCH_EXTSUM"] == "true"
 
 
 def test_ccache_env_disabled_when_binary_missing(tmp_path: Path) -> None:

tests/unit_tests/test_espidf_toolchain.py

@@ -93,6 +93,13 @@ def test_get_configured_targets_ci_installs_all(monkeypatch: pytest.MonkeyPatch)
     assert toolchain._get_configured_targets() is None
 
 
+@pytest.fixture(autouse=True)
+def _no_ccache(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Deterministic run_compile: no host ccache probe, no pch work."""
+    monkeypatch.setenv("IDF_CCACHE_ENABLE", "0")
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+
+
 def _setup_build(setup_core: Path) -> tuple[Path, Path]:
     """Point CORE at a build dir; return (compile_commands, idedata cache) paths."""
     CORE.name = "test"
@@ -690,6 +697,27 @@ def test_run_compile_without_compile_process_limit(setup_core: Path) -> None:
     mock_run.assert_called_once_with("build", "size", jobs=None)
 
 
+def test_run_compile_writes_the_pch_checksum_before_the_build(
+    setup_core: Path,
+) -> None:
+    _setup_build(setup_core)
+    order: list[str] = []
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=False),
+        patch.object(
+            toolchain, "run_idf_py", side_effect=lambda *a, **k: order.append("build")
+        ),
+        patch.object(toolchain, "print_summary"),
+        patch(
+            "esphome.build_gen.espidf.write_pch_checksum",
+            side_effect=lambda: order.append("checksum"),
+        ),
+    ):
+        toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
+    assert order[:2] == ["checksum", "build"]
+
+
 def test_get_core_framework_version_from_core_data():
     """The version is read from CORE.data when validation populated it."""
     from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION