[esp32] Precompile the core headers for ESP-IDF source compiles (#19820)

tests/unit_tests/build_gen/test_espidf.py

@@ -500,3 +500,78 @@ def test_get_component_cmakelists_no_compile_features() -> None:
         content = get_component_cmakelists()
 
     assert "target_compile_features" not in content
+
+
+def _make_pch_project(tmp_path: Path) -> Path:
+    """A build path with the core headers, an sdkconfig and a lock file."""
+    from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS
+
+    CORE.build_path = tmp_path
+    for header in PCH_DEFAULT_HEADERS:
+        path = tmp_path / "src" / header
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text('#include "esphome/core/defines.h"\n')
+    (tmp_path / "src" / "esphome" / "core" / "defines.h").write_text("#define M 1\n")
+    (tmp_path / "sdkconfig.test").write_text("CONFIG_X=y\n")
+    (tmp_path / "dependencies.lock").write_text("espressif/mdns: 1.12.0\n")
+    return tmp_path
+
+
+def _pch_checksum() -> str:
+    from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum
+
+    with patch.object(CORE, "name", "test"):
+        write_pch_checksum()
+    return CORE.relative_build_path(_PCH_SUM_PATH).read_text()
+
+
+def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None:
+    from esphome.build_gen.espidf import get_component_cmakelists
+
+    content = get_component_cmakelists()
+    assert (
+        "target_precompile_headers(${COMPONENT_LIB} PRIVATE\n"
+        '    "$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/'
+        'esphome/core/pch_prefix.h>"\n)'
+    ) in content
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    assert "target_precompile_headers" not in get_component_cmakelists()
+
+
+@pytest.mark.parametrize(
+    ("file", "content"),
+    [
+        ("src/esphome/core/defines.h", "#define M 2\n"),
+        ("sdkconfig.test", "CONFIG_X=n\n"),
+        ("dependencies.lock", "espressif/mdns: 1.13.0\n"),
+    ],
+)
+def test_pch_checksum_tracks_its_inputs(
+    tmp_path: Path, file: str, content: str
+) -> None:
+    """The checksum stands in for the .gch in ccache, so it has to change
+    with a core header, the sdkconfig and a managed component version."""
+    project = _make_pch_project(tmp_path)
+    first = _pch_checksum()
+    assert len(first.strip()) == 64
+    (project / file).write_text(content)
+    assert _pch_checksum() != first
+
+
+def test_pch_checksum_is_the_same_for_two_devices(tmp_path: Path) -> None:
+    sums = []
+    for name in ("dev_a", "dev_b"):
+        _make_pch_project(tmp_path / name)
+        sums.append(_pch_checksum())
+    assert sums[0] == sums[1]
+
+
+def test_pch_checksum_disabled_writes_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum
+
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    _make_pch_project(tmp_path)
+    write_pch_checksum()
+    assert not CORE.relative_build_path(_PCH_SUM_PATH).exists()

tests/unit_tests/build_helpers/test_pch.py

@@ -0,0 +1,111 @@
+"""Tests for esphome.build_helpers.pch."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.build_helpers import pch
+
+
+def _write(src_dir: Path, name: str, content: str) -> None:
+    path = src_dir / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        (None, True),
+        ("1", True),
+        ("0", False),
+        ("false", False),
+        ("", False),
+    ],
+)
+def test_pch_enabled(value: str | None, expected: bool) -> None:
+    env = {} if value is None else {"ESPHOME_PCH_ENABLE": value}
+    with patch.dict(os.environ, env, clear=True):
+        assert pch.pch_enabled() is expected
+
+
+def test_ccache_pch_env_enabled() -> None:
+    with patch.dict(os.environ, {}, clear=True):
+        env = pch.ccache_pch_env()
+    assert env == {
+        "CCACHE_SLOPPINESS": "pch_defines,time_macros",
+        "CCACHE_PCH_EXTSUM": "true",
+    }
+
+
+def test_ccache_pch_env_keeps_user_values() -> None:
+    """A user sloppiness list without the pch entries would stop ccache
+    from caching every compile that loads the .gch."""
+    user = {"CCACHE_SLOPPINESS": "locale, time_macros", "CCACHE_PCH_EXTSUM": "false"}
+    with patch.dict(os.environ, user, clear=True):
+        assert pch.ccache_pch_env() == {
+            "CCACHE_SLOPPINESS": "locale,time_macros,pch_defines"
+        }
+
+
+def test_ccache_pch_env_disabled() -> None:
+    with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True):
+        assert pch.ccache_pch_env() == {}
+
+
+def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None:
+    """Sibling includes resolve against the includer's directory first,
+    full paths against the src root; unresolvable names end the walk."""
+    _write(tmp_path, "esphome/components/x/a.h", '#include "b.h"\n')
+    _write(
+        tmp_path,
+        "esphome/components/x/b.h",
+        '#include "esphome/core/deep.h"\n#include <system.h>\n#include "missing.h"\n',
+    )
+    _write(tmp_path, "esphome/core/deep.h", "")
+    closure = pch._include_closure(tmp_path, ["esphome/components/x/a.h"])
+    assert sorted(closure) == [
+        "esphome/components/x/a.h",
+        "esphome/components/x/b.h",
+        "esphome/core/deep.h",
+    ]
+
+
+def test_include_closure_handles_cycles(tmp_path: Path) -> None:
+    _write(tmp_path, "a.h", '#include "b.h"\n')
+    _write(tmp_path, "b.h", '#include "a.h"\n')
+    assert sorted(pch._include_closure(tmp_path, ["a.h"])) == ["a.h", "b.h"]
+
+
+def test_include_closure_blocks_parent_escape(tmp_path: Path) -> None:
+    _write(tmp_path / "src", "a.h", '#include "../outside.h"\n')
+    (tmp_path / "outside.h").write_text("")
+    assert sorted(pch._include_closure(tmp_path / "src", ["a.h"])) == ["a.h"]
+
+
+def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None:
+    """A transitive header edit or an extra-identity change must change the
+    digest; unrelated files must not."""
+    _write(tmp_path, "root.h", '#include "nested.h"\n')
+    _write(tmp_path, "nested.h", "int a;\n")
+    _write(tmp_path, "unrelated.h", "int u;\n")
+    base = pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    assert base != pch.pch_checksum(tmp_path, ["root.h"], ["other-id"])
+    _write(tmp_path, "unrelated.h", "int changed;\n")
+    assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+    _write(tmp_path, "nested.h", "int b;\n")
+    assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"])
+
+
+def test_include_closure_walks_angle_includes_under_src(tmp_path: Path) -> None:
+    """An angle include resolving under src/ must enter the digest; one
+    that does not simply ends the walk."""
+    _write(tmp_path, "a.h", "#include <local.h>\n#include <Arduino.h>\n")
+    (tmp_path / "local.h").write_text("")
+    closure = pch._include_closure(tmp_path, ["a.h"])
+    assert set(closure) == {"a.h", "local.h"}

tests/unit_tests/conftest.py

@@ -174,3 +174,10 @@ def held_lock() -> Callable[..., Callable[..., None]]:
         return acquire
 
     return make
+
+
+@pytest.fixture(autouse=True)
+def _default_pch_knobs(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The precompiled header knobs change what the generators emit; a value
+    exported in the developer's or CI's shell must not reach the tests."""
+    monkeypatch.delenv("ESPHOME_PCH_ENABLE", raising=False)

tests/unit_tests/test_espidf_framework.py

@@ -1609,6 +1609,9 @@ def test_ccache_env_default_enabled_when_available(tmp_path: Path) -> None:
     assert env["CCACHE_NOHASHDIR"] == "true"
     assert env["CCACHE_DEPEND"] == "1"
     assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve())
+    # The pch cannot cache under ccache without these
+    assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
+    assert env["CCACHE_PCH_EXTSUM"] == "true"
 
 
 def test_ccache_env_disabled_when_binary_missing(tmp_path: Path) -> None:

tests/unit_tests/test_espidf_toolchain.py

@@ -93,6 +93,13 @@ def test_get_configured_targets_ci_installs_all(monkeypatch: pytest.MonkeyPatch)
     assert toolchain._get_configured_targets() is None
 
 
+@pytest.fixture(autouse=True)
+def _no_ccache(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Deterministic run_compile: no host ccache probe, no pch work."""
+    monkeypatch.setenv("IDF_CCACHE_ENABLE", "0")
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+
+
 def _setup_build(setup_core: Path) -> tuple[Path, Path]:
     """Point CORE at a build dir; return (compile_commands, idedata cache) paths."""
     CORE.name = "test"
@@ -690,6 +697,27 @@ def test_run_compile_without_compile_process_limit(setup_core: Path) -> None:
     mock_run.assert_called_once_with("build", "size", jobs=None)
 
 
+def test_run_compile_writes_the_pch_checksum_before_the_build(
+    setup_core: Path,
+) -> None:
+    _setup_build(setup_core)
+    order: list[str] = []
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=False),
+        patch.object(
+            toolchain, "run_idf_py", side_effect=lambda *a, **k: order.append("build")
+        ),
+        patch.object(toolchain, "print_summary"),
+        patch(
+            "esphome.build_gen.espidf.write_pch_checksum",
+            side_effect=lambda: order.append("checksum"),
+        ),
+    ):
+        toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
+    assert order[:2] == ["checksum", "build"]
+
+
 def test_get_core_framework_version_from_core_data():
     """The version is read from CORE.data when validation populated it."""
     from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION