From 787ee418b033b50251d61210b2fe566ef7c63129 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Sep 2026 18:28:03 +0200 Subject: [PATCH] [esp32] Precompile the core headers for ESP-IDF source compiles (#19820) --- .github/workflows/ci-docker.yml | 1 + esphome/build_gen/espidf.py | 47 +++++++++ esphome/build_helpers/pch.py | 94 +++++++++++++++++ esphome/core/pch_prefix.h | 5 + esphome/espidf/framework.py | 2 + esphome/espidf/toolchain.py | 4 + esphome/writer.py | 2 + tests/unit_tests/build_gen/test_espidf.py | 75 ++++++++++++++ tests/unit_tests/build_helpers/test_pch.py | 111 +++++++++++++++++++++ tests/unit_tests/conftest.py | 7 ++ tests/unit_tests/test_espidf_framework.py | 3 + tests/unit_tests/test_espidf_toolchain.py | 28 ++++++ 12 files changed, 379 insertions(+) create mode 100644 esphome/build_helpers/pch.py create mode 100644 esphome/core/pch_prefix.h create mode 100644 tests/unit_tests/build_helpers/test_pch.py diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 5ef51be02c..bf099d4ebf 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -19,6 +19,7 @@ on: - "esphome/core/**" - "esphome/writer.py" - "esphome/build_gen/**" + - "esphome/build_helpers/**" - "esphome/espidf/**" - "esphome/platformio/**" - "esphome/components/bk72xx/**" diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 7689fc93b0..766c945467 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -4,6 +4,7 @@ import json import logging from pathlib import Path +from esphome.build_helpers import pch from esphome.components.esp32 import ( get_esp32_variant, get_excluded_builtin_components, @@ -282,9 +283,55 @@ idf_component_register( target_link_options(${{COMPONENT_LIB}} PUBLIC {link_opts_str} ) +{_pch_cmake_block()}""" + + +# Where CMake puts the .gch of the src component; ccache reads the checksum +# next to it in place of the .gch +_PCH_SUM_PATH = "build/esp-idf/src/CMakeFiles/__idf_src.dir/cmake_pch.hxx.gch.sum" + + +def _pch_cmake_block() -> str: + """The CMake block that precompiles the core headers for the C++ sources + of the src component; empty when disabled.""" + if not pch.pch_enabled(): + return "" + headers = "\n".join( + f' "$<$:${{CMAKE_CURRENT_SOURCE_DIR}}/{header}>"' + for header in pch.PCH_DEFAULT_HEADERS + ) + return f""" +# ESPHome precompiled header +target_precompile_headers(${{COMPONENT_LIB}} PRIVATE +{headers} +) """ +def _read_if_exists(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.is_file() else "" + + +def write_pch_checksum() -> None: + """Write the checksum ccache uses in place of the .gch: the core headers, + the framework version, the sdkconfig and the managed component versions.""" + if not pch.pch_enabled(): + return + pch.log_pch_in_use() + checksum = pch.pch_checksum( + CORE.relative_src_path(), + pch.PCH_DEFAULT_HEADERS, + ( + str(idf_version()), + _read_if_exists(CORE.relative_build_path(f"sdkconfig.{CORE.name}")), + _read_if_exists(CORE.relative_build_path("dependencies.lock")), + ), + ) + path = CORE.relative_build_path(_PCH_SUM_PATH) + path.parent.mkdir(parents=True, exist_ok=True) + write_file_if_changed(path, checksum + "\n") + + def write_project( minimal: bool = False, builtin_components: list[str] | None = None ) -> None: diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py new file mode 100644 index 0000000000..d93c186c7c --- /dev/null +++ b/esphome/build_helpers/pch.py @@ -0,0 +1,94 @@ +"""Shared precompiled header policy for the build backends.""" + +from __future__ import annotations + +from collections.abc import Iterable +import hashlib +import logging +import os +from pathlib import Path +import posixpath +import re + +from esphome.build_helpers.ccache import parse_enable_env + +_LOGGER = logging.getLogger(__name__) + +# The core headers every backend precompiles +PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) + +# What ccache needs to cache compiles that load a .gch +_CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros") + +# Both include forms: an angle include resolving under src/ enters the digest +_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE) + + +def pch_enabled() -> bool: + """Precompiled-header knob: default on, ``ESPHOME_PCH_ENABLE=0`` opts out.""" + return parse_enable_env("ESPHOME_PCH_ENABLE") is not False + + +def ccache_pch_env() -> dict[str, str]: + """What ccache needs to cache compiles that load a .gch, added to what + the user already set.""" + if not pch_enabled(): + return {} + sloppiness = [ + item.strip() + for item in os.environ.get("CCACHE_SLOPPINESS", "").split(",") + if item.strip() + ] + sloppiness += [item for item in _CCACHE_PCH_SLOPPINESS if item not in sloppiness] + env = {"CCACHE_SLOPPINESS": ",".join(sloppiness)} + if "CCACHE_PCH_EXTSUM" not in os.environ: + env["CCACHE_PCH_EXTSUM"] = "true" + return env + + +def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]: + """Include closure of ``roots``: src-relative name -> contents. + + Resolution mirrors the compiler (includer's dir, then src root). No + #ifdef evaluation: including too much is the safe direction. Headers + outside ``src_dir`` are covered by the version strings of the caller. + """ + seen: dict[str, bytes] = {} + stack: list[tuple[str, str]] = [(name, "") for name in roots] + while stack: + name, from_dir = stack.pop() + for candidate in (f"{from_dir}/{name}" if from_dir else name, name): + rel = posixpath.normpath(candidate) + if not rel.startswith("..") and (src_dir / rel).is_file(): + break + else: + continue + if rel in seen: + continue + data = seen[rel] = (src_dir / rel).read_bytes() + parent = posixpath.dirname(rel) + stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data)) + return seen + + +def pch_checksum( + src_dir: Path, include_headers: Iterable[str], extra: Iterable[str] +) -> str: + """Digest of the prefix header's include closure plus ``extra``.""" + digest = hashlib.sha256() + closure = _include_closure(src_dir, include_headers) + for name in sorted(closure): + digest.update(name.encode()) + digest.update(closure[name]) + digest.update(b"\0") + for item in extra: + digest.update(item.encode()) + digest.update(b"\0") + return digest.hexdigest() + + +_DISABLE_HINT = " (set ESPHOME_PCH_ENABLE=0 to disable)" + + +def log_pch_in_use() -> None: + _LOGGER.info("Compiling with a precompiled header%s", _DISABLE_HINT) diff --git a/esphome/core/pch_prefix.h b/esphome/core/pch_prefix.h new file mode 100644 index 0000000000..fbb4eaf79f --- /dev/null +++ b/esphome/core/pch_prefix.h @@ -0,0 +1,5 @@ +#pragma once +// The core headers the precompiled header holds +#include "esphome/core/defines.h" +#include "esphome/core/application.h" +#include "esphome/core/automation.h" diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 9373b5f569..8885da05cc 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -16,6 +16,7 @@ from esphome.build_helpers.ccache import ( parse_enable_env, resolve_ccache_path, ) +from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path from esphome.core import Version from esphome.framework_helpers import ( @@ -1240,6 +1241,7 @@ def _ccache_env() -> dict[str, str]: return {"IDF_CCACHE_ENABLE": "0"} env = ccache_defaults_env(get_idf_tools_path() / "ccache") + env.update(ccache_pch_env()) # Exactly one canonical spelling ever reaches idf.py, whatever the # accepted input spelling was ("enable", "yes", ...) env["IDF_CCACHE_ENABLE"] = "1" diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index f695bdb7ab..f8575234f8 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -528,6 +528,10 @@ def run_compile(config, verbose: bool) -> int: return result.returncode _patch_memory_segments() + from esphome.build_gen.espidf import write_pch_checksum + + write_pch_checksum() + # Build args = [] diff --git a/esphome/writer.py b/esphome/writer.py index bddb6a0f5f..bdd60af1dc 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -220,6 +220,8 @@ ESPHOME_H_EXCLUDE = { Path("esphome/core/controller_dispatch.h"), # moved to components/ring_buffer/, removed in 2026.11.0 Path("esphome/core/ring_buffer.h"), + # build machinery, not user API + Path("esphome/core/pch_prefix.h"), } ESPHOME_README_TXT = """ THIS DIRECTORY IS AUTO-GENERATED, DO NOT MODIFY diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 2848d7202d..ec8e2f3a36 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -500,3 +500,78 @@ def test_get_component_cmakelists_no_compile_features() -> None: content = get_component_cmakelists() assert "target_compile_features" not in content + + +def _make_pch_project(tmp_path: Path) -> Path: + """A build path with the core headers, an sdkconfig and a lock file.""" + from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS + + CORE.build_path = tmp_path + for header in PCH_DEFAULT_HEADERS: + path = tmp_path / "src" / header + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#include "esphome/core/defines.h"\n') + (tmp_path / "src" / "esphome" / "core" / "defines.h").write_text("#define M 1\n") + (tmp_path / "sdkconfig.test").write_text("CONFIG_X=y\n") + (tmp_path / "dependencies.lock").write_text("espressif/mdns: 1.12.0\n") + return tmp_path + + +def _pch_checksum() -> str: + from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum + + with patch.object(CORE, "name", "test"): + write_pch_checksum() + return CORE.relative_build_path(_PCH_SUM_PATH).read_text() + + +def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None: + from esphome.build_gen.espidf import get_component_cmakelists + + content = get_component_cmakelists() + assert ( + "target_precompile_headers(${COMPONENT_LIB} PRIVATE\n" + ' "$<$:${CMAKE_CURRENT_SOURCE_DIR}/' + 'esphome/core/pch_prefix.h>"\n)' + ) in content + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + assert "target_precompile_headers" not in get_component_cmakelists() + + +@pytest.mark.parametrize( + ("file", "content"), + [ + ("src/esphome/core/defines.h", "#define M 2\n"), + ("sdkconfig.test", "CONFIG_X=n\n"), + ("dependencies.lock", "espressif/mdns: 1.13.0\n"), + ], +) +def test_pch_checksum_tracks_its_inputs( + tmp_path: Path, file: str, content: str +) -> None: + """The checksum stands in for the .gch in ccache, so it has to change + with a core header, the sdkconfig and a managed component version.""" + project = _make_pch_project(tmp_path) + first = _pch_checksum() + assert len(first.strip()) == 64 + (project / file).write_text(content) + assert _pch_checksum() != first + + +def test_pch_checksum_is_the_same_for_two_devices(tmp_path: Path) -> None: + sums = [] + for name in ("dev_a", "dev_b"): + _make_pch_project(tmp_path / name) + sums.append(_pch_checksum()) + assert sums[0] == sums[1] + + +def test_pch_checksum_disabled_writes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum + + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + _make_pch_project(tmp_path) + write_pch_checksum() + assert not CORE.relative_build_path(_PCH_SUM_PATH).exists() diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py new file mode 100644 index 0000000000..5a3ba2cdd7 --- /dev/null +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -0,0 +1,111 @@ +"""Tests for esphome.build_helpers.pch.""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.build_helpers import pch + + +def _write(src_dir: Path, name: str, content: str) -> None: + path = src_dir / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, True), + ("1", True), + ("0", False), + ("false", False), + ("", False), + ], +) +def test_pch_enabled(value: str | None, expected: bool) -> None: + env = {} if value is None else {"ESPHOME_PCH_ENABLE": value} + with patch.dict(os.environ, env, clear=True): + assert pch.pch_enabled() is expected + + +def test_ccache_pch_env_enabled() -> None: + with patch.dict(os.environ, {}, clear=True): + env = pch.ccache_pch_env() + assert env == { + "CCACHE_SLOPPINESS": "pch_defines,time_macros", + "CCACHE_PCH_EXTSUM": "true", + } + + +def test_ccache_pch_env_keeps_user_values() -> None: + """A user sloppiness list without the pch entries would stop ccache + from caching every compile that loads the .gch.""" + user = {"CCACHE_SLOPPINESS": "locale, time_macros", "CCACHE_PCH_EXTSUM": "false"} + with patch.dict(os.environ, user, clear=True): + assert pch.ccache_pch_env() == { + "CCACHE_SLOPPINESS": "locale,time_macros,pch_defines" + } + + +def test_ccache_pch_env_disabled() -> None: + with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True): + assert pch.ccache_pch_env() == {} + + +def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None: + """Sibling includes resolve against the includer's directory first, + full paths against the src root; unresolvable names end the walk.""" + _write(tmp_path, "esphome/components/x/a.h", '#include "b.h"\n') + _write( + tmp_path, + "esphome/components/x/b.h", + '#include "esphome/core/deep.h"\n#include \n#include "missing.h"\n', + ) + _write(tmp_path, "esphome/core/deep.h", "") + closure = pch._include_closure(tmp_path, ["esphome/components/x/a.h"]) + assert sorted(closure) == [ + "esphome/components/x/a.h", + "esphome/components/x/b.h", + "esphome/core/deep.h", + ] + + +def test_include_closure_handles_cycles(tmp_path: Path) -> None: + _write(tmp_path, "a.h", '#include "b.h"\n') + _write(tmp_path, "b.h", '#include "a.h"\n') + assert sorted(pch._include_closure(tmp_path, ["a.h"])) == ["a.h", "b.h"] + + +def test_include_closure_blocks_parent_escape(tmp_path: Path) -> None: + _write(tmp_path / "src", "a.h", '#include "../outside.h"\n') + (tmp_path / "outside.h").write_text("") + assert sorted(pch._include_closure(tmp_path / "src", ["a.h"])) == ["a.h"] + + +def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None: + """A transitive header edit or an extra-identity change must change the + digest; unrelated files must not.""" + _write(tmp_path, "root.h", '#include "nested.h"\n') + _write(tmp_path, "nested.h", "int a;\n") + _write(tmp_path, "unrelated.h", "int u;\n") + base = pch.pch_checksum(tmp_path, ["root.h"], ["id"]) + assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"]) + assert base != pch.pch_checksum(tmp_path, ["root.h"], ["other-id"]) + _write(tmp_path, "unrelated.h", "int changed;\n") + assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"]) + _write(tmp_path, "nested.h", "int b;\n") + assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"]) + + +def test_include_closure_walks_angle_includes_under_src(tmp_path: Path) -> None: + """An angle include resolving under src/ must enter the digest; one + that does not simply ends the walk.""" + _write(tmp_path, "a.h", "#include \n#include \n") + (tmp_path / "local.h").write_text("") + closure = pch._include_closure(tmp_path, ["a.h"]) + assert set(closure) == {"a.h", "local.h"} diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index ad9c0bb11f..0353c8afc8 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -174,3 +174,10 @@ def held_lock() -> Callable[..., Callable[..., None]]: return acquire return make + + +@pytest.fixture(autouse=True) +def _default_pch_knobs(monkeypatch: pytest.MonkeyPatch) -> None: + """The precompiled header knobs change what the generators emit; a value + exported in the developer's or CI's shell must not reach the tests.""" + monkeypatch.delenv("ESPHOME_PCH_ENABLE", raising=False) diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 3eeace9914..9654c7cfc5 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -1609,6 +1609,9 @@ def test_ccache_env_default_enabled_when_available(tmp_path: Path) -> None: assert env["CCACHE_NOHASHDIR"] == "true" assert env["CCACHE_DEPEND"] == "1" assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve()) + # The pch cannot cache under ccache without these + assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros" + assert env["CCACHE_PCH_EXTSUM"] == "true" def test_ccache_env_disabled_when_binary_missing(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index bb2aab17a2..2d65563663 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -93,6 +93,13 @@ def test_get_configured_targets_ci_installs_all(monkeypatch: pytest.MonkeyPatch) assert toolchain._get_configured_targets() is None +@pytest.fixture(autouse=True) +def _no_ccache(monkeypatch: pytest.MonkeyPatch) -> None: + """Deterministic run_compile: no host ccache probe, no pch work.""" + monkeypatch.setenv("IDF_CCACHE_ENABLE", "0") + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + + def _setup_build(setup_core: Path) -> tuple[Path, Path]: """Point CORE at a build dir; return (compile_commands, idedata cache) paths.""" CORE.name = "test" @@ -690,6 +697,27 @@ def test_run_compile_without_compile_process_limit(setup_core: Path) -> None: mock_run.assert_called_once_with("build", "size", jobs=None) +def test_run_compile_writes_the_pch_checksum_before_the_build( + setup_core: Path, +) -> None: + _setup_build(setup_core) + order: list[str] = [] + + with ( + patch.object(toolchain, "need_reconfigure", return_value=False), + patch.object( + toolchain, "run_idf_py", side_effect=lambda *a, **k: order.append("build") + ), + patch.object(toolchain, "print_summary"), + patch( + "esphome.build_gen.espidf.write_pch_checksum", + side_effect=lambda: order.append("checksum"), + ), + ): + toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) + assert order[:2] == ["checksum", "build"] + + def test_get_core_framework_version_from_core_data(): """The version is read from CORE.data when validation populated it.""" from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION