diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 69b76e0231..bb1bdeac44 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -25,7 +25,6 @@ import subprocess from typing import TYPE_CHECKING, NamedTuple from esphome.arduino8266.framework import toolchain_tool -from esphome.build_helpers.ccache import effective_ccache_basedir from esphome.build_helpers.idedata import is_joined_include from esphome.build_helpers.ninja import ( escape as _e, @@ -38,22 +37,11 @@ from esphome.build_helpers.ninja_gen import ( compile_edges, compile_rule_lines, library_edges, + pch_edges, pch_rule_lines, tool_lines, ) -from esphome.build_helpers.pch import ( - PCH_CORE_HEADER, - PCH_HEADER_NAME, - mark_pch_emitted, - pch_checksum, - pch_consumer_escalation, - pch_degraded, - pch_disabled_degraded, - pch_enabled, - pch_header_text, - pch_probe_args, - pch_strict, -) +from esphome.build_helpers.pch import PCH_CORE_HEADER from esphome.components.esp8266 import build_surgery from esphome.components.esp8266.boards import ( BOARDS, @@ -1151,99 +1139,16 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # One shared variable instead of repeating the flags line on every src # edge (hundreds of edges in a real project) lines.append(f"srcflags = {' '.join(src_other + include_flags)}") - src_cxx_override = None - if pch_enabled() and any(tok.startswith("-include") for tok in flag_sets.cxxflags): - # $cxxflags expands first, so a user -include there means GCC would - # never load the .gch - _LOGGER.warning( - "A -include in build_flags prevents the precompiled header from " - "loading; compiling without it" - ) - pch_degraded("a user -include precedes the pch") - elif pch_enabled(): - # C++ src edges swap the force-includes for one precompiled prefix - # header (same content plus defines.h); C/assembly keep srcflags - pch_header = build_dir / PCH_HEADER_NAME - pch_includes = (*src_includes, PCH_CORE_HEADER) - pch_text = pch_header_text(pch_includes) - checksum = None - try: - if ccache: - # The .sum exists only for CCACHE_PCH_EXTSUM; ninja's depfile - # handles staleness. Strip resolved and raw build paths - # (symlinks) so identical configs share cache entries - flags_id = ( - " ".join(flag_sets.cxxflags) - .replace(effective_ccache_basedir(), "") - .replace(str(CORE.build_path), "") - ) - # The header text covers include order - checksum = pch_checksum( - src_dir, - pch_includes, - ( - pch_text, - str(paths.framework), - str(paths.toolchain), - flags_id, - ), - ) - except (OSError, UnicodeError) as err: - # Identity unknown: a stale cache entry must never be served - _LOGGER.warning( - "Could not establish the pch identity; compiling without it: %s", err - ) - pch_degraded(f"identity unknown: {err}") - else: - _LOGGER.info( - "Compiling with a precompiled header " - "(set ESPHOME_PCH_ENABLE=0 to disable)" - ) - write_file_if_changed(pch_header, pch_text) - sum_path = build_dir / f"{PCH_HEADER_NAME}.gch.sum" - if checksum is not None: - # Generate-time stamp: a hand-run ninja can rebuild the .gch - # while this .sum lags - write_file_if_changed(sum_path, checksum + "\n") - else: - # A stale .sum from an earlier ccache run must not survive - sum_path.unlink(missing_ok=True) - gch = _e(f"{PCH_HEADER_NAME}.gch") - lines.append(f"build {gch}: pch {_e(pch_header)}") - if src_other: - lines.append(f" flags = {' '.join(src_other)}") - # Relative -include: absolute would break cross-device ccache. - # -Wno-error keeps a rejected .gch a warning under user -Werror; - # strict inverts it so any consumer rejection reds the build - # (rejection is per-process, so the probe alone cannot prove - # the consumers) - escalation = pch_consumer_escalation() - cxx_parts = src_other + [ - f"-Winvalid-pch {escalation} -include {PCH_HEADER_NAME}" - ] - lines.append(f"srccxxflags = {' '.join(cxx_parts)}") - pch_dep = gch - if pch_strict(): - # Consumers wait on the probe stamp, so an unloadable .gch - # reds the build here instead of warning ~100 times - probe = " ".join(pch_probe_args(PCH_HEADER_NAME, source=os.devnull)) - lines.append("rule pchprobe") - # $out only expands in rule text, hence the inline stamp - lines.append( - f" command = $cxx $cxxflags $flags {probe}" - " && $python $buildtool touch $out" - ) - lines.append(" description = PCHPROBE $out") - # Runs when the .gch is (re)built; strict consumer -Werror - # covers a cached .gch this process cannot load - lines.append(f"build esphome_pch.probe: pchprobe {gch}") - if src_other: - lines.append(f" flags = {' '.join(src_other)}") - pch_dep = f"{gch} esphome_pch.probe" - src_cxx_override = ("$srccxxflags", pch_dep) - mark_pch_emitted() - else: - pch_disabled_degraded() + src_cxx_override = pch_edges( + lines, + build_dir, + src_dir, + (*src_includes, PCH_CORE_HEADER), + flag_sets.cxxflags, + src_other, + ccache, + (str(paths.framework), str(paths.toolchain)), + ) src_objs = compile_edges( lines, collect_sources(src_dir), diff --git a/esphome/build_gen/host.py b/esphome/build_gen/host.py index c883d912eb..c9625217a8 100644 --- a/esphome/build_gen/host.py +++ b/esphome/build_gen/host.py @@ -28,8 +28,11 @@ from esphome.build_helpers.ninja_gen import ( compile_edges, compile_rule_lines, library_edges, + pch_edges, + pch_rule_lines, tool_lines, ) +from esphome.build_helpers.pch import PCH_CORE_HEADER from esphome.core import CORE, EsphomeError from esphome.framework_helpers import get_project_cxx_compile_flags from esphome.helpers import mkdir_p, write_file_if_changed @@ -228,6 +231,7 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool: lines = [ *tool_lines(compilers.cc, compilers.cxx, ccache), *compile_rule_lines(), + *pch_rule_lines(), "rule link", " command = $cxx -o $out $linkflags @$out.rsp $archives $libdirflags $libflags", " rspfile = $out.rsp", @@ -251,7 +255,27 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool: archives, direct_objs = library_edges(lines, libraries) - src_objs = compile_edges(lines, collect_sources(src_dir), src_dir, "src") + from esphome.components.host import HOST_PCH_PREFIX + + # Host has no framework force-includes; the per-TU cost is the STL + # closure behind the core headers, so those are what is precompiled + src_cxx_override = pch_edges( + lines, + build_dir, + src_dir, + (HOST_PCH_PREFIX, PCH_CORE_HEADER), + cxxflags, + (), + ccache, + compilers.cxx, + ) + src_objs = compile_edges( + lines, + collect_sources(src_dir), + src_dir, + "src", + cxx_override=src_cxx_override, + ) if not src_objs: raise EsphomeError(f"No source files found under {src_dir}") diff --git a/esphome/build_helpers/ninja_gen.py b/esphome/build_helpers/ninja_gen.py index 56f054f23f..84137e4ea5 100644 --- a/esphome/build_helpers/ninja_gen.py +++ b/esphome/build_helpers/ninja_gen.py @@ -13,12 +13,27 @@ from pathlib import Path import sys from typing import TYPE_CHECKING +from esphome.build_helpers.ccache import effective_ccache_basedir from esphome.build_helpers.ninja import ( escape as _e, quote_path as _q, shell_token as _shell_token, ) +from esphome.build_helpers.pch import ( + PCH_HEADER_NAME, + mark_pch_emitted, + pch_checksum, + pch_consumer_escalation, + pch_degraded, + pch_disabled_degraded, + pch_enabled, + pch_header_text, + pch_probe_args, + pch_strict, +) +from esphome.core import CORE from esphome.framework_helpers import strip_win_long_path_prefix +from esphome.helpers import write_file_if_changed from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX if TYPE_CHECKING: @@ -107,6 +122,106 @@ def pch_rule_lines() -> list[str]: ] +def pch_edges( + lines: list[str], + build_dir: Path, + src_dir: Path, + headers: Sequence[str], + cxxflags: Sequence[str], + src_flags: Sequence[str], + ccache: str | None, + identity: Sequence[str], +) -> tuple[str, str] | None: + """Emit the precompiled header for the C++ src edges. + + ``headers`` are folded into one prefix header, ``src_flags`` are the + flags every src edge carries, and ``identity`` names what the compile + depends on beyond the headers and flags (versioned install paths). + Returns the ``cxx_override`` for ``compile_edges``, or None when the + build goes without a precompiled header. + """ + if not pch_enabled(): + pch_disabled_degraded() + return None + if any(tok.startswith("-include") for tok in cxxflags): + # $cxxflags expands first, so a user -include there means GCC would + # never load the .gch + _LOGGER.warning( + "A -include in build_flags prevents the precompiled header from " + "loading; compiling without it" + ) + pch_degraded("a user -include precedes the pch") + return None + pch_header = build_dir / PCH_HEADER_NAME + pch_text = pch_header_text(headers) + checksum = None + try: + if ccache: + # The .sum exists only for CCACHE_PCH_EXTSUM; ninja's depfile + # handles staleness. Strip resolved and raw build paths + # (symlinks) so identical configs share cache entries + flags_id = ( + " ".join(cxxflags) + .replace(effective_ccache_basedir(), "") + .replace(str(CORE.build_path), "") + ) + # The header text covers include order + checksum = pch_checksum(src_dir, headers, (pch_text, *identity, flags_id)) + except (OSError, UnicodeError) as err: + # Identity unknown: a stale cache entry must never be served + _LOGGER.warning( + "Could not establish the pch identity; compiling without it: %s", err + ) + pch_degraded(f"identity unknown: {err}") + return None + _LOGGER.info( + "Compiling with a precompiled header (set ESPHOME_PCH_ENABLE=0 to disable)" + ) + write_file_if_changed(pch_header, pch_text) + sum_path = build_dir / f"{PCH_HEADER_NAME}.gch.sum" + if checksum is not None: + # Generate-time stamp: a hand-run ninja can rebuild the .gch + # while this .sum lags + write_file_if_changed(sum_path, checksum + "\n") + else: + # A stale .sum from an earlier ccache run must not survive + sum_path.unlink(missing_ok=True) + gch = _e(f"{PCH_HEADER_NAME}.gch") + lines.append(f"build {gch}: pch {_e(pch_header)}") + if src_flags: + lines.append(f" flags = {' '.join(src_flags)}") + # Relative -include: absolute would break cross-device ccache. + # -Wno-error keeps a rejected .gch a warning under user -Werror; + # strict inverts it so any consumer rejection reds the build + # (rejection is per-process, so the probe alone cannot prove + # the consumers) + cxx_parts = [ + *src_flags, + f"-Winvalid-pch {pch_consumer_escalation()} -include {PCH_HEADER_NAME}", + ] + lines.append(f"srccxxflags = {' '.join(cxx_parts)}") + pch_dep = gch + if pch_strict(): + # Consumers wait on the probe stamp, so an unloadable .gch + # reds the build here instead of warning ~100 times + probe = " ".join(pch_probe_args(PCH_HEADER_NAME, source=os.devnull)) + lines.append("rule pchprobe") + # $out only expands in rule text, hence the inline stamp + lines.append( + f" command = $cxx $cxxflags $flags {probe}" + " && $python $buildtool touch $out" + ) + lines.append(" description = PCHPROBE $out") + # Runs when the .gch is (re)built; strict consumer -Werror + # covers a cached .gch this process cannot load + lines.append(f"build esphome_pch.probe: pchprobe {gch}") + if src_flags: + lines.append(f" flags = {' '.join(src_flags)}") + pch_dep = f"{gch} esphome_pch.probe" + mark_pch_emitted() + return ("$srccxxflags", pch_dep) + + def ar_rule_lines(ar: Path | str) -> list[str]: return [ "rule ar", diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index 755abe845b..457ea06cdd 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -20,7 +20,7 @@ from .const import KEY_HOST # force import gpio to register pin schema from .gpio import host_pin_to_code # noqa: F401 -# Guarded wrapper: build_src_flags reaches C/assembly edges too +# The curated headers the host build precompiles HOST_PCH_PREFIX = "esphome/core/pch_prefix.h" CODEOWNERS = ["@esphome/core", "@clydebarrow"] diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 9a8a2e4448..847dbee8e7 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -410,7 +410,7 @@ def test_write_project_pch_identity_unknown_skips_pch( paths = _make_framework(tmp_path) _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") with patch( - "esphome.build_gen.arduino8266.pch_checksum", + "esphome.build_helpers.ninja_gen.pch_checksum", side_effect=OSError("stat failed"), ): content = _write_ninja(paths, ccache="/usr/bin/ccache") diff --git a/tests/unit_tests/build_gen/test_host.py b/tests/unit_tests/build_gen/test_host.py index d7fa73e897..64db7b31b0 100644 --- a/tests/unit_tests/build_gen/test_host.py +++ b/tests/unit_tests/build_gen/test_host.py @@ -207,7 +207,7 @@ def test_write_project_emits_every_source_kind(tmp_path: Path) -> None: ) assert "build obj/src/x.S.o: aspp " in ninja assert "build obj/src/y.s.o: asm " in ninja - assert "h.h" not in ninja + assert _e(src / "h.h") not in ninja # __FILE__ stays relative to the build path, as it was under PlatformIO, # whether the source arrives absolute or rewritten by ccache sep = os.sep @@ -232,6 +232,58 @@ def test_write_project_emits_every_source_kind(tmp_path: Path) -> None: assert changed is False +def test_write_project_precompiles_the_core_headers(tmp_path: Path) -> None: + """C++ src edges load one precompiled prefix; C and assembly do not.""" + src = _make_src(tmp_path, "main.cpp", "esphome/core/a.c", "x.S") + _, ninja = _render() + build_dir = tmp_path / ".pioenvs" / "dev" + header = build_dir / "esphome_pch.h" + assert header.read_text() == ( + '#include "esphome/core/pch_prefix.h"\n#include "esphome/core/defines.h"\n' + ) + assert "rule pch\n" in ninja + assert f"build esphome_pch.h.gch: pch {_e(header)}\n" in ninja + assert "srccxxflags = -Winvalid-pch " in ninja + assert ( + f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')} | esphome_pch.h.gch\n" + " flags = $srccxxflags\n" + ) in ninja + assert ( + f"build obj/src/esphome/core/a.c.o: c {_e(src / 'esphome/core/a.c')}\n" + ) in ninja + assert f"build obj/src/x.S.o: aspp {_e(src / 'x.S')}\n" in ninja + # Without ccache nothing reads the checksum sidecar + assert not (build_dir / "esphome_pch.h.gch.sum").exists() + + +def test_write_project_pch_checksum_with_ccache(tmp_path: Path) -> None: + _make_src(tmp_path, "main.cpp") + _render(ccache="/usr/bin/ccache") + assert (tmp_path / ".pioenvs" / "dev" / "esphome_pch.h.gch.sum").is_file() + + +def test_write_project_pch_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + src = _make_src(tmp_path, "main.cpp") + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + _, ninja = _render() + assert "esphome_pch" not in ninja + assert f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')}\n" in ninja + + +def test_write_project_pch_skipped_for_a_user_force_include( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A -include in the build flags precedes the prefix, so the compiler + would never load the precompiled header.""" + _make_src(tmp_path, "main.cpp") + CORE.build_flags = {"-include pre.h"} + _, ninja = _render() + assert "esphome_pch" not in ninja + assert "prevents the precompiled header" in caplog.text + + def test_write_project_without_ccache(tmp_path: Path) -> None: _make_src(tmp_path, "main.cpp") _changed, ninja = _render()