Merge branch 'esp32-pio-pch' into nrf52-pch

esphome/build_gen/arduino8266.py

@@ -25,7 +25,6 @@ import subprocess
 from typing import TYPE_CHECKING, NamedTuple
 
 from esphome.arduino8266.framework import toolchain_tool
-from esphome.build_helpers.ccache import effective_ccache_basedir
 from esphome.build_helpers.idedata import is_joined_include
 from esphome.build_helpers.ninja import (
     escape as _e,
@@ -38,22 +37,11 @@ from esphome.build_helpers.ninja_gen import (
     compile_edges,
     compile_rule_lines,
     library_edges,
+    pch_edges,
     pch_rule_lines,
     tool_lines,
 )
-from esphome.build_helpers.pch import (
-    PCH_CORE_HEADER,
-    PCH_HEADER_NAME,
-    mark_pch_emitted,
-    pch_checksum,
-    pch_consumer_escalation,
-    pch_degraded,
-    pch_disabled_degraded,
-    pch_enabled,
-    pch_header_text,
-    pch_probe_args,
-    pch_strict,
-)
+from esphome.build_helpers.pch import PCH_CORE_HEADER
 from esphome.components.esp8266 import build_surgery
 from esphome.components.esp8266.boards import (
     BOARDS,
@@ -1151,99 +1139,16 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
     # One shared variable instead of repeating the flags line on every src
     # edge (hundreds of edges in a real project)
     lines.append(f"srcflags = {' '.join(src_other + include_flags)}")
-    src_cxx_override = None
-    if pch_enabled() and any(tok.startswith("-include") for tok in flag_sets.cxxflags):
-        # $cxxflags expands first, so a user -include there means GCC would
-        # never load the .gch
-        _LOGGER.warning(
-            "A -include in build_flags prevents the precompiled header from "
-            "loading; compiling without it"
-        )
-        pch_degraded("a user -include precedes the pch")
-    elif pch_enabled():
-        # C++ src edges swap the force-includes for one precompiled prefix
-        # header (same content plus defines.h); C/assembly keep srcflags
-        pch_header = build_dir / PCH_HEADER_NAME
-        pch_includes = (*src_includes, PCH_CORE_HEADER)
-        pch_text = pch_header_text(pch_includes)
-        checksum = None
-        try:
-            if ccache:
-                # The .sum exists only for CCACHE_PCH_EXTSUM; ninja's depfile
-                # handles staleness. Strip resolved and raw build paths
-                # (symlinks) so identical configs share cache entries
-                flags_id = (
-                    " ".join(flag_sets.cxxflags)
-                    .replace(effective_ccache_basedir(), "")
-                    .replace(str(CORE.build_path), "")
-                )
-                # The header text covers include order
-                checksum = pch_checksum(
-                    src_dir,
-                    pch_includes,
-                    (
-                        pch_text,
-                        str(paths.framework),
-                        str(paths.toolchain),
-                        flags_id,
-                    ),
-                )
-        except (OSError, UnicodeError) as err:
-            # Identity unknown: a stale cache entry must never be served
-            _LOGGER.warning(
-                "Could not establish the pch identity; compiling without it: %s", err
-            )
-            pch_degraded(f"identity unknown: {err}")
-        else:
-            _LOGGER.info(
-                "Compiling with a precompiled header "
-                "(set ESPHOME_PCH_ENABLE=0 to disable)"
-            )
-            write_file_if_changed(pch_header, pch_text)
-            sum_path = build_dir / f"{PCH_HEADER_NAME}.gch.sum"
-            if checksum is not None:
-                # Generate-time stamp: a hand-run ninja can rebuild the .gch
-                # while this .sum lags
-                write_file_if_changed(sum_path, checksum + "\n")
-            else:
-                # A stale .sum from an earlier ccache run must not survive
-                sum_path.unlink(missing_ok=True)
-            gch = _e(f"{PCH_HEADER_NAME}.gch")
-            lines.append(f"build {gch}: pch {_e(pch_header)}")
-            if src_other:
-                lines.append(f"  flags = {' '.join(src_other)}")
-            # Relative -include: absolute would break cross-device ccache.
-            # -Wno-error keeps a rejected .gch a warning under user -Werror;
-            # strict inverts it so any consumer rejection reds the build
-            # (rejection is per-process, so the probe alone cannot prove
-            # the consumers)
-            escalation = pch_consumer_escalation()
-            cxx_parts = src_other + [
-                f"-Winvalid-pch {escalation} -include {PCH_HEADER_NAME}"
-            ]
-            lines.append(f"srccxxflags = {' '.join(cxx_parts)}")
-            pch_dep = gch
-            if pch_strict():
-                # Consumers wait on the probe stamp, so an unloadable .gch
-                # reds the build here instead of warning ~100 times
-                probe = " ".join(pch_probe_args(PCH_HEADER_NAME, source=os.devnull))
-                lines.append("rule pchprobe")
-                # $out only expands in rule text, hence the inline stamp
-                lines.append(
-                    f"  command = $cxx $cxxflags $flags {probe}"
-                    " && $python $buildtool touch $out"
-                )
-                lines.append("  description = PCHPROBE $out")
-                # Runs when the .gch is (re)built; strict consumer -Werror
-                # covers a cached .gch this process cannot load
-                lines.append(f"build esphome_pch.probe: pchprobe {gch}")
-                if src_other:
-                    lines.append(f"  flags = {' '.join(src_other)}")
-                pch_dep = f"{gch} esphome_pch.probe"
-            src_cxx_override = ("$srccxxflags", pch_dep)
-            mark_pch_emitted()
-    else:
-        pch_disabled_degraded()
+    src_cxx_override = pch_edges(
+        lines,
+        build_dir,
+        src_dir,
+        (*src_includes, PCH_CORE_HEADER),
+        flag_sets.cxxflags,
+        src_other,
+        ccache,
+        (str(paths.framework), str(paths.toolchain)),
+    )
     src_objs = compile_edges(
         lines,
         collect_sources(src_dir),

esphome/build_gen/host.py

@@ -28,8 +28,11 @@ from esphome.build_helpers.ninja_gen import (
     compile_edges,
     compile_rule_lines,
     library_edges,
+    pch_edges,
+    pch_rule_lines,
     tool_lines,
 )
+from esphome.build_helpers.pch import PCH_CORE_HEADER, PCH_PREFIX_HEADER
 from esphome.core import CORE, EsphomeError
 from esphome.framework_helpers import get_project_cxx_compile_flags
 from esphome.helpers import mkdir_p, write_file_if_changed
@@ -228,6 +231,7 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool:
     lines = [
         *tool_lines(compilers.cc, compilers.cxx, ccache),
         *compile_rule_lines(),
+        *pch_rule_lines(),
         "rule link",
         "  command = $cxx -o $out $linkflags @$out.rsp $archives $libdirflags $libflags",
         "  rspfile = $out.rsp",
@@ -251,7 +255,25 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool:
 
     archives, direct_objs = library_edges(lines, libraries)
 
-    src_objs = compile_edges(lines, collect_sources(src_dir), src_dir, "src")
+    # Host has no framework force-includes; the per-TU cost is the STL
+    # closure behind the core headers, so those are what is precompiled
+    src_cxx_override = pch_edges(
+        lines,
+        build_dir,
+        src_dir,
+        (PCH_PREFIX_HEADER, PCH_CORE_HEADER),
+        cxxflags,
+        (),
+        ccache,
+        compilers.cxx,
+    )
+    src_objs = compile_edges(
+        lines,
+        collect_sources(src_dir),
+        src_dir,
+        "src",
+        cxx_override=src_cxx_override,
+    )
     if not src_objs:
         raise EsphomeError(f"No source files found under {src_dir}")
 

esphome/build_helpers/ninja_gen.py

@@ -13,12 +13,27 @@ from pathlib import Path
 import sys
 from typing import TYPE_CHECKING
 
+from esphome.build_helpers.ccache import effective_ccache_basedir
 from esphome.build_helpers.ninja import (
     escape as _e,
     quote_path as _q,
     shell_token as _shell_token,
 )
+from esphome.build_helpers.pch import (
+    PCH_HEADER_NAME,
+    mark_pch_emitted,
+    pch_checksum,
+    pch_consumer_escalation,
+    pch_degraded,
+    pch_disabled_degraded,
+    pch_enabled,
+    pch_header_text,
+    pch_probe_args,
+    pch_strict,
+)
+from esphome.core import CORE
 from esphome.framework_helpers import strip_win_long_path_prefix
+from esphome.helpers import write_file_if_changed
 from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX
 
 if TYPE_CHECKING:
@@ -107,6 +122,106 @@ def pch_rule_lines() -> list[str]:
     ]
 
 
+def pch_edges(
+    lines: list[str],
+    build_dir: Path,
+    src_dir: Path,
+    headers: Sequence[str],
+    cxxflags: Sequence[str],
+    src_flags: Sequence[str],
+    ccache: str | None,
+    identity: Sequence[str],
+) -> tuple[str, str] | None:
+    """Emit the precompiled header for the C++ src edges.
+
+    ``headers`` are folded into one prefix header, ``src_flags`` are the
+    flags every src edge carries, and ``identity`` names what the compile
+    depends on beyond the headers and flags (versioned install paths).
+    Returns the ``cxx_override`` for ``compile_edges``, or None when the
+    build goes without a precompiled header.
+    """
+    if not pch_enabled():
+        pch_disabled_degraded()
+        return None
+    if any(tok.startswith("-include") for tok in cxxflags):
+        # $cxxflags expands first, so a user -include there means GCC would
+        # never load the .gch
+        _LOGGER.warning(
+            "A -include in build_flags prevents the precompiled header from "
+            "loading; compiling without it"
+        )
+        pch_degraded("a user -include precedes the pch")
+        return None
+    pch_header = build_dir / PCH_HEADER_NAME
+    pch_text = pch_header_text(headers)
+    checksum = None
+    try:
+        if ccache:
+            # The .sum exists only for CCACHE_PCH_EXTSUM; ninja's depfile
+            # handles staleness. Strip resolved and raw build paths
+            # (symlinks) so identical configs share cache entries
+            flags_id = (
+                " ".join(cxxflags)
+                .replace(effective_ccache_basedir(), "")
+                .replace(str(CORE.build_path), "")
+            )
+            # The header text covers include order
+            checksum = pch_checksum(src_dir, headers, (pch_text, *identity, flags_id))
+    except (OSError, UnicodeError) as err:
+        # Identity unknown: a stale cache entry must never be served
+        _LOGGER.warning(
+            "Could not establish the pch identity; compiling without it: %s", err
+        )
+        pch_degraded(f"identity unknown: {err}")
+        return None
+    _LOGGER.info(
+        "Compiling with a precompiled header (set ESPHOME_PCH_ENABLE=0 to disable)"
+    )
+    write_file_if_changed(pch_header, pch_text)
+    sum_path = build_dir / f"{PCH_HEADER_NAME}.gch.sum"
+    if checksum is not None:
+        # Generate-time stamp: a hand-run ninja can rebuild the .gch
+        # while this .sum lags
+        write_file_if_changed(sum_path, checksum + "\n")
+    else:
+        # A stale .sum from an earlier ccache run must not survive
+        sum_path.unlink(missing_ok=True)
+    gch = _e(f"{PCH_HEADER_NAME}.gch")
+    lines.append(f"build {gch}: pch {_e(pch_header)}")
+    if src_flags:
+        lines.append(f"  flags = {' '.join(src_flags)}")
+    # Relative -include: absolute would break cross-device ccache.
+    # -Wno-error keeps a rejected .gch a warning under user -Werror;
+    # strict inverts it so any consumer rejection reds the build
+    # (rejection is per-process, so the probe alone cannot prove
+    # the consumers)
+    cxx_parts = [
+        *src_flags,
+        f"-Winvalid-pch {pch_consumer_escalation()} -include {PCH_HEADER_NAME}",
+    ]
+    lines.append(f"srccxxflags = {' '.join(cxx_parts)}")
+    pch_dep = gch
+    if pch_strict():
+        # Consumers wait on the probe stamp, so an unloadable .gch
+        # reds the build here instead of warning ~100 times
+        probe = " ".join(pch_probe_args(PCH_HEADER_NAME, source=os.devnull))
+        lines.append("rule pchprobe")
+        # $out only expands in rule text, hence the inline stamp
+        lines.append(
+            f"  command = $cxx $cxxflags $flags {probe}"
+            " && $python $buildtool touch $out"
+        )
+        lines.append("  description = PCHPROBE $out")
+        # Runs when the .gch is (re)built; strict consumer -Werror
+        # covers a cached .gch this process cannot load
+        lines.append(f"build esphome_pch.probe: pchprobe {gch}")
+        if src_flags:
+            lines.append(f"  flags = {' '.join(src_flags)}")
+        pch_dep = f"{gch} esphome_pch.probe"
+    mark_pch_emitted()
+    return ("$srccxxflags", pch_dep)
+
+
 def ar_rule_lines(ar: Path | str) -> list[str]:
     return [
         "rule ar",

tests/unit_tests/build_gen/test_arduino8266.py

@@ -410,7 +410,7 @@ def test_write_project_pch_identity_unknown_skips_pch(
     paths = _make_framework(tmp_path)
     _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
     with patch(
-        "esphome.build_gen.arduino8266.pch_checksum",
+        "esphome.build_helpers.ninja_gen.pch_checksum",
         side_effect=OSError("stat failed"),
     ):
         content = _write_ninja(paths, ccache="/usr/bin/ccache")

tests/unit_tests/build_gen/test_host.py

@@ -207,7 +207,7 @@ def test_write_project_emits_every_source_kind(tmp_path: Path) -> None:
     )
     assert "build obj/src/x.S.o: aspp " in ninja
     assert "build obj/src/y.s.o: asm " in ninja
-    assert "h.h" not in ninja
+    assert _e(src / "h.h") not in ninja
     # __FILE__ stays relative to the build path, as it was under PlatformIO,
     # whether the source arrives absolute or rewritten by ccache
     sep = os.sep
@@ -232,6 +232,58 @@ def test_write_project_emits_every_source_kind(tmp_path: Path) -> None:
     assert changed is False
 
 
+def test_write_project_precompiles_the_core_headers(tmp_path: Path) -> None:
+    """C++ src edges load one precompiled prefix; C and assembly do not."""
+    src = _make_src(tmp_path, "main.cpp", "esphome/core/a.c", "x.S")
+    _, ninja = _render()
+    build_dir = tmp_path / ".pioenvs" / "dev"
+    header = build_dir / "esphome_pch.h"
+    assert header.read_text() == (
+        '#include "esphome/core/pch_prefix.h"\n#include "esphome/core/defines.h"\n'
+    )
+    assert "rule pch\n" in ninja
+    assert f"build esphome_pch.h.gch: pch {_e(header)}\n" in ninja
+    assert "srccxxflags = -Winvalid-pch " in ninja
+    assert (
+        f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')} | esphome_pch.h.gch\n"
+        "  flags = $srccxxflags\n"
+    ) in ninja
+    assert (
+        f"build obj/src/esphome/core/a.c.o: c {_e(src / 'esphome/core/a.c')}\n"
+    ) in ninja
+    assert f"build obj/src/x.S.o: aspp {_e(src / 'x.S')}\n" in ninja
+    # Without ccache nothing reads the checksum sidecar
+    assert not (build_dir / "esphome_pch.h.gch.sum").exists()
+
+
+def test_write_project_pch_checksum_with_ccache(tmp_path: Path) -> None:
+    _make_src(tmp_path, "main.cpp")
+    _render(ccache="/usr/bin/ccache")
+    assert (tmp_path / ".pioenvs" / "dev" / "esphome_pch.h.gch.sum").is_file()
+
+
+def test_write_project_pch_disabled(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    src = _make_src(tmp_path, "main.cpp")
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    _, ninja = _render()
+    assert "esphome_pch" not in ninja
+    assert f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')}\n" in ninja
+
+
+def test_write_project_pch_skipped_for_a_user_force_include(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A -include in the build flags precedes the prefix, so the compiler
+    would never load the precompiled header."""
+    _make_src(tmp_path, "main.cpp")
+    CORE.build_flags = {"-include pre.h"}
+    _, ninja = _render()
+    assert "esphome_pch" not in ninja
+    assert "prevents the precompiled header" in caplog.text
+
+
 def test_write_project_without_ccache(tmp_path: Path) -> None:
     _make_src(tmp_path, "main.cpp")
     _changed, ninja = _render()