[esp8266] Precompile the shared prefix header for src compiles

esphome/arduino8266/framework.py

@@ -19,6 +19,7 @@ from typing import NamedTuple
 
 from esphome.build_helpers.ccache import ccache_defaults_env
 from esphome.build_helpers.ninja import find_ninja
+from esphome.build_helpers.pch import ccache_pch_env
 from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path
 from esphome.core import EsphomeError, Version
 from esphome.framework_helpers import str_to_lst_of_str
@@ -161,4 +162,6 @@ def ccache_env(ccache: str | None) -> dict[str, str]:
     """
     if ccache is None:
         return {}
-    return ccache_defaults_env(get_arduino8266_tools_path() / "ccache")
+    env = ccache_defaults_env(get_arduino8266_tools_path() / "ccache")
+    env.update(ccache_pch_env())
+    return env

esphome/build_gen/arduino8266.py

@@ -32,6 +32,13 @@ from esphome.build_helpers.ninja import (
     quote_path as _q,
     shell_token as _shell_token,
 )
+from esphome.build_helpers.pch import (
+    PCH_CORE_HEADER,
+    PCH_HEADER_NAME,
+    pch_checksum,
+    pch_enabled,
+    pch_header_text,
+)
 from esphome.components.esp8266 import build_surgery
 from esphome.components.esp8266.boards import (
     BOARDS,
@@ -874,18 +881,26 @@ def _ninja_compile_edges(
     root: Path,
     group: str,
     flags: str = "",
+    cxx_flags: str = "",
+    cxx_implicit: str = "",
 ) -> list[str]:
-    """Emit compile edges for ``sources``; return the object paths."""
+    """Emit compile edges for ``sources``; return the object paths.
+
+    ``cxx_flags``/``cxx_implicit`` override ``flags`` and add an implicit
+    dependency on C++ edges only (used for the precompiled header).
+    """
     objects = []
     for src in sources:
         rel = src.relative_to(root).as_posix()
         obj = f"obj/{group}/{rel}.o"
         escaped_obj = _e(obj)
-        lines.append(
-            f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}"
-        )
-        if flags:
-            lines.append(f"  flags = {flags}")
+        kind = SOURCE_KIND_FOR_SUFFIX[src.suffix]
+        is_cxx = kind == "cxx"
+        implicit = f" | {cxx_implicit}" if is_cxx and cxx_implicit else ""
+        lines.append(f"build {escaped_obj}: {kind} {_e(src)}{implicit}")
+        edge_flags = cxx_flags if is_cxx and cxx_flags else flags
+        if edge_flags:
+            lines.append(f"  flags = {edge_flags}")
         # Escaped once here: the returned paths only ever appear in build
         # statements (archive/link inputs), which use ninja escaping.
         objects.append(escaped_obj)
@@ -1087,6 +1102,13 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         "  depfile = $out.d",
         "  deps = gcc",
         "  description = AS $out",
+        # No $ccache: the .gch is compiled once per build dir and ccache
+        # cannot cache it usefully (its bytes embed build-dir paths)
+        "rule pch",
+        "  command = $cxx -MMD -MF $out.d -x c++-header $cxxflags $flags -c $in -o $out",
+        "  depfile = $out.d",
+        "  deps = gcc",
+        "  description = PCH $out",
         # Plain assembler, as SCons's ASCOM: no preprocessor, so no
         # depfile and no $flags (defines/includes) either
         "rule asm",
@@ -1175,7 +1197,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
     # One source of truth with the PlatformIO path: esp8266/__init__ pins
     # build_src_flags (the throw_stubs force-include); -include paths
     # resolve against the source root
-    src_parts: list[str] = []
+    src_other: list[str] = []
+    src_includes: list[str] = []
     src_it = iter(
         lex_build_flags(_pio_option("build_src_flags", ""), "build_src_flags")
     )
@@ -1186,15 +1209,57 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
                 raise EsphomeError(
                     "build_src_flags has a trailing '-include' with no header"
                 )
-            src_parts.append(f"-include {_q(src_dir / header)}")
+            src_includes.append(header)
         else:
-            src_parts.append(_shell_token(tok))
-    src_extra = " ".join(src_parts)
+            src_other.append(_shell_token(tok))
+    include_flags = [f"-include {_q(src_dir / h)}" for h in src_includes]
     # One shared variable instead of repeating the flags line on every src
     # edge (hundreds of edges in a real project)
-    lines.append(f"srcflags = {src_extra}")
+    lines.append(f"srcflags = {' '.join(src_other + include_flags)}")
+    src_cxx_flags = None
+    src_cxx_implicit = ""
+    if pch_enabled():
+        # C++ src edges swap the force-includes for one precompiled prefix
+        # header holding the same content plus defines.h; C and assembly
+        # edges keep srcflags (a .gch is a C++ artifact)
+        pch_header = build_dir / PCH_HEADER_NAME
+        pch_includes = (*src_includes, PCH_CORE_HEADER)
+        write_file_if_changed(pch_header, pch_header_text(pch_includes))
+        if ccache:
+            # The .sum sidecar only exists for CCACHE_PCH_EXTSUM; ninja's
+            # depfile handles staleness. Mirror CCACHE_BASEDIR: strip the
+            # per-device build path so identically-configured devices
+            # produce identical .sum files and share cache entries
+            flags_id = " ".join(cxxflags).replace(
+                str(Path(CORE.build_path).resolve()), ""
+            )
+            checksum = pch_checksum(
+                src_dir,
+                pch_includes,
+                (str(paths.framework), str(paths.toolchain), flags_id),
+            )
+            write_file_if_changed(
+                build_dir / f"{PCH_HEADER_NAME}.gch.sum", checksum + "\n"
+            )
+        gch = _e(f"{PCH_HEADER_NAME}.gch")
+        lines.append(f"build {gch}: pch {_e(pch_header)}")
+        if src_other:
+            lines.append(f"  flags = {' '.join(src_other)}")
+        # Relative -include (resolved from the ninja cwd, where the header
+        # lives): an absolute path would put the per-device build path on
+        # every compile command and defeat cross-device ccache sharing
+        cxx_parts = src_other + [f"-include {PCH_HEADER_NAME}"]
+        lines.append(f"srccxxflags = {' '.join(cxx_parts)}")
+        src_cxx_flags = "$srccxxflags"
+        src_cxx_implicit = gch
     src_objs = _ninja_compile_edges(
-        lines, _collect_sources(src_dir), src_dir, "src", flags="$srcflags"
+        lines,
+        _collect_sources(src_dir),
+        src_dir,
+        "src",
+        flags="$srcflags",
+        cxx_flags=src_cxx_flags,
+        cxx_implicit=src_cxx_implicit,
     )
 
     ld_deps = [f"ld/{_COMMON_LD_NAME}"]

esphome/build_helpers/pch.py

@@ -0,0 +1,105 @@
+"""Shared precompiled-header policy for the build backends.
+
+Safe by construction when the prefix header mirrors what the TUs already
+include first (ESP8266); a backend may instead inject a curated set of
+self-contained core headers (ESP-IDF).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import hashlib
+import os
+from pathlib import Path
+import posixpath
+import re
+
+from esphome.build_helpers.ccache import parse_enable_env
+
+# The header and its .gch/.sum sidecars live in the build directory.
+PCH_HEADER_NAME = "esphome_pch.h"
+
+# Last include of the ESP8266 prefix header.
+PCH_CORE_HEADER = "esphome/core/defines.h"
+
+# ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the
+# .sum sidecar instead of the .gch bytes, which are not reproducible.
+# Keep in sync with the literals in components/esp8266/pch.py.script.
+_CCACHE_PCH_ENV = {
+    "CCACHE_SLOPPINESS": "pch_defines,time_macros",
+    "CCACHE_PCH_EXTSUM": "true",
+}
+
+_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+"([^"]+)"', re.MULTILINE)
+
+
+def pch_enabled() -> bool:
+    """Precompiled-header knob: default on, ``ESPHOME_PCH_ENABLE=0`` opts out."""
+    return parse_enable_env("ESPHOME_PCH_ENABLE") is not False
+
+
+def ccache_pch_env() -> dict[str, str]:
+    """ccache settings required to cache compiles that consume the .gch;
+    empty when the pch is disabled. User-set values win."""
+    if not pch_enabled():
+        return {}
+    return {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ}
+
+
+def pch_header_text(include_headers: Iterable[str]) -> str:
+    """The prefix-header source: exactly these includes, in order."""
+    return "".join(f'#include "{name}"\n' for name in include_headers)
+
+
+def quoted_includes(path: Path) -> tuple[str, ...]:
+    """The quoted #include targets of one file ([] when unreadable)."""
+    try:
+        data = path.read_bytes()
+    except OSError:
+        return ()
+    return tuple(m.decode() for m in _INCLUDE_RE.findall(data))
+
+
+def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]:
+    """Quoted-include closure of ``roots`` (src-relative names).
+
+    Resolves each include against the includer's directory first, then the
+    src root, matching the compiler's quoted-include lookup. Names that do
+    not resolve under ``src_dir`` end the walk; they live in versioned
+    framework/toolchain installs the caller identifies separately.
+    Over-approximates (no #ifdef evaluation) — the safe direction for
+    cache invalidation.
+    """
+    seen: set[str] = set()
+    stack: list[tuple[str, str]] = [(name, "") for name in roots]
+    while stack:
+        name, from_dir = stack.pop()
+        for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
+            rel = posixpath.normpath(candidate)
+            if not rel.startswith("..") and (src_dir / rel).is_file():
+                break
+        else:
+            continue
+        if rel in seen:
+            continue
+        seen.add(rel)
+        parent = posixpath.dirname(rel)
+        stack.extend((inc, parent) for inc in quoted_includes(src_dir / rel))
+    return seen
+
+
+def pch_checksum(
+    src_dir: Path, include_headers: Iterable[str], extra: Iterable[str]
+) -> str:
+    """Digest standing in for the .gch in ccache's hash: the include closure
+    of the prefix header plus caller-supplied identity strings (versioned
+    install paths, flags)."""
+    digest = hashlib.sha256()
+    for name in sorted(include_closure(src_dir, include_headers)):
+        digest.update(name.encode())
+        digest.update((src_dir / name).read_bytes())
+        digest.update(b"\0")
+    for item in extra:
+        digest.update(item.encode())
+        digest.update(b"\0")
+    return digest.hexdigest()

esphome/components/esp8266/__init__.py

@@ -6,6 +6,7 @@ import subprocess
 import time
 from typing import Any
 
+from esphome.build_helpers.pch import pch_enabled
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.const import (
@@ -420,6 +421,9 @@ async def to_code(config: ConfigType) -> None:
         ]
         if not enable_scanf_float:
             extra_scripts.append("pre:remove_float_scanf.py")
+        # Generation-time gate: the script itself has no enable check
+        if pch_enabled():
+            extra_scripts.append("post:pch.py")
         extra_scripts.append("post:post_build.py")
         cg.add_platformio_option("extra_scripts", extra_scripts)
 
@@ -575,6 +579,7 @@ def copy_files() -> None:
     dir = Path(__file__).parent
     for script in (
         "post_build",
+        "pch",
         "testing_mode",
         "exclude_updater",
         "exclude_waveform",

esphome/components/esp8266/pch.py.script

@@ -0,0 +1,110 @@
+import hashlib
+import os
+from pathlib import Path
+import shlex
+import subprocess
+
+# pylint: disable=E0602
+Import("env", "projenv")  # noqa: F821
+
+# Precompile the src force-includes plus defines.h (which pulls in
+# Arduino.h) and force-include the result into C++ src compiles only; those
+# TUs already include this content first, so their preprocessed output is
+# unchanged. Post script: the final src flags exist, nothing compiled yet.
+# Registration is gated host-side (esp8266/__init__.py, pch_enabled()).
+# Keep the header tail and ccache values in sync with build_helpers/pch.py.
+
+
+def _esp8266_setup_pch() -> None:
+    # Project root, not $BUILD_DIR: SCons compiles run with the project dir
+    # as cwd, so "-include esphome_pch.h" resolves here as a relative path.
+    # An absolute path would put the per-device build path on every compile
+    # command and defeat cross-device ccache sharing.
+    proj_dir = Path(env.subst("$PROJECT_DIR"))  # noqa: F821
+    src_dir = Path(env.subst("$PROJECT_SRC_DIR"))  # noqa: F821
+    header = proj_dir / "esphome_pch.h"
+    gch = Path(f"{header}.gch")
+    sum_path = Path(f"{gch}.sum")
+
+    cxx = projenv.subst("$CXX")  # noqa: F821
+    # The header holds the -include entries itself, so the .gch compile must
+    # not see them; consumers keep theirs, which the .gch then satisfies.
+    flags = []
+    include_headers = []
+    flag_it = iter(shlex.split(projenv.subst("$CXXFLAGS $CCFLAGS $_CCCOMCOM")))  # noqa: F821
+    for tok in flag_it:
+        if tok == "-include":
+            include_headers.append(next(flag_it, ""))
+        else:
+            flags.append(tok)
+    content = "".join(
+        f'#include "{name}"\n' for name in (*include_headers, "esphome/core/defines.h")
+    )
+
+    digest = hashlib.sha256()
+    digest.update(content.encode())
+    digest.update(cxx.encode())
+    # Mirror CCACHE_BASEDIR: strip the per-device build path so identical
+    # configs produce identical .sum files and share cache entries
+    flags_id = " ".join(flags)
+    if basedir := os.environ.get("CCACHE_BASEDIR"):
+        flags_id = flags_id.replace(basedir, "")
+    digest.update(flags_id.encode())
+    # GCC never validates a .gch against its source headers, and PlatformIO
+    # package paths carry no version, so a platform bump must invalidate here
+    platform = env.PioPlatform()  # noqa: F821
+    for package in ("framework-arduinoespressif8266", "toolchain-xtensa"):
+        digest.update(str(platform.get_package_version(package)).encode())
+        digest.update(b"\0")
+    for name in (*include_headers, "esphome/core/defines.h", "esphome/core/macros.h"):
+        path = Path(name) if Path(name).is_absolute() else src_dir / name
+        try:
+            digest.update(path.read_bytes())
+        except OSError:
+            digest.update(b"unreadable")
+        digest.update(b"\0")
+    checksum = digest.hexdigest()
+
+    # The ccache .sum sidecar doubles as the freshness stamp
+    if (
+        not gch.is_file()
+        or not sum_path.is_file()
+        or (sum_path.read_text(encoding="utf-8").strip() != checksum)
+    ):
+        failed_marker = Path(f"{gch}.failed")
+        if (
+            failed_marker.is_file()
+            and failed_marker.read_text(encoding="utf-8").strip() == checksum
+        ):
+            return
+        header.write_text(content, encoding="utf-8")
+        result = subprocess.run(  # noqa: PLW1510
+            [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            print("ESPHome: precompiled header failed; compiling without it")
+            print(result.stderr)
+            # Skip retries until a flag/header/platform change alters the checksum
+            failed_marker.write_text(checksum + "\n", encoding="utf-8")
+            return
+        failed_marker.unlink(missing_ok=True)
+        sum_path.write_text(checksum + "\n", encoding="utf-8")
+
+    # Scoped to src compiles: framework/library TUs never consume the .gch
+    # and keep strict ccache hashing. User-set values win.
+    for key, value in (
+        ("CCACHE_SLOPPINESS", "pch_defines,time_macros"),
+        ("CCACHE_PCH_EXTSUM", "true"),
+    ):
+        if key not in os.environ:
+            projenv["ENV"][key] = value  # noqa: F821
+
+    # Prepended so it is processed before the build_src_flags -include
+    # entries: GCC only uses a .gch while no other tokens have been seen.
+    projenv.Prepend(CXXFLAGS=["-include", header.name])  # noqa: F821
+    print("ESPHome: Compiling with precompiled header")
+
+
+_esp8266_setup_pch()

tests/unit_tests/build_gen/test_arduino8266.py

@@ -364,7 +364,50 @@ def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None:
         line for line in content.splitlines() if line.startswith("  flags = ")
     ]
     assert flags_lines
-    assert all(line == "  flags = $srcflags" for line in flags_lines)
+    # C++ src edges consume the precompiled header; C/assembly keep srcflags
+    assert set(flags_lines) == {"  flags = $srcflags", "  flags = $srccxxflags"}
+
+
+def test_write_project_pch(tmp_path: Path) -> None:
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    content = _write_ninja(paths, ccache="/usr/bin/ccache")
+    build_dir = CORE.relative_pioenvs_path(CORE.name)
+    assert "rule pch" in content
+    assert "build esphome_pch.h.gch: pch" in content
+    for line in content.splitlines():
+        # C++ edges wait on the .gch; the C edge must not reference it
+        if line.startswith("build obj/src/main.cpp.o:"):
+            assert line.endswith("| esphome_pch.h.gch")
+        if line.startswith("build obj/src/esphome/vendor.c.o:"):
+            assert "esphome_pch" not in line
+    assert (build_dir / "esphome_pch.h").read_text().splitlines() == [
+        '#include "esphome/components/esp8266/throw_stubs.h"',
+        '#include "esphome/core/defines.h"',
+    ]
+    assert (build_dir / "esphome_pch.h.gch.sum").read_text().strip()
+
+
+def test_write_project_pch_sum_only_with_ccache(tmp_path: Path) -> None:
+    """The .sum sidecar exists solely for ccache; skip it when disabled."""
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    content = _write_ninja(paths)
+    build_dir = CORE.relative_pioenvs_path(CORE.name)
+    assert "build esphome_pch.h.gch: pch" in content
+    assert not (build_dir / "esphome_pch.h.gch.sum").exists()
+
+
+def test_write_project_pch_disabled(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    content = _write_ninja(paths)
+    assert "esphome_pch" not in content
+    assert "srccxxflags" not in content
+    assert "  flags = $srcflags" in content
 
 
 def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None:
@@ -1711,3 +1754,22 @@ def test_write_project_rejects_spaced_ldscript_override(tmp_path: Path) -> None:
     _set_flags()
     with pytest.raises(EsphomeError, match="Invalid flash linker script name"):
         arduino8266.write_project(paths, None)
+
+
+def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None:
+    """Regression: the -include stays relative and the .sum carries no
+    per-device path, or cross-device ccache sharing breaks."""
+    paths = _make_framework(tmp_path / "shared")
+    sums = []
+    for name in ("dev_a", "dev_b"):
+        CORE.name = name
+        CORE.build_path = tmp_path / name
+        _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+        content = _write_ninja(paths, ccache="/usr/bin/ccache")
+        assert "srccxxflags = -include esphome_pch.h" in content
+        sums.append(
+            (
+                CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum"
+            ).read_text()
+        )
+    assert sums[0] == sums[1]

tests/unit_tests/test_arduino8266_toolchain.py

@@ -629,3 +629,25 @@ def test_get_idedata_accepts_preresolved_ccache() -> None:
         assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True}
     mock_resolve.assert_not_called()
     assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache"
+
+
+def test_ccache_env_includes_pch_settings() -> None:
+    """The native build exports the ccache settings the pch needs."""
+    with patch.dict(os.environ, {}, clear=True):
+        env = framework.ccache_env("/usr/bin/ccache")
+    assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
+    assert env["CCACHE_PCH_EXTSUM"] == "true"
+
+
+def test_ccache_env_pch_disabled() -> None:
+    with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True):
+        env = framework.ccache_env("/usr/bin/ccache")
+    assert "CCACHE_SLOPPINESS" not in env
+    assert "CCACHE_PCH_EXTSUM" not in env
+
+
+def test_ccache_env_respects_user_sloppiness() -> None:
+    with patch.dict(os.environ, {"CCACHE_SLOPPINESS": "locale"}, clear=True):
+        env = framework.ccache_env("/usr/bin/ccache")
+    assert "CCACHE_SLOPPINESS" not in env
+    assert env["CCACHE_PCH_EXTSUM"] == "true"