[esp8266] Precompile the prefix header on the native toolchain (#19821)

.github/workflows/ci-docker.yml

@@ -196,6 +196,7 @@ jobs:
         # the default.
         id:
           - esp8266-arduino
+          - esp8266-arduino-native
           - esp32-arduino-platformio
           - esp32-arduino-esp-idf
           - esp32-idf-platformio

docker/test_configs/esp8266-arduino-native.yaml

@@ -0,0 +1,8 @@
+esphome:
+  name: docker-test-esp8266-native
+
+esp8266:
+  board: d1_mini
+  toolchain: arduino
+
+logger:

esphome/build_gen/arduino8266.py

@@ -25,6 +25,7 @@ import subprocess
 from typing import TYPE_CHECKING, NamedTuple
 
 from esphome.arduino8266.framework import toolchain_tool
+from esphome.build_helpers.idedata import is_joined_include
 from esphome.build_helpers.ninja import (
     escape as _e,
     quote_path as _q,
@@ -36,8 +37,11 @@ from esphome.build_helpers.ninja_gen import (
     compile_edges,
     compile_rule_lines,
     library_edges,
+    pch_edges,
+    pch_rule_lines,
     tool_lines,
 )
+from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS
 from esphome.components.esp8266 import build_surgery
 from esphome.components.esp8266.boards import (
     BOARDS,
@@ -1057,6 +1061,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         ),
         *compile_rule_lines(),
         *ar_rule_lines(toolchain_tool(paths.toolchain, "ar")),
+        *pch_rule_lines(),
         "rule link",
         "  command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group",
         "  rspfile = $out.rsp",
@@ -1111,7 +1116,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
     # One source of truth with the PlatformIO path: esp8266/__init__ pins
     # build_src_flags (the throw_stubs force-include); -include paths
     # resolve against the source root
-    src_parts: list[str] = []
+    src_other: list[str] = []
+    src_includes: list[str] = []
     src_it = iter(
         lex_build_flags(_pio_option("build_src_flags", ""), "build_src_flags")
     )
@@ -1122,15 +1128,32 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
                 raise EsphomeError(
                     "build_src_flags has a trailing '-include' with no header"
                 )
-            src_parts.append(f"-include {_q(src_dir / header)}")
+            src_includes.append(header)
+        elif is_joined_include(tok):
+            # Left in src_other it would precede the pch include
+            src_includes.append(tok[len("-include") :])
         else:
-            src_parts.append(_shell_token(tok))
-    src_extra = " ".join(src_parts)
+            src_other.append(_shell_token(tok))
+    include_flags = [f"-include {_q(src_dir / h)}" for h in src_includes]
     # One shared variable instead of repeating the flags line on every src
     # edge (hundreds of edges in a real project)
-    lines.append(f"srcflags = {src_extra}")
+    lines.append(f"srcflags = {' '.join(src_other + include_flags)}")
+    src_cxx_override = pch_edges(
+        lines,
+        build_dir,
+        src_dir,
+        (*src_includes, *PCH_DEFAULT_HEADERS),
+        flag_sets.cxxflags,
+        src_other,
+        (str(paths.framework), str(paths.toolchain)),
+    )
     src_objs = compile_edges(
-        lines, collect_sources(src_dir), src_dir, "src", flags="$srcflags"
+        lines,
+        collect_sources(src_dir),
+        src_dir,
+        "src",
+        flags="$srcflags",
+        cxx_override=src_cxx_override,
     )
 
     ld_deps = [f"ld/{_COMMON_LD_NAME}"]

esphome/build_helpers/ccache.py

@@ -102,15 +102,30 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
     return {k: v for k, v in defaults.items() if k not in os.environ}
 
 
+def effective_ccache_basedir() -> str:
+    """The prefix ccache strips from hashed paths: a usable user
+    CCACHE_BASEDIR, else the resolved build path."""
+    from esphome.core import CORE
+
+    raw = os.environ.get("CCACHE_BASEDIR")
+    if raw is not None and Path(raw).is_absolute() and len(Path(raw).parts) > 1:
+        return raw
+    return str(Path(CORE.build_path).resolve())
+
+
 def ccache_env(ccache: str | None, tools_cache: tuple[str, str]) -> dict[str, str]:
     """The ccache settings for a build subprocess (not os.environ).
 
     ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None when
     disabled; ``tools_cache`` is the backend's tools cache spec, which holds
-    its ccache dir.
+    its ccache dir. The pch settings include ``time_macros``: a cached
+    object can keep an older ``__DATE__`` or ``__TIME__``.
     """
     if ccache is None:
         return {}
+    from esphome.build_helpers.pch import ccache_pch_env
     from esphome.build_helpers.tools_cache import tools_cache_path
 
-    return ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache")
+    env = ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache")
+    env.update(ccache_pch_env())
+    return env

esphome/build_helpers/idedata.py

@@ -152,6 +152,15 @@ def _is_launcher(token: str) -> bool:
     return Path(token).stem.lower() in _LAUNCHER_STEMS
 
 
+def is_joined_include(tok: str) -> bool:
+    """The joined ``-includefoo.h`` spelling; excludes clang's -include-pch."""
+    return (
+        tok.startswith("-include")
+        and tok != "-include"
+        and not tok.startswith("-include-")
+    )
+
+
 def parse_entry(
     entry: dict, launcher: str | None = None
 ) -> tuple[str, list[str], list[str], list[str]]:

esphome/build_helpers/ninja_gen.py

@@ -13,12 +13,23 @@ from pathlib import Path
 import sys
 from typing import TYPE_CHECKING
 
+from esphome.build_helpers.idedata import is_joined_include
 from esphome.build_helpers.ninja import (
     escape as _e,
     quote_path as _q,
     shell_token as _shell_token,
 )
+from esphome.build_helpers.pch import (
+    PCH_GCH_NAME,
+    PCH_SUM_NAME,
+    log_pch_in_use,
+    pch_consumer_flags,
+    pch_enabled,
+    pch_identity,
+    write_pch_headers,
+)
 from esphome.framework_helpers import strip_win_long_path_prefix
+from esphome.helpers import write_file_if_changed
 from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX
 
 if TYPE_CHECKING:
@@ -94,6 +105,62 @@ def compile_rule_lines() -> list[str]:
     ]
 
 
+def pch_rule_lines() -> list[str]:
+    """The precompiled header rule, for a generator that emits one."""
+    return [
+        # No $ccache: the .gch embeds build dir paths
+        "rule pch",
+        "  command = $cxx -MMD -MF $out.d -x c++-header $cxxflags $flags -c $in -o $out",
+        "  depfile = $out.d",
+        "  deps = gcc",
+        "  description = PCH $out",
+    ]
+
+
+def pch_edges(
+    lines: list[str],
+    build_dir: Path,
+    src_dir: Path,
+    headers: Sequence[str],
+    cxxflags: Sequence[str],
+    src_flags: Sequence[str],
+    identity: Sequence[str],
+) -> tuple[str, str] | None:
+    """Emit the precompiled header for the C++ src edges.
+
+    ``headers`` are folded into one prefix header, ``src_flags`` are the
+    flags every src edge carries and ``identity`` names what else the
+    compile depends on, the compiler included. Returns the ``cxx_override``
+    for ``compile_edges``, or None without a pch.
+    """
+    if not pch_enabled():
+        return None
+    if any(
+        tok == "-include" or tok.startswith("--include") or is_joined_include(tok)
+        for tok in cxxflags
+    ):
+        # $cxxflags expands first and GCC only loads a .gch for the first
+        # -include
+        _LOGGER.warning(
+            "A -include in the compiler flags prevents the precompiled header from "
+            "loading; compiling without it"
+        )
+        return None
+    log_pch_in_use()
+    source = write_pch_headers(build_dir, headers)
+    write_file_if_changed(
+        build_dir / PCH_SUM_NAME,
+        pch_identity([*cxxflags, *src_flags], src_dir, tuple(headers), identity) + "\n",
+    )
+    gch = _e(PCH_GCH_NAME)
+    # The checksum file changes with anything the .gch depends on
+    lines.append(f"build {gch}: pch {_e(source)} | {_e(PCH_SUM_NAME)}")
+    if src_flags:
+        lines.append(f"  flags = {' '.join(src_flags)}")
+    lines.append(f"srccxxflags = {' '.join([*src_flags, *pch_consumer_flags()])}")
+    return ("$srccxxflags", gch)
+
+
 def ar_rule_lines(ar: Path | str) -> list[str]:
     return [
         "rule ar",
@@ -111,20 +178,26 @@ def compile_edges(
     group: str,
     flags: str = "",
     own_includes: str = "",
+    cxx_override: tuple[str, str] | None = None,
 ) -> list[str]:
-    """Emit compile edges for ``sources``; return the object paths."""
+    """Emit compile edges for ``sources``; return the object paths.
+
+    ``cxx_override`` is a (flags, implicit-dep) pair applied to C++ edges
+    only, replacing ``flags`` (used for the precompiled header).
+    """
     objects = []
     for src in sources:
         rel = src.relative_to(root).as_posix()
         obj = f"obj/{group}/{rel}.o"
         escaped_obj = _e(obj)
-        lines.append(
-            f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}"
-        )
+        kind = SOURCE_KIND_FOR_SUFFIX[src.suffix]
+        override = cxx_override if kind == "cxx" else None
+        implicit = f" | {override[1]}" if override else ""
+        lines.append(f"build {escaped_obj}: {kind} {_e(src)}{implicit}")
         if own_includes:
             lines.append(f"  own_includes = {own_includes}")
-        if flags:
-            lines.append(f"  flags = {flags}")
+        if edge_flags := override[0] if override else flags:
+            lines.append(f"  flags = {edge_flags}")
         # Escaped once here: the returned paths only ever appear in build
         # statements (archive/link inputs), which use ninja escaping
         objects.append(escaped_obj)

esphome/build_helpers/pch.py

@@ -10,10 +10,29 @@ from pathlib import Path
 import posixpath
 import re
 
-from esphome.build_helpers.ccache import parse_enable_env
+from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env
+from esphome.helpers import write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
 
+# The header and its sidecars live in the build directory
+PCH_HEADER_NAME = "esphome_pch.h"
+PCH_GCH_NAME = f"{PCH_HEADER_NAME}.gch"
+# ccache hashes this instead of the .gch; also the freshness stamp
+PCH_SUM_NAME = f"{PCH_GCH_NAME}.sum"
+# The include list the .gch is compiled from
+PCH_SOURCE_NAME = "esphome_pch_src.h"
+
+# GCC can skip a .gch without a diagnostic and read the header of the same
+# name, so that header is an error. Other tools get the include list.
+PCH_GUARD_TEXT = f"""\
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTELLISENSE__)
+#error "The precompiled header was not loaded"
+#else
+#include "{PCH_SOURCE_NAME}"
+#endif
+"""
+
 # The core headers every backend precompiles
 PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",)
 
@@ -29,6 +48,12 @@ def pch_enabled() -> bool:
     return parse_enable_env("ESPHOME_PCH_ENABLE") is not False
 
 
+def pch_consumer_flags() -> list[str]:
+    """Flags a C++ src compile loads the pch with. The -include stays
+    relative: an absolute path would enter the ccache key."""
+    return ["-Winvalid-pch", "-Werror=invalid-pch", "-include", PCH_HEADER_NAME]
+
+
 def ccache_pch_env() -> dict[str, str]:
     """What ccache needs to cache compiles that load a .gch, added to what
     the user already set."""
@@ -46,6 +71,20 @@ def ccache_pch_env() -> dict[str, str]:
     return env
 
 
+def pch_header_text(include_headers: Iterable[str]) -> str:
+    """The prefix-header source: exactly these includes, in order."""
+    return "".join(f'#include "{name}"\n' for name in include_headers)
+
+
+def write_pch_headers(build_dir: Path, include_headers: Iterable[str]) -> Path:
+    """Write the guard header and the include list; return the latter,
+    which is what the .gch compiles from."""
+    write_file_if_changed(build_dir / PCH_HEADER_NAME, PCH_GUARD_TEXT)
+    source = build_dir / PCH_SOURCE_NAME
+    write_file_if_changed(source, pch_header_text(include_headers))
+    return source
+
+
 def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
     """Include closure of ``roots``: src-relative name -> contents.
 
@@ -87,6 +126,27 @@ def pch_checksum(
     return digest.hexdigest()
 
 
+def pch_identity(
+    tokens: Iterable[str],
+    src_dir: Path,
+    include_headers: tuple[str, ...],
+    extra: Iterable[str],
+) -> str:
+    """The .sum digest: include closure, header text, ``extra`` and the
+    compile flags with the build path stripped, as ccache does."""
+    from esphome.core import CORE
+
+    flags = (
+        " ".join(tokens)
+        .replace(str(CORE.build_path), "")
+        .replace(effective_ccache_basedir(), "")
+    )
+    # The closure is sorted, so header order only enters via the text
+    return pch_checksum(
+        src_dir, include_headers, (pch_header_text(include_headers), *extra, flags)
+    )
+
+
 _DISABLE_HINT = " (set ESPHOME_PCH_ENABLE=0 to disable)"
 
 

tests/unit_tests/build_gen/test_arduino8266.py

@@ -364,7 +364,87 @@ def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None:
         line for line in content.splitlines() if line.startswith("  flags = ")
     ]
     assert flags_lines
-    assert all(line == "  flags = $srcflags" for line in flags_lines)
+    # C++ src edges consume the precompiled header; C/assembly keep srcflags
+    assert set(flags_lines) == {"  flags = $srcflags", "  flags = $srccxxflags"}
+
+
+def test_write_project_pch(tmp_path: Path) -> None:
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    content = _write_ninja(paths, ccache="/usr/bin/ccache")
+    build_dir = CORE.relative_pioenvs_path(CORE.name)
+    assert "rule pch" in content
+    # Compiled from the include list, and again when the checksum changes
+    gch_edge = next(
+        line
+        for line in content.splitlines()
+        if line.startswith("build esphome_pch.h.gch: pch ")
+    )
+    assert gch_edge.endswith("esphome_pch_src.h | esphome_pch.h.gch.sum")
+    for line in content.splitlines():
+        # C++ edges wait on the .gch; the C edge must not reference it
+        if line.startswith("build obj/src/main.cpp.o:"):
+            assert line.endswith("| esphome_pch.h.gch")
+        if line.startswith("build obj/src/esphome/vendor.c.o:"):
+            assert "esphome_pch" not in line
+    assert (build_dir / "esphome_pch_src.h").read_text().splitlines() == [
+        '#include "esphome/components/esp8266/throw_stubs.h"',
+        '#include "esphome/core/pch_prefix.h"',
+    ]
+    assert "#error" in (build_dir / "esphome_pch.h").read_text()
+    assert (build_dir / "esphome_pch.h.gch.sum").read_text().strip()
+
+
+def test_write_project_pch_folds_joined_src_force_include(
+    tmp_path: Path,
+) -> None:
+    """-includefoo.h in build_src_flags must fold into the pch like the
+    separated spelling, not precede and defeat it."""
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    CORE.platformio_options["build_src_flags"] = "-includeesphome/core/defines.h"
+    content = _write_ninja(paths, ccache="/usr/bin/ccache")
+    assert "build esphome_pch.h.gch: pch" in content
+    assert "srccxxflags" in content
+    assert "-includeesphome" not in content
+
+
+@pytest.mark.parametrize("flag", ["-includefoo.h", "--include=foo.h"])
+def test_write_project_pch_skipped_for_other_force_include_spellings(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture, flag: str
+) -> None:
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", flag)
+    content = _write_ninja(paths, ccache="/usr/bin/ccache")
+    assert "esphome_pch" not in content
+    assert "prevents the precompiled header" in caplog.text
+
+
+def test_write_project_pch_skipped_when_user_force_include_precedes(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A -include in build_flags lands ahead of the pch include, so GCC
+    would never load the .gch; skip it and say so."""
+    paths = _make_framework(tmp_path)
+    _set_flags(
+        "-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", "-include foo.h"
+    )
+    content = _write_ninja(paths, ccache="/usr/bin/ccache")
+    assert "esphome_pch" not in content
+    assert "srccxxflags" not in content
+    assert "prevents the precompiled header" in caplog.text
+
+
+def test_write_project_pch_disabled(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    content = _write_ninja(paths)
+    assert "esphome_pch" not in content
+    assert "srccxxflags" not in content
+    assert "  flags = $srcflags" in content
 
 
 def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None:
@@ -1716,3 +1796,37 @@ def test_write_project_rejects_spaced_ldscript_override(tmp_path: Path) -> None:
     CORE.relative_src_path().mkdir(parents=True, exist_ok=True)
     with pytest.raises(EsphomeError, match="Invalid flash linker script name"):
         arduino8266.write_project(paths, None)
+
+
+def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None:
+    """Regression: the -include stays relative and the .sum carries no
+    per-device path, or cross-device ccache sharing breaks."""
+    paths = _make_framework(tmp_path / "shared")
+    sums = []
+    for name in ("dev_a", "dev_b"):
+        CORE.name = name
+        CORE.build_path = tmp_path / name
+        _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+        content = _write_ninja(paths, ccache="/usr/bin/ccache")
+        assert (
+            "srccxxflags = -Winvalid-pch -Werror=invalid-pch "
+            "-include esphome_pch.h" in content
+        )
+        sums.append(
+            (CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum").read_text()
+        )
+    assert sums[0] == sums[1]
+
+
+def test_write_project_pch_sum_tracks_src_flags(tmp_path: Path) -> None:
+    """The header compiles with the src flags too, so they are part of what
+    ccache is told about it."""
+    paths = _make_framework(tmp_path)
+    _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")
+    sum_path = CORE.relative_pioenvs_path(CORE.name) / "esphome_pch.h.gch.sum"
+    sums = []
+    for value in ("1", "2"):
+        CORE.platformio_options["build_src_flags"] = f"-DSRC_ONLY={value}"
+        _write_ninja(paths)
+        sums.append(sum_path.read_text())
+    assert sums[0] != sums[1]

tests/unit_tests/build_helpers/test_ccache.py

@@ -10,6 +10,7 @@ from unittest.mock import patch
 import pytest
 
 from esphome.build_helpers import ccache
+from esphome.core import CORE
 
 
 def test_resolve_opt_out() -> None:
@@ -122,6 +123,20 @@ def test_parse_enable_env_spelling_tables(
     assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected
 
 
+def test_effective_ccache_basedir_prefers_user_value(tmp_path: Path) -> None:
+    CORE.build_path = tmp_path
+    # Drive-qualified on Windows: "/custom/base" is not absolute there
+    base = "C:\\custom\\base" if os.name == "nt" else "/custom/base"
+    with patch.dict(os.environ, {"CCACHE_BASEDIR": base}, clear=True):
+        assert ccache.effective_ccache_basedir() == base
+    with patch.dict(os.environ, {}, clear=True):
+        assert ccache.effective_ccache_basedir() == str(tmp_path.resolve())
+    # Degenerate values would strip substrings ccache never rewrites
+    for bad in ("", "/", "a/b"):
+        with patch.dict(os.environ, {"CCACHE_BASEDIR": bad}, clear=True):
+            assert ccache.effective_ccache_basedir() == str(tmp_path.resolve())
+
+
 def test_resolve_absolute_ccache_path(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -165,3 +180,18 @@ def test_ccache_defaults_env_escapes_a_dollar(
     env = ccache.ccache_defaults_env(tmp_path / "c$d")
     assert env["CCACHE_DIR"].endswith("c$$d")
     assert env["CCACHE_BASEDIR"].endswith("a$$b")
+
+
+def _pch_ccache_env(tmp_path: Path, environ: dict[str, str]) -> dict[str, str]:
+    CORE.build_path = tmp_path / "build"
+    spec = ("ESPHOME_TEST_PREFIX", "test")
+    environ = {"ESPHOME_TEST_PREFIX": str(tmp_path / "cache"), **environ}
+    with patch.dict(os.environ, environ, clear=True):
+        return ccache.ccache_env("/usr/bin/ccache", spec)
+
+
+def test_ccache_env_includes_pch_settings(tmp_path: Path) -> None:
+    """A native build exports the ccache settings the pch needs."""
+    env = _pch_ccache_env(tmp_path, {})
+    assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
+    assert env["CCACHE_PCH_EXTSUM"] == "true"

tests/unit_tests/build_helpers/test_pch.py

@@ -57,6 +57,11 @@ def test_ccache_pch_env_disabled() -> None:
         assert pch.ccache_pch_env() == {}
 
 
+def test_pch_header_text_preserves_order() -> None:
+    text = pch.pch_header_text(["b.h", "a.h"])
+    assert text == '#include "b.h"\n#include "a.h"\n'
+
+
 def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None:
     """Sibling includes resolve against the includer's directory first,
     full paths against the src root; unresolvable names end the walk."""