Walk the include closure in the pch checksum and honor a user CCACHE_BASEDIR

esphome/build_gen/arduino8266.py

@@ -27,6 +27,7 @@ import sys
 from typing import TYPE_CHECKING, NamedTuple
 
 from esphome.arduino8266.framework import toolchain_tool
+from esphome.build_helpers.ccache import effective_ccache_basedir
 from esphome.build_helpers.ninja import (
     escape as _e,
     quote_path as _q,
@@ -1216,7 +1217,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
     # One shared variable instead of repeating the flags line on every src
     # edge (hundreds of edges in a real project)
     lines.append(f"srcflags = {' '.join(src_other + include_flags)}")
-    src_cxx_flags = None
+    src_cxx_flags = ""
     src_cxx_implicit = ""
     if pch_enabled():
         # C++ src edges swap the force-includes for one precompiled prefix
@@ -1230,9 +1231,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
             # depfile handles staleness. Mirror CCACHE_BASEDIR: strip the
             # per-device build path so identically-configured devices
             # produce identical .sum files and share cache entries
-            flags_id = " ".join(cxxflags).replace(
-                str(Path(CORE.build_path).resolve()), ""
-            )
+            flags_id = " ".join(cxxflags).replace(effective_ccache_basedir(), "")
             checksum = pch_checksum(
                 src_dir,
                 pch_includes,

esphome/build_helpers/ccache.py

@@ -90,3 +90,11 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
         "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
     }
     return {k: v for k, v in defaults.items() if k not in os.environ}
+
+
+def effective_ccache_basedir() -> str:
+    """The prefix ccache rewrites out of hashed paths: a user CCACHE_BASEDIR
+    wins, else the resolved build path (matching ccache_defaults_env)."""
+    from esphome.core import CORE
+
+    return os.environ.get("CCACHE_BASEDIR") or str(Path(CORE.build_path).resolve())

esphome/build_helpers/pch.py

@@ -19,7 +19,7 @@ from esphome.build_helpers.ccache import parse_enable_env
 # The header and its .gch/.sum sidecars live in the build directory.
 PCH_HEADER_NAME = "esphome_pch.h"
 
-# Last include of the ESP8266 prefix header.
+# The core defines header every backend anchors its prefix on.
 PCH_CORE_HEADER = "esphome/core/defines.h"
 
 # ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the
@@ -39,8 +39,9 @@ def pch_enabled() -> bool:
 
 
 def ccache_pch_env() -> dict[str, str]:
-    """ccache settings required to cache compiles that consume the .gch;
-    empty when the pch is disabled. User-set values win."""
+    """Settings ccache needs to cache compiles that consume the .gch;
+    empty when the pch is disabled. User-set values win. Native backends
+    export these process-wide; only time_macros affects non-pch TUs."""
     if not pch_enabled():
         return {}
     return {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ}
@@ -51,17 +52,8 @@ def pch_header_text(include_headers: Iterable[str]) -> str:
     return "".join(f'#include "{name}"\n' for name in include_headers)
 
 
-def quoted_includes(path: Path) -> tuple[str, ...]:
-    """The quoted #include targets of one file ([] when unreadable)."""
-    try:
-        data = path.read_bytes()
-    except OSError:
-        return ()
-    return tuple(m.decode() for m in _INCLUDE_RE.findall(data))
-
-
-def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]:
-    """Quoted-include closure of ``roots`` (src-relative names).
+def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
+    """Quoted-include closure of ``roots``: src-relative name -> contents.
 
     Resolves each include against the includer's directory first, then the
     src root, matching the compiler's quoted-include lookup. Names that do
@@ -70,7 +62,7 @@ def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]:
     Over-approximates (no #ifdef evaluation) — the safe direction for
     cache invalidation.
     """
-    seen: set[str] = set()
+    seen: dict[str, bytes] = {}
     stack: list[tuple[str, str]] = [(name, "") for name in roots]
     while stack:
         name, from_dir = stack.pop()
@@ -82,9 +74,13 @@ def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]:
             continue
         if rel in seen:
             continue
-        seen.add(rel)
+        try:
+            data = (src_dir / rel).read_bytes()
+        except OSError:
+            continue
+        seen[rel] = data
         parent = posixpath.dirname(rel)
-        stack.extend((inc, parent) for inc in quoted_includes(src_dir / rel))
+        stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data))
     return seen
 
 
@@ -95,9 +91,10 @@ def pch_checksum(
     of the prefix header plus caller-supplied identity strings (versioned
     install paths, flags)."""
     digest = hashlib.sha256()
-    for name in sorted(include_closure(src_dir, include_headers)):
+    closure = _include_closure(src_dir, include_headers)
+    for name in sorted(closure):
         digest.update(name.encode())
-        digest.update((src_dir / name).read_bytes())
+        digest.update(closure[name])
         digest.update(b"\0")
     for item in extra:
         digest.update(item.encode())

esphome/components/esp8266/pch.py.script

@@ -1,6 +1,8 @@
 import hashlib
 import os
 from pathlib import Path
+import posixpath
+import re
 import shlex
 import subprocess
 
@@ -12,7 +14,36 @@ Import("env", "projenv")  # noqa: F821
 # TUs already include this content first, so their preprocessed output is
 # unchanged. Post script: the final src flags exist, nothing compiled yet.
 # Registration is gated host-side (esp8266/__init__.py, pch_enabled()).
-# Keep the header tail and ccache values in sync with build_helpers/pch.py.
+# Keep the header tail, ccache values, include-closure recipe, and the
+# checksum/failed-marker stamp flow in sync with build_helpers/pch.py.
+
+_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+"([^"]+)"', re.MULTILINE)
+_CORE_HEADER = "esphome/core/defines.h"
+
+
+def _include_closure(src_dir: Path, roots: list) -> dict:
+    """Quoted-include closure: src-relative name -> contents (mirror of
+    build_helpers/pch.py)."""
+    seen = {}
+    stack = [(name, "") for name in roots]
+    while stack:
+        name, from_dir = stack.pop()
+        for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
+            rel = posixpath.normpath(candidate)
+            if not rel.startswith("..") and (src_dir / rel).is_file():
+                break
+        else:
+            continue
+        if rel in seen:
+            continue
+        try:
+            data = (src_dir / rel).read_bytes()
+        except OSError:
+            continue
+        seen[rel] = data
+        parent = posixpath.dirname(rel)
+        stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data))
+    return seen
 
 
 def _esp8266_setup_pch() -> None:
@@ -38,7 +69,7 @@ def _esp8266_setup_pch() -> None:
         else:
             flags.append(tok)
     content = "".join(
-        f'#include "{name}"\n' for name in (*include_headers, "esphome/core/defines.h")
+        f'#include "{name}"\n' for name in (*include_headers, _CORE_HEADER)
     )
 
     digest = hashlib.sha256()
@@ -56,12 +87,10 @@ def _esp8266_setup_pch() -> None:
     for package in ("framework-arduinoespressif8266", "toolchain-xtensa"):
         digest.update(str(platform.get_package_version(package)).encode())
         digest.update(b"\0")
-    for name in (*include_headers, "esphome/core/defines.h", "esphome/core/macros.h"):
-        path = Path(name) if Path(name).is_absolute() else src_dir / name
-        try:
-            digest.update(path.read_bytes())
-        except OSError:
-            digest.update(b"unreadable")
+    closure = _include_closure(src_dir, [*include_headers, _CORE_HEADER])
+    for rel in sorted(closure):
+        digest.update(rel.encode())
+        digest.update(closure[rel])
         digest.update(b"\0")
     checksum = digest.hexdigest()
 
@@ -78,14 +107,20 @@ def _esp8266_setup_pch() -> None:
         ):
             return
         header.write_text(content, encoding="utf-8")
-        result = subprocess.run(  # noqa: PLW1510
-            [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)],
-            capture_output=True,
-            text=True,
-        )
-        if result.returncode != 0:
+        try:
+            result = subprocess.run(  # noqa: PLW1510
+                [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)],
+                capture_output=True,
+                text=True,
+            )
+            error = result.stderr if result.returncode != 0 else None
+        except OSError as err:
+            error = str(err)
+        if error is not None:
             print("ESPHome: precompiled header failed; compiling without it")
-            print(result.stderr)
+            print(error)
+            gch.unlink(missing_ok=True)
+            sum_path.unlink(missing_ok=True)
             # Skip retries until a flag/header/platform change alters the checksum
             failed_marker.write_text(checksum + "\n", encoding="utf-8")
             return

tests/unit_tests/build_gen/test_arduino8266.py

@@ -1768,8 +1768,6 @@ def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None:
         content = _write_ninja(paths, ccache="/usr/bin/ccache")
         assert "srccxxflags = -include esphome_pch.h" in content
         sums.append(
-            (
-                CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum"
-            ).read_text()
+            (CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum").read_text()
         )
     assert sums[0] == sums[1]