diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index fd7c1ee81f..37e149474b 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -27,6 +27,7 @@ import sys from typing import TYPE_CHECKING, NamedTuple from esphome.arduino8266.framework import toolchain_tool +from esphome.build_helpers.ccache import effective_ccache_basedir from esphome.build_helpers.ninja import ( escape as _e, quote_path as _q, @@ -1216,7 +1217,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # One shared variable instead of repeating the flags line on every src # edge (hundreds of edges in a real project) lines.append(f"srcflags = {' '.join(src_other + include_flags)}") - src_cxx_flags = None + src_cxx_flags = "" src_cxx_implicit = "" if pch_enabled(): # C++ src edges swap the force-includes for one precompiled prefix @@ -1230,9 +1231,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # depfile handles staleness. Mirror CCACHE_BASEDIR: strip the # per-device build path so identically-configured devices # produce identical .sum files and share cache entries - flags_id = " ".join(cxxflags).replace( - str(Path(CORE.build_path).resolve()), "" - ) + flags_id = " ".join(cxxflags).replace(effective_ccache_basedir(), "") checksum = pch_checksum( src_dir, pch_includes, diff --git a/esphome/build_helpers/ccache.py b/esphome/build_helpers/ccache.py index 5b5c7f247f..99d1bbc111 100644 --- a/esphome/build_helpers/ccache.py +++ b/esphome/build_helpers/ccache.py @@ -90,3 +90,11 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), } return {k: v for k, v in defaults.items() if k not in os.environ} + + +def effective_ccache_basedir() -> str: + """The prefix ccache rewrites out of hashed paths: a user CCACHE_BASEDIR + wins, else the resolved build path (matching ccache_defaults_env).""" + from esphome.core import CORE + + return os.environ.get("CCACHE_BASEDIR") or str(Path(CORE.build_path).resolve()) diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index 35559ad9bc..6a013259f8 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -19,7 +19,7 @@ from esphome.build_helpers.ccache import parse_enable_env # The header and its .gch/.sum sidecars live in the build directory. PCH_HEADER_NAME = "esphome_pch.h" -# Last include of the ESP8266 prefix header. +# The core defines header every backend anchors its prefix on. PCH_CORE_HEADER = "esphome/core/defines.h" # ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the @@ -39,8 +39,9 @@ def pch_enabled() -> bool: def ccache_pch_env() -> dict[str, str]: - """ccache settings required to cache compiles that consume the .gch; - empty when the pch is disabled. User-set values win.""" + """Settings ccache needs to cache compiles that consume the .gch; + empty when the pch is disabled. User-set values win. Native backends + export these process-wide; only time_macros affects non-pch TUs.""" if not pch_enabled(): return {} return {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ} @@ -51,17 +52,8 @@ def pch_header_text(include_headers: Iterable[str]) -> str: return "".join(f'#include "{name}"\n' for name in include_headers) -def quoted_includes(path: Path) -> tuple[str, ...]: - """The quoted #include targets of one file ([] when unreadable).""" - try: - data = path.read_bytes() - except OSError: - return () - return tuple(m.decode() for m in _INCLUDE_RE.findall(data)) - - -def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]: - """Quoted-include closure of ``roots`` (src-relative names). +def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]: + """Quoted-include closure of ``roots``: src-relative name -> contents. Resolves each include against the includer's directory first, then the src root, matching the compiler's quoted-include lookup. Names that do @@ -70,7 +62,7 @@ def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]: Over-approximates (no #ifdef evaluation) — the safe direction for cache invalidation. """ - seen: set[str] = set() + seen: dict[str, bytes] = {} stack: list[tuple[str, str]] = [(name, "") for name in roots] while stack: name, from_dir = stack.pop() @@ -82,9 +74,13 @@ def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]: continue if rel in seen: continue - seen.add(rel) + try: + data = (src_dir / rel).read_bytes() + except OSError: + continue + seen[rel] = data parent = posixpath.dirname(rel) - stack.extend((inc, parent) for inc in quoted_includes(src_dir / rel)) + stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data)) return seen @@ -95,9 +91,10 @@ def pch_checksum( of the prefix header plus caller-supplied identity strings (versioned install paths, flags).""" digest = hashlib.sha256() - for name in sorted(include_closure(src_dir, include_headers)): + closure = _include_closure(src_dir, include_headers) + for name in sorted(closure): digest.update(name.encode()) - digest.update((src_dir / name).read_bytes()) + digest.update(closure[name]) digest.update(b"\0") for item in extra: digest.update(item.encode()) diff --git a/esphome/components/esp8266/pch.py.script b/esphome/components/esp8266/pch.py.script index f96339e745..444fa1aa69 100644 --- a/esphome/components/esp8266/pch.py.script +++ b/esphome/components/esp8266/pch.py.script @@ -1,6 +1,8 @@ import hashlib import os from pathlib import Path +import posixpath +import re import shlex import subprocess @@ -12,7 +14,36 @@ Import("env", "projenv") # noqa: F821 # TUs already include this content first, so their preprocessed output is # unchanged. Post script: the final src flags exist, nothing compiled yet. # Registration is gated host-side (esp8266/__init__.py, pch_enabled()). -# Keep the header tail and ccache values in sync with build_helpers/pch.py. +# Keep the header tail, ccache values, include-closure recipe, and the +# checksum/failed-marker stamp flow in sync with build_helpers/pch.py. + +_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+"([^"]+)"', re.MULTILINE) +_CORE_HEADER = "esphome/core/defines.h" + + +def _include_closure(src_dir: Path, roots: list) -> dict: + """Quoted-include closure: src-relative name -> contents (mirror of + build_helpers/pch.py).""" + seen = {} + stack = [(name, "") for name in roots] + while stack: + name, from_dir = stack.pop() + for candidate in (f"{from_dir}/{name}" if from_dir else name, name): + rel = posixpath.normpath(candidate) + if not rel.startswith("..") and (src_dir / rel).is_file(): + break + else: + continue + if rel in seen: + continue + try: + data = (src_dir / rel).read_bytes() + except OSError: + continue + seen[rel] = data + parent = posixpath.dirname(rel) + stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data)) + return seen def _esp8266_setup_pch() -> None: @@ -38,7 +69,7 @@ def _esp8266_setup_pch() -> None: else: flags.append(tok) content = "".join( - f'#include "{name}"\n' for name in (*include_headers, "esphome/core/defines.h") + f'#include "{name}"\n' for name in (*include_headers, _CORE_HEADER) ) digest = hashlib.sha256() @@ -56,12 +87,10 @@ def _esp8266_setup_pch() -> None: for package in ("framework-arduinoespressif8266", "toolchain-xtensa"): digest.update(str(platform.get_package_version(package)).encode()) digest.update(b"\0") - for name in (*include_headers, "esphome/core/defines.h", "esphome/core/macros.h"): - path = Path(name) if Path(name).is_absolute() else src_dir / name - try: - digest.update(path.read_bytes()) - except OSError: - digest.update(b"unreadable") + closure = _include_closure(src_dir, [*include_headers, _CORE_HEADER]) + for rel in sorted(closure): + digest.update(rel.encode()) + digest.update(closure[rel]) digest.update(b"\0") checksum = digest.hexdigest() @@ -78,14 +107,20 @@ def _esp8266_setup_pch() -> None: ): return header.write_text(content, encoding="utf-8") - result = subprocess.run( # noqa: PLW1510 - [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)], - capture_output=True, - text=True, - ) - if result.returncode != 0: + try: + result = subprocess.run( # noqa: PLW1510 + [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)], + capture_output=True, + text=True, + ) + error = result.stderr if result.returncode != 0 else None + except OSError as err: + error = str(err) + if error is not None: print("ESPHome: precompiled header failed; compiling without it") - print(result.stderr) + print(error) + gch.unlink(missing_ok=True) + sum_path.unlink(missing_ok=True) # Skip retries until a flag/header/platform change alters the checksum failed_marker.write_text(checksum + "\n", encoding="utf-8") return diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index b24a4b86c1..de6abba6dc 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -1768,8 +1768,6 @@ def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None: content = _write_ninja(paths, ccache="/usr/bin/ccache") assert "srccxxflags = -include esphome_pch.h" in content sums.append( - ( - CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum" - ).read_text() + (CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum").read_text() ) assert sums[0] == sums[1]