diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..dae0ed4b39 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -19,6 +19,7 @@ from typing import NamedTuple from esphome.build_helpers.ccache import ccache_defaults_env from esphome.build_helpers.ninja import find_ninja +from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path from esphome.core import EsphomeError, Version from esphome.framework_helpers import str_to_lst_of_str @@ -161,4 +162,6 @@ def ccache_env(ccache: str | None) -> dict[str, str]: """ if ccache is None: return {} - return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") + env = ccache_defaults_env(get_arduino8266_tools_path() / "ccache") + env.update(ccache_pch_env()) + return env diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 50b506ef82..fd7c1ee81f 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -32,6 +32,13 @@ from esphome.build_helpers.ninja import ( quote_path as _q, shell_token as _shell_token, ) +from esphome.build_helpers.pch import ( + PCH_CORE_HEADER, + PCH_HEADER_NAME, + pch_checksum, + pch_enabled, + pch_header_text, +) from esphome.components.esp8266 import build_surgery from esphome.components.esp8266.boards import ( BOARDS, @@ -874,18 +881,26 @@ def _ninja_compile_edges( root: Path, group: str, flags: str = "", + cxx_flags: str = "", + cxx_implicit: str = "", ) -> list[str]: - """Emit compile edges for ``sources``; return the object paths.""" + """Emit compile edges for ``sources``; return the object paths. + + ``cxx_flags``/``cxx_implicit`` override ``flags`` and add an implicit + dependency on C++ edges only (used for the precompiled header). + """ objects = [] for src in sources: rel = src.relative_to(root).as_posix() obj = f"obj/{group}/{rel}.o" escaped_obj = _e(obj) - lines.append( - f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}" - ) - if flags: - lines.append(f" flags = {flags}") + kind = SOURCE_KIND_FOR_SUFFIX[src.suffix] + is_cxx = kind == "cxx" + implicit = f" | {cxx_implicit}" if is_cxx and cxx_implicit else "" + lines.append(f"build {escaped_obj}: {kind} {_e(src)}{implicit}") + edge_flags = cxx_flags if is_cxx and cxx_flags else flags + if edge_flags: + lines.append(f" flags = {edge_flags}") # Escaped once here: the returned paths only ever appear in build # statements (archive/link inputs), which use ninja escaping. objects.append(escaped_obj) @@ -1087,6 +1102,13 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: " depfile = $out.d", " deps = gcc", " description = AS $out", + # No $ccache: the .gch is compiled once per build dir and ccache + # cannot cache it usefully (its bytes embed build-dir paths) + "rule pch", + " command = $cxx -MMD -MF $out.d -x c++-header $cxxflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = PCH $out", # Plain assembler, as SCons's ASCOM: no preprocessor, so no # depfile and no $flags (defines/includes) either "rule asm", @@ -1175,7 +1197,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # One source of truth with the PlatformIO path: esp8266/__init__ pins # build_src_flags (the throw_stubs force-include); -include paths # resolve against the source root - src_parts: list[str] = [] + src_other: list[str] = [] + src_includes: list[str] = [] src_it = iter( lex_build_flags(_pio_option("build_src_flags", ""), "build_src_flags") ) @@ -1186,15 +1209,57 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: raise EsphomeError( "build_src_flags has a trailing '-include' with no header" ) - src_parts.append(f"-include {_q(src_dir / header)}") + src_includes.append(header) else: - src_parts.append(_shell_token(tok)) - src_extra = " ".join(src_parts) + src_other.append(_shell_token(tok)) + include_flags = [f"-include {_q(src_dir / h)}" for h in src_includes] # One shared variable instead of repeating the flags line on every src # edge (hundreds of edges in a real project) - lines.append(f"srcflags = {src_extra}") + lines.append(f"srcflags = {' '.join(src_other + include_flags)}") + src_cxx_flags = None + src_cxx_implicit = "" + if pch_enabled(): + # C++ src edges swap the force-includes for one precompiled prefix + # header holding the same content plus defines.h; C and assembly + # edges keep srcflags (a .gch is a C++ artifact) + pch_header = build_dir / PCH_HEADER_NAME + pch_includes = (*src_includes, PCH_CORE_HEADER) + write_file_if_changed(pch_header, pch_header_text(pch_includes)) + if ccache: + # The .sum sidecar only exists for CCACHE_PCH_EXTSUM; ninja's + # depfile handles staleness. Mirror CCACHE_BASEDIR: strip the + # per-device build path so identically-configured devices + # produce identical .sum files and share cache entries + flags_id = " ".join(cxxflags).replace( + str(Path(CORE.build_path).resolve()), "" + ) + checksum = pch_checksum( + src_dir, + pch_includes, + (str(paths.framework), str(paths.toolchain), flags_id), + ) + write_file_if_changed( + build_dir / f"{PCH_HEADER_NAME}.gch.sum", checksum + "\n" + ) + gch = _e(f"{PCH_HEADER_NAME}.gch") + lines.append(f"build {gch}: pch {_e(pch_header)}") + if src_other: + lines.append(f" flags = {' '.join(src_other)}") + # Relative -include (resolved from the ninja cwd, where the header + # lives): an absolute path would put the per-device build path on + # every compile command and defeat cross-device ccache sharing + cxx_parts = src_other + [f"-include {PCH_HEADER_NAME}"] + lines.append(f"srccxxflags = {' '.join(cxx_parts)}") + src_cxx_flags = "$srccxxflags" + src_cxx_implicit = gch src_objs = _ninja_compile_edges( - lines, _collect_sources(src_dir), src_dir, "src", flags="$srcflags" + lines, + _collect_sources(src_dir), + src_dir, + "src", + flags="$srcflags", + cxx_flags=src_cxx_flags, + cxx_implicit=src_cxx_implicit, ) ld_deps = [f"ld/{_COMMON_LD_NAME}"] diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py new file mode 100644 index 0000000000..35559ad9bc --- /dev/null +++ b/esphome/build_helpers/pch.py @@ -0,0 +1,105 @@ +"""Shared precompiled-header policy for the build backends. + +Safe by construction when the prefix header mirrors what the TUs already +include first (ESP8266); a backend may instead inject a curated set of +self-contained core headers (ESP-IDF). +""" + +from __future__ import annotations + +from collections.abc import Iterable +import hashlib +import os +from pathlib import Path +import posixpath +import re + +from esphome.build_helpers.ccache import parse_enable_env + +# The header and its .gch/.sum sidecars live in the build directory. +PCH_HEADER_NAME = "esphome_pch.h" + +# Last include of the ESP8266 prefix header. +PCH_CORE_HEADER = "esphome/core/defines.h" + +# ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the +# .sum sidecar instead of the .gch bytes, which are not reproducible. +# Keep in sync with the literals in components/esp8266/pch.py.script. +_CCACHE_PCH_ENV = { + "CCACHE_SLOPPINESS": "pch_defines,time_macros", + "CCACHE_PCH_EXTSUM": "true", +} + +_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+"([^"]+)"', re.MULTILINE) + + +def pch_enabled() -> bool: + """Precompiled-header knob: default on, ``ESPHOME_PCH_ENABLE=0`` opts out.""" + return parse_enable_env("ESPHOME_PCH_ENABLE") is not False + + +def ccache_pch_env() -> dict[str, str]: + """ccache settings required to cache compiles that consume the .gch; + empty when the pch is disabled. User-set values win.""" + if not pch_enabled(): + return {} + return {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ} + + +def pch_header_text(include_headers: Iterable[str]) -> str: + """The prefix-header source: exactly these includes, in order.""" + return "".join(f'#include "{name}"\n' for name in include_headers) + + +def quoted_includes(path: Path) -> tuple[str, ...]: + """The quoted #include targets of one file ([] when unreadable).""" + try: + data = path.read_bytes() + except OSError: + return () + return tuple(m.decode() for m in _INCLUDE_RE.findall(data)) + + +def include_closure(src_dir: Path, roots: Iterable[str]) -> set[str]: + """Quoted-include closure of ``roots`` (src-relative names). + + Resolves each include against the includer's directory first, then the + src root, matching the compiler's quoted-include lookup. Names that do + not resolve under ``src_dir`` end the walk; they live in versioned + framework/toolchain installs the caller identifies separately. + Over-approximates (no #ifdef evaluation) — the safe direction for + cache invalidation. + """ + seen: set[str] = set() + stack: list[tuple[str, str]] = [(name, "") for name in roots] + while stack: + name, from_dir = stack.pop() + for candidate in (f"{from_dir}/{name}" if from_dir else name, name): + rel = posixpath.normpath(candidate) + if not rel.startswith("..") and (src_dir / rel).is_file(): + break + else: + continue + if rel in seen: + continue + seen.add(rel) + parent = posixpath.dirname(rel) + stack.extend((inc, parent) for inc in quoted_includes(src_dir / rel)) + return seen + + +def pch_checksum( + src_dir: Path, include_headers: Iterable[str], extra: Iterable[str] +) -> str: + """Digest standing in for the .gch in ccache's hash: the include closure + of the prefix header plus caller-supplied identity strings (versioned + install paths, flags).""" + digest = hashlib.sha256() + for name in sorted(include_closure(src_dir, include_headers)): + digest.update(name.encode()) + digest.update((src_dir / name).read_bytes()) + digest.update(b"\0") + for item in extra: + digest.update(item.encode()) + digest.update(b"\0") + return digest.hexdigest() diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 124655ea53..32cdee7c5a 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -6,6 +6,7 @@ import subprocess import time from typing import Any +from esphome.build_helpers.pch import pch_enabled import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import ( @@ -420,6 +421,9 @@ async def to_code(config: ConfigType) -> None: ] if not enable_scanf_float: extra_scripts.append("pre:remove_float_scanf.py") + # Generation-time gate: the script itself has no enable check + if pch_enabled(): + extra_scripts.append("post:pch.py") extra_scripts.append("post:post_build.py") cg.add_platformio_option("extra_scripts", extra_scripts) @@ -575,6 +579,7 @@ def copy_files() -> None: dir = Path(__file__).parent for script in ( "post_build", + "pch", "testing_mode", "exclude_updater", "exclude_waveform", diff --git a/esphome/components/esp8266/pch.py.script b/esphome/components/esp8266/pch.py.script new file mode 100644 index 0000000000..f96339e745 --- /dev/null +++ b/esphome/components/esp8266/pch.py.script @@ -0,0 +1,110 @@ +import hashlib +import os +from pathlib import Path +import shlex +import subprocess + +# pylint: disable=E0602 +Import("env", "projenv") # noqa: F821 + +# Precompile the src force-includes plus defines.h (which pulls in +# Arduino.h) and force-include the result into C++ src compiles only; those +# TUs already include this content first, so their preprocessed output is +# unchanged. Post script: the final src flags exist, nothing compiled yet. +# Registration is gated host-side (esp8266/__init__.py, pch_enabled()). +# Keep the header tail and ccache values in sync with build_helpers/pch.py. + + +def _esp8266_setup_pch() -> None: + # Project root, not $BUILD_DIR: SCons compiles run with the project dir + # as cwd, so "-include esphome_pch.h" resolves here as a relative path. + # An absolute path would put the per-device build path on every compile + # command and defeat cross-device ccache sharing. + proj_dir = Path(env.subst("$PROJECT_DIR")) # noqa: F821 + src_dir = Path(env.subst("$PROJECT_SRC_DIR")) # noqa: F821 + header = proj_dir / "esphome_pch.h" + gch = Path(f"{header}.gch") + sum_path = Path(f"{gch}.sum") + + cxx = projenv.subst("$CXX") # noqa: F821 + # The header holds the -include entries itself, so the .gch compile must + # not see them; consumers keep theirs, which the .gch then satisfies. + flags = [] + include_headers = [] + flag_it = iter(shlex.split(projenv.subst("$CXXFLAGS $CCFLAGS $_CCCOMCOM"))) # noqa: F821 + for tok in flag_it: + if tok == "-include": + include_headers.append(next(flag_it, "")) + else: + flags.append(tok) + content = "".join( + f'#include "{name}"\n' for name in (*include_headers, "esphome/core/defines.h") + ) + + digest = hashlib.sha256() + digest.update(content.encode()) + digest.update(cxx.encode()) + # Mirror CCACHE_BASEDIR: strip the per-device build path so identical + # configs produce identical .sum files and share cache entries + flags_id = " ".join(flags) + if basedir := os.environ.get("CCACHE_BASEDIR"): + flags_id = flags_id.replace(basedir, "") + digest.update(flags_id.encode()) + # GCC never validates a .gch against its source headers, and PlatformIO + # package paths carry no version, so a platform bump must invalidate here + platform = env.PioPlatform() # noqa: F821 + for package in ("framework-arduinoespressif8266", "toolchain-xtensa"): + digest.update(str(platform.get_package_version(package)).encode()) + digest.update(b"\0") + for name in (*include_headers, "esphome/core/defines.h", "esphome/core/macros.h"): + path = Path(name) if Path(name).is_absolute() else src_dir / name + try: + digest.update(path.read_bytes()) + except OSError: + digest.update(b"unreadable") + digest.update(b"\0") + checksum = digest.hexdigest() + + # The ccache .sum sidecar doubles as the freshness stamp + if ( + not gch.is_file() + or not sum_path.is_file() + or (sum_path.read_text(encoding="utf-8").strip() != checksum) + ): + failed_marker = Path(f"{gch}.failed") + if ( + failed_marker.is_file() + and failed_marker.read_text(encoding="utf-8").strip() == checksum + ): + return + header.write_text(content, encoding="utf-8") + result = subprocess.run( # noqa: PLW1510 + [cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print("ESPHome: precompiled header failed; compiling without it") + print(result.stderr) + # Skip retries until a flag/header/platform change alters the checksum + failed_marker.write_text(checksum + "\n", encoding="utf-8") + return + failed_marker.unlink(missing_ok=True) + sum_path.write_text(checksum + "\n", encoding="utf-8") + + # Scoped to src compiles: framework/library TUs never consume the .gch + # and keep strict ccache hashing. User-set values win. + for key, value in ( + ("CCACHE_SLOPPINESS", "pch_defines,time_macros"), + ("CCACHE_PCH_EXTSUM", "true"), + ): + if key not in os.environ: + projenv["ENV"][key] = value # noqa: F821 + + # Prepended so it is processed before the build_src_flags -include + # entries: GCC only uses a .gch while no other tokens have been seen. + projenv.Prepend(CXXFLAGS=["-include", header.name]) # noqa: F821 + print("ESPHome: Compiling with precompiled header") + + +_esp8266_setup_pch() diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index cd9ffcd3ce..b24a4b86c1 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -364,7 +364,50 @@ def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None: line for line in content.splitlines() if line.startswith(" flags = ") ] assert flags_lines - assert all(line == " flags = $srcflags" for line in flags_lines) + # C++ src edges consume the precompiled header; C/assembly keep srcflags + assert set(flags_lines) == {" flags = $srcflags", " flags = $srccxxflags"} + + +def test_write_project_pch(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths, ccache="/usr/bin/ccache") + build_dir = CORE.relative_pioenvs_path(CORE.name) + assert "rule pch" in content + assert "build esphome_pch.h.gch: pch" in content + for line in content.splitlines(): + # C++ edges wait on the .gch; the C edge must not reference it + if line.startswith("build obj/src/main.cpp.o:"): + assert line.endswith("| esphome_pch.h.gch") + if line.startswith("build obj/src/esphome/vendor.c.o:"): + assert "esphome_pch" not in line + assert (build_dir / "esphome_pch.h").read_text().splitlines() == [ + '#include "esphome/components/esp8266/throw_stubs.h"', + '#include "esphome/core/defines.h"', + ] + assert (build_dir / "esphome_pch.h.gch.sum").read_text().strip() + + +def test_write_project_pch_sum_only_with_ccache(tmp_path: Path) -> None: + """The .sum sidecar exists solely for ccache; skip it when disabled.""" + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths) + build_dir = CORE.relative_pioenvs_path(CORE.name) + assert "build esphome_pch.h.gch: pch" in content + assert not (build_dir / "esphome_pch.h.gch.sum").exists() + + +def test_write_project_pch_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths) + assert "esphome_pch" not in content + assert "srccxxflags" not in content + assert " flags = $srcflags" in content def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None: @@ -1711,3 +1754,22 @@ def test_write_project_rejects_spaced_ldscript_override(tmp_path: Path) -> None: _set_flags() with pytest.raises(EsphomeError, match="Invalid flash linker script name"): arduino8266.write_project(paths, None) + + +def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None: + """Regression: the -include stays relative and the .sum carries no + per-device path, or cross-device ccache sharing breaks.""" + paths = _make_framework(tmp_path / "shared") + sums = [] + for name in ("dev_a", "dev_b"): + CORE.name = name + CORE.build_path = tmp_path / name + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths, ccache="/usr/bin/ccache") + assert "srccxxflags = -include esphome_pch.h" in content + sums.append( + ( + CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum" + ).read_text() + ) + assert sums[0] == sums[1] diff --git a/tests/unit_tests/test_arduino8266_toolchain.py b/tests/unit_tests/test_arduino8266_toolchain.py index 15f0e9717d..7b229f54ef 100644 --- a/tests/unit_tests/test_arduino8266_toolchain.py +++ b/tests/unit_tests/test_arduino8266_toolchain.py @@ -629,3 +629,25 @@ def test_get_idedata_accepts_preresolved_ccache() -> None: assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True} mock_resolve.assert_not_called() assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache" + + +def test_ccache_env_includes_pch_settings() -> None: + """The native build exports the ccache settings the pch needs.""" + with patch.dict(os.environ, {}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") + assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros" + assert env["CCACHE_PCH_EXTSUM"] == "true" + + +def test_ccache_env_pch_disabled() -> None: + with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") + assert "CCACHE_SLOPPINESS" not in env + assert "CCACHE_PCH_EXTSUM" not in env + + +def test_ccache_env_respects_user_sloppiness() -> None: + with patch.dict(os.environ, {"CCACHE_SLOPPINESS": "locale"}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") + assert "CCACHE_SLOPPINESS" not in env + assert env["CCACHE_PCH_EXTSUM"] == "true"