From c52ce65d1c5b0f7e241831f071291cc59e4979a9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Sep 2026 19:19:46 +0200 Subject: [PATCH] [esp8266] Precompile the prefix header on PlatformIO builds (#19823) --- esphome/build_gen/platformio.py | 6 + esphome/build_helpers/pch.py | 30 ++ esphome/platformio/pch.py.script | 206 ++++++++++ esphome/platformio/toolchain.py | 13 + esphome/writer.py | 5 + tests/unit_tests/build_gen/test_platformio.py | 35 ++ tests/unit_tests/build_helpers/test_pch.py | 16 + .../unit_tests/test_platformio_pch_script.py | 383 ++++++++++++++++++ tests/unit_tests/test_platformio_toolchain.py | 24 ++ tests/unit_tests/test_writer.py | 23 ++ 10 files changed, 741 insertions(+) create mode 100644 esphome/platformio/pch.py.script create mode 100644 tests/unit_tests/test_platformio_pch_script.py diff --git a/esphome/build_gen/platformio.py b/esphome/build_gen/platformio.py index 0a12d344a0..b68ee90f12 100644 --- a/esphome/build_gen/platformio.py +++ b/esphome/build_gen/platformio.py @@ -1,6 +1,8 @@ +from esphome.build_helpers.pch import pch_script_enabled from esphome.const import __version__ from esphome.core import CORE from esphome.helpers import mkdir_p, read_file, write_file_if_changed +from esphome.platformio.toolchain import copy_pch_script from esphome.writer import find_begin_end INI_AUTO_GENERATE_BEGIN = "; ========== AUTO GENERATED CODE BEGIN ===========" @@ -62,6 +64,8 @@ def get_ini_content(): # Add extra script for C++ flags CORE.add_platformio_option("extra_scripts", [f"pre:{CXX_FLAGS_FILE_NAME}"]) + if pch_script_enabled(): + CORE.add_platformio_option("extra_scripts", ["post:pch.py"]) # Add CMake args. A user-supplied value (str or list) is deliberately # replaced; this option was always overwritten at FINAL priority. @@ -106,6 +110,8 @@ def write_project(): # Write extra script for C++ specific flags write_cxx_flags_script() + if pch_script_enabled(): + copy_pch_script() CXX_FLAGS_FILE_NAME = "cxx_flags.py" diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index bfe81d8504..7889978f06 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -11,6 +11,14 @@ import posixpath import re from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env +from esphome.const import ( + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_LN882X, + PLATFORM_NRF52, + PLATFORM_RP2, + PLATFORM_RTL87XX, +) from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -33,9 +41,24 @@ PCH_GUARD_TEXT = f"""\ #endif """ +# What the PlatformIO script leaves in the project root, for cleanup +PCH_ARTIFACT_NAMES = (PCH_HEADER_NAME, PCH_GCH_NAME, PCH_SUM_NAME, PCH_SOURCE_NAME) + # The core headers every backend precompiles PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) +# PlatformIO platforms that do not take the pch script +PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset( + { + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_LN882X, + PLATFORM_NRF52, + PLATFORM_RP2, + PLATFORM_RTL87XX, + } +) + # What ccache needs to cache compiles that load a .gch _CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros") @@ -71,6 +94,13 @@ def ccache_pch_env() -> dict[str, str]: return env +def pch_script_enabled() -> bool: + """Whether this PlatformIO build takes the pch script.""" + from esphome.core import CORE + + return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS + + def pch_header_text(include_headers: Iterable[str]) -> str: """The prefix-header source: exactly these includes, in order.""" return "".join(f'#include "{name}"\n' for name in include_headers) diff --git a/esphome/platformio/pch.py.script b/esphome/platformio/pch.py.script new file mode 100644 index 0000000000..962f5c3f2c --- /dev/null +++ b/esphome/platformio/pch.py.script @@ -0,0 +1,206 @@ +import hashlib +import os +from pathlib import Path +import posixpath +import re +import shlex +import subprocess + +# pylint: disable=E0602 +Import("env") # noqa: F821 +try: + Import("projenv") # noqa: F821 +except Exception: # noqa: BLE001 -- not exported under -t nobuild + projenv = None + +# Precompile the force-includes plus the core headers and force-include +# the result into C++ src compiles only. Any failure stops the build. This +# script cannot import esphome: a test pins the values below to +# build_helpers/pch.py. +_HEADER_NAME = "esphome_pch.h" +_SOURCE_NAME = "esphome_pch_src.h" +_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) +_CONSUMER_FLAGS = ["-Winvalid-pch", "-Werror=invalid-pch", "-include", _HEADER_NAME] +_GUARD_TEXT = f"""\ +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTELLISENSE__) +#error "The precompiled header was not loaded" +#else +#include "{_SOURCE_NAME}" +#endif +""" +_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE) +_CXX_SUFFIXES = (".cpp", ".cc", ".cxx") +_HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".inc") +_INCLUDE_DIR_OPTIONS = ("-I", "-isystem", "-iquote", "-idirafter") +# Build output and library trees; package versions stand in for libraries +_SKIPPED_DIRS = (".pioenvs", ".piolibdeps", "managed_components") + + +def _include_closure(src_dir: Path, roots: list) -> dict: + """Include closure: src-relative name -> contents.""" + seen = {} + stack = [(name, "") for name in roots] + while stack: + name, from_dir = stack.pop() + for candidate in (f"{from_dir}/{name}" if from_dir else name, name): + rel = posixpath.normpath(candidate) + if not rel.startswith("..") and (src_dir / rel).is_file(): + break + else: + continue + if rel in seen: + continue + data = seen[rel] = (src_dir / rel).read_bytes() + parent = posixpath.dirname(rel) + stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data)) + return seen + + +def _shell_arg(element) -> str: + """One compiler argument from one SCons element, as the real spawn + passes it (shlex would eat path backslashes on Windows).""" + arg = str(element) + if " " in arg or os.name == "nt": + return arg.replace('\\"', '"') + return (shlex.split(arg) or [""])[0] + + +def _src_envs(src_dir: Path) -> list: + """The environments the C++ src objects compile from: projenv, or the + ESP-IDF builder's own.""" + found = {} + src_dir = src_dir.resolve() + nodes = env.Flatten(env.get("PIOBUILDFILES", [])) # noqa: F821 + for node in nodes: + source = Path(str(node.sources[0])).resolve() + if source.suffix in _CXX_SUFFIXES and source.is_relative_to(src_dir): + found[id(node.env)] = node.env + if nodes and not found: + # Flags on projenv would reach no compile + raise RuntimeError("ESPHome: no C++ source takes the precompiled header") + return list(found.values()) or [projenv] # noqa: F821 + + +def _setup_pch() -> None: + if projenv is None: + if "nobuild" in COMMAND_LINE_TARGETS: # noqa: F821 + return + raise RuntimeError("ESPHome: no project environment for the pch") + # SCons compiles run in the project root, so the relative -include + # resolves; an absolute path would enter the ccache key + proj_dir = Path(env.subst("$PROJECT_DIR")) # noqa: F821 + src_dir = Path(env.subst("$PROJECT_SRC_DIR")) # noqa: F821 + header = proj_dir / _HEADER_NAME + source = proj_dir / _SOURCE_NAME + gch = Path(f"{header}.gch") + sum_path = Path(f"{gch}.sum") + src_envs = _src_envs(src_dir) + + cxx = src_envs[0].subst("$CXX") + # The header holds the -include entries itself, so the .gch compile must + # not see them; consumers keep theirs, which the .gch then satisfies + flags = [] + include_headers = [] + flag_it = iter( + _shell_arg(element) + for element in src_envs[0].subst_list("$CXXFLAGS $CCFLAGS $_CCCOMCOM")[0] + ) + for tok in flag_it: + if tok == "-include": + include_headers.append(next(flag_it, "")) + elif tok.startswith("-include") and not tok.startswith("-include-"): + include_headers.append(tok[len("-include") :]) + else: + flags.append(tok) + # Folded first and in order, as the compiler would have read them. An + # absolute path would enter the checksum, so it cannot be folded + if absolute := [name for name in include_headers if Path(name).is_absolute()]: + print( + f"ESPHome: the force-include of {absolute} prevents the precompiled " + "header; compiling without it" + ) + return + headers = [*include_headers, *_DEFAULT_HEADERS] + content = "".join(f'#include "{name}"\n' for name in headers) + + digest = hashlib.sha256() + digest.update(content.encode()) + digest.update(cxx.encode()) + # As ccache does: strip the build path so devices share cache entries + flags_id = " ".join(flags) + if basedir := os.environ.get("CCACHE_BASEDIR"): + flags_id = flags_id.replace(basedir, "") + digest.update(flags_id.encode()) + # GCC does not check a .gch against its headers and PlatformIO package + # paths carry no version + platform = env.PioPlatform() # noqa: F821 + for package in sorted(platform.packages): + digest.update(f"{package}={platform.get_package_version(package)}\0".encode()) + # Libraries carry their version in lib_deps + digest.update(repr(env.GetProjectOption("lib_deps", [])).encode()) # noqa: F821 + closure = _include_closure(src_dir, headers) + for rel in sorted(closure): + digest.update(rel.encode()) + digest.update(closure[rel]) + digest.update(b"\0") + # The ESP-IDF configuration: sdkconfig.h sits in the build output + for config in sorted(proj_dir.glob("sdkconfig.*")): + digest.update(config.read_bytes()) + # Project-local include dirs (rp2's lwip_override) hold generated + # headers outside src/ + prev = "" + for tok in flags: + inc = "" + for opt in _INCLUDE_DIR_OPTIONS: + if prev == opt: + inc = tok + elif tok.startswith(opt) and len(tok) > len(opt): + inc = tok[len(opt) :] + prev = tok + # Relative dirs are relative to the project, where compiles run + inc_dir = Path(os.path.normpath(proj_dir / inc)) + if ( + not inc + or inc_dir == proj_dir + or not inc_dir.is_relative_to(proj_dir) + or inc_dir.is_relative_to(src_dir) + or any(inc_dir.is_relative_to(proj_dir / name) for name in _SKIPPED_DIRS) + ): + continue + for local in sorted(inc_dir.rglob("*")): + if local.suffix in _HEADER_SUFFIXES and local.is_file(): + digest.update(str(local.relative_to(proj_dir)).encode()) + digest.update(local.read_bytes()) + digest.update(b"\0") + checksum = digest.hexdigest() + + # The ccache .sum file doubles as the freshness stamp + if not ( + gch.is_file() + and sum_path.is_file() + and sum_path.read_text(encoding="utf-8").strip() == checksum + ): + header.write_text(_GUARD_TEXT, encoding="utf-8") + source.write_text(content, encoding="utf-8") + result = subprocess.run( # noqa: PLW1510 + [cxx, "-x", "c++-header", *flags, "-c", str(source), "-o", str(gch)], + cwd=proj_dir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"ESPHome: could not compile the precompiled header:\n{result.stderr}" + ) + sum_path.write_text(checksum + "\n", encoding="utf-8") + + # Prepended: GCC only loads a .gch for the first -include + for src_env in src_envs: + src_env.Prepend(CXXFLAGS=_CONSUMER_FLAGS) + print( + "ESPHome: Compiling with a precompiled header " + "(set ESPHOME_PCH_ENABLE=0 to disable)" + ) + + +_setup_pch() diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 2322b37dfc..6a906a42d0 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -286,6 +286,14 @@ def copy_ccache_script() -> None: ) +def copy_pch_script() -> None: + """Copy the precompiled header SCons script into the build dir.""" + copy_file_if_changed( + Path(__file__).parent / "pch.py.script", + CORE.relative_build_path("pch.py"), + ) + + def default_libdeps_dir() -> str: """The PLATFORMIO_LIBDEPS_DIR value a pio run defaults to; the package prefetch must resolve installed libraries against the same dir.""" @@ -327,6 +335,11 @@ def run_platformio_cli(*args, **kwargs) -> str | int: base_env = kwargs.pop("env", None) env = dict(os.environ if base_env is None else base_env) env.update(_ccache_env()) + if CORE.is_configured: + from esphome.build_helpers.pch import ccache_pch_env, pch_script_enabled + + if pch_script_enabled(): + env.update(ccache_pch_env()) # The runner offers the out-of-flash tip but has no configured CORE, so # tell it. Ask CORE, not is_esp32_arduino_build(), which reads this same # variable; clear an inherited one so it cannot reach the wrong build. diff --git a/esphome/writer.py b/esphome/writer.py index bdd60af1dc..9f2ff85cff 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -610,6 +610,11 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False): if idf_path.is_dir(): _LOGGER.info("Deleting %s", idf_path) rmtree(idf_path) + # The PlatformIO pch files live in the project root + from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES + + for name in PCH_ARTIFACT_NAMES: + CORE.relative_build_path(name).unlink(missing_ok=True) # idedata caches live under the data dir, not the build path; globbed # so a future backend suffix cannot drift out of clean-all diff --git a/tests/unit_tests/build_gen/test_platformio.py b/tests/unit_tests/build_gen/test_platformio.py index 20acbe302c..935265a004 100644 --- a/tests/unit_tests/build_gen/test_platformio.py +++ b/tests/unit_tests/build_gen/test_platformio.py @@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch import pytest from esphome.build_gen import platformio +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM from esphome.core import CORE @@ -170,6 +171,40 @@ def clean_core(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(CORE, "build_flags", set()) monkeypatch.setattr(CORE, "build_unflags", set()) monkeypatch.setattr(CORE, "cmake_args", {}) + # A platform that does not take the pch script + monkeypatch.setitem(CORE.data, KEY_CORE, {KEY_TARGET_PLATFORM: "nrf52"}) + + +def test_pch_script_is_registered_and_copied( + clean_core: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """One place registers the pch script for every PlatformIO platform + that takes it.""" + monkeypatch.setitem(CORE.data, KEY_CORE, {KEY_TARGET_PLATFORM: "esp8266"}) + monkeypatch.setattr(CORE, "build_path", tmp_path) + + platformio.write_project() + + assert "post:pch.py" in (tmp_path / "platformio.ini").read_text() + assert (tmp_path / "pch.py").is_file() + + +@pytest.mark.parametrize("disabled_by", ["platform", "knob"]) +def test_pch_script_is_left_out( + clean_core: None, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + disabled_by: str, +) -> None: + if disabled_by == "knob": + monkeypatch.setitem(CORE.data, KEY_CORE, {KEY_TARGET_PLATFORM: "esp8266"}) + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + monkeypatch.setattr(CORE, "build_path", tmp_path) + + platformio.write_project() + + assert "pch.py" not in (tmp_path / "platformio.ini").read_text() + assert not (tmp_path / "pch.py").exists() def test_get_ini_content_pins_cpp_standard( diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index 99109bec5e..0820da5be3 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -9,6 +9,8 @@ from unittest.mock import patch import pytest from esphome.build_helpers import pch +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM +from esphome.core import CORE def _write(src_dir: Path, name: str, content: str) -> None: @@ -107,6 +109,20 @@ def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None: assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"]) +@pytest.mark.parametrize("platform", ["esp8266"]) +def test_pch_script_enabled(platform: str, monkeypatch: pytest.MonkeyPatch) -> None: + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} + assert pch.pch_script_enabled() + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + assert not pch.pch_script_enabled() + + +@pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_EXCLUDED_PLATFORMS)) +def test_pch_script_excluded_platform(platform: str) -> None: + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} + assert not pch.pch_script_enabled() + + def test_include_closure_walks_angle_includes_under_src(tmp_path: Path) -> None: """An angle include resolving under src/ must enter the digest; one that does not simply ends the walk.""" diff --git a/tests/unit_tests/test_platformio_pch_script.py b/tests/unit_tests/test_platformio_pch_script.py new file mode 100644 index 0000000000..bccfc17477 --- /dev/null +++ b/tests/unit_tests/test_platformio_pch_script.py @@ -0,0 +1,383 @@ +"""Tests for esphome/platformio/pch.py.script against a fake SCons env.""" + +from __future__ import annotations + +from collections.abc import Callable +import os +from pathlib import Path +import stat +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from esphome.build_helpers import pch +from esphome.platformio import toolchain + +pytestmark = pytest.mark.skipif( + os.name == "nt", reason="the fake compiler is a POSIX shell script" +) + +_SCRIPT = Path(toolchain.__file__).parent / "pch.py.script" + + +class _FakePlatform: + packages = {"framework-x": {}, "toolchain-y": {}} + + def get_package_version(self, name: str) -> str | None: + # None for an optional package that is not installed + return None if name == "toolchain-y" else "1.2.3" + + +class _FakeSConsEnv(dict): + """Just enough of a SCons construction environment for pch.py.""" + + def __init__( + self, + proj_dir: Path, + src_dir: Path, + cxx: str, + flags: list[str], + platform_cls: type[_FakePlatform] = _FakePlatform, + ): + super().__init__(ENV={}) + self._subst = { + "$PROJECT_DIR": str(proj_dir), + "$PROJECT_SRC_DIR": str(src_dir), + "$CXX": cxx, + } + self._flags = flags + self._platform_cls = platform_cls + self.prepended: list[str] = [] + + def subst(self, expr: str) -> str: # noqa: N802 + return self._subst[expr] + + def subst_list(self, expr: str) -> list[list[str]]: # noqa: N802 + return [self._flags] + + def PioPlatform(self) -> _FakePlatform: # noqa: N802 + return self._platform_cls() + + def Prepend(self, CXXFLAGS: list[str]) -> None: # noqa: N802, N803 + self.prepended = CXXFLAGS + + def Flatten(self, nodes: list) -> list: # noqa: N802 + return nodes + + def GetProjectOption(self, name: str, default: list) -> list: # noqa: N802 + return self.get(f"option:{name}", default) + + +def _fake_cxx(tmp_path: Path, fail: bool = False) -> Path: + """A compiler stand-in that records its argv and writes the -o target.""" + cxx = tmp_path / "fake-gxx" + body = ( + 'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n' + ) + if fail: + body += "echo boom >&2\nexit 1\n" + else: + body += 'out=""; prev=""\nfor a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done\n' + body += '[ -n "$out" ] && echo gch > "$out"\n' + cxx.write_text("#!/bin/sh\n" + body) + cxx.chmod(cxx.stat().st_mode | stat.S_IEXEC) + return cxx + + +def _run_script( + tmp_path: Path, + flags: list[str] | None = None, + fail: bool = False, + env_vars: dict[str, str] | None = None, + name: str = "dev", + platform_cls: type[_FakePlatform] = _FakePlatform, + build_files: Callable[[tuple], list] | None = None, +) -> _FakeSConsEnv: + proj = tmp_path / name + src = proj / "src" + (src / "esphome" / "core").mkdir(parents=True, exist_ok=True) + (src / "esphome" / "core" / "defines.h").write_text("#define USE_X\n") + (src / "esphome" / "core" / "pch_prefix.h").write_text( + '#include "esphome/core/defines.h"\n' + ) + cxx = _fake_cxx(tmp_path, fail=fail) + args = (proj, src, str(cxx), flags or ["-DX=1"], platform_cls) + # Distinct objects: the -include flags must land on projenv only + global_env = _FakeSConsEnv(*args) + projenv = _FakeSConsEnv(*args) + projenv.global_env = global_env + if build_files is not None: + global_env["PIOBUILDFILES"] = build_files(args) + source = _SCRIPT.read_text() + with patch.dict(os.environ, env_vars or {}, clear=True): + exec( # noqa: S102 + compile(source, "pch.py", "exec"), + { + "Import": lambda *_names: None, + "env": global_env, + "projenv": projenv, + "COMMAND_LINE_TARGETS": [], + }, + ) + return projenv + + +def test_pch_script_builds_and_prepends_relative_include(tmp_path: Path) -> None: + scons_env = _run_script(tmp_path) + proj = tmp_path / "dev" + assert (proj / "esphome_pch_src.h").read_text() == pch.pch_header_text( + pch.PCH_DEFAULT_HEADERS + ) + assert (proj / "esphome_pch.h").read_text() == pch.PCH_GUARD_TEXT + assert (proj / "esphome_pch.h.gch").is_file() + assert len((proj / "esphome_pch.h.gch.sum").read_text().strip()) == 64 + assert scons_env.prepended == pch.pch_consumer_flags() + # The -include flags are scoped to projenv (src compiles) + assert scons_env.global_env.prepended == [] + + +def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: + """The script cannot import esphome, so its copies are pinned.""" + namespace: dict[str, object] = { + "Import": lambda *_names: None, + "env": _FakeSConsEnv(tmp_path, tmp_path, "g++", []), + "projenv": None, + "COMMAND_LINE_TARGETS": ["nobuild"], + } + exec(compile(_SCRIPT.read_text(), "pch.py", "exec"), namespace) # noqa: S102 + assert namespace["_HEADER_NAME"] == pch.PCH_HEADER_NAME + assert namespace["_SOURCE_NAME"] == pch.PCH_SOURCE_NAME + assert namespace["_DEFAULT_HEADERS"] == pch.PCH_DEFAULT_HEADERS + assert namespace["_CONSUMER_FLAGS"] == pch.pch_consumer_flags() + assert namespace["_GUARD_TEXT"] == pch.PCH_GUARD_TEXT + assert namespace["_INCLUDE_RE"].pattern == pch._INCLUDE_RE.pattern + + +def test_pch_script_compile_failure_stops_the_build(tmp_path: Path) -> None: + """The pch holds only ESPHome's own headers: a failure is a defect.""" + with pytest.raises(RuntimeError, match="could not compile") as raised: + _run_script(tmp_path, fail=True) + assert "boom" in str(raised.value) + assert not (tmp_path / "dev" / "esphome_pch.h.gch.sum").exists() + + +def test_pch_script_preserves_spaced_flag_elements(tmp_path: Path) -> None: + """One SCons element stays one compiler argv; -include pairs are + stripped from the .gch compile.""" + spaced = tmp_path / "My Configs" + spaced.mkdir() + (tmp_path / "dev" / "src").mkdir(parents=True, exist_ok=True) + (tmp_path / "dev" / "src" / "other.h").write_text("") + flags = ['-DUSB_PRODUCT=\\"Pico 2W\\"', "-I", str(spaced), "-include", "other.h"] + _run_script(tmp_path, flags=flags) + calls = (tmp_path / "fake-gxx.argv").read_text().split("---call---\n") + gch_call = next(c for c in calls if "c++-header" in c).splitlines() + assert '-DUSB_PRODUCT="Pico 2W"' in gch_call + assert str(spaced) in gch_call + assert "-include" not in gch_call + # The stripped src-resolvable -include is folded into the prefix header + pch = (tmp_path / "dev" / "esphome_pch_src.h").read_text() + assert pch.splitlines()[0] == '#include "other.h"' + + +def test_pch_script_folds_joined_force_include_spelling(tmp_path: Path) -> None: + """-includefoo.h folds like the separated form, matching the native path.""" + (tmp_path / "dev" / "src").mkdir(parents=True, exist_ok=True) + (tmp_path / "dev" / "src" / "other.h").write_text("") + _run_script(tmp_path, flags=["-DX=1", "-includeother.h"]) + pch = (tmp_path / "dev" / "esphome_pch_src.h").read_text() + assert pch.splitlines()[0] == '#include "other.h"' + + +def test_pch_script_sum_is_device_independent(tmp_path: Path) -> None: + """Regression: identical configs in different dirs share cache keys.""" + sums = [] + for name in ("dev_a", "dev_b"): + proj = tmp_path / name + _run_script( + tmp_path, + flags=["-DX=1", "-I", str(proj / "include")], + env_vars={"CCACHE_BASEDIR": str(proj)}, + name=name, + ) + sums.append((proj / "esphome_pch.h.gch.sum").read_text()) + (tmp_path / "fake-gxx").unlink() + (tmp_path / "fake-gxx.argv").unlink(missing_ok=True) + assert sums[0] == sums[1] + + +def test_pch_script_uses_the_envs_of_existing_src_objects(tmp_path: Path) -> None: + """The ESP-IDF builder creates the src objects from its own environments + before this script runs; flags on projenv would never reach them.""" + made: list[_FakeSConsEnv] = [] + + def build_files(args: tuple) -> list: + src = args[1] + made.extend(_FakeSConsEnv(*args) for _ in range(3)) + return [ + SimpleNamespace(env=made[0], sources=[src / "main.cpp"]), + SimpleNamespace(env=made[0], sources=[src / "esphome" / "a.cpp"]), + # C and framework objects keep their environments untouched + SimpleNamespace(env=made[1], sources=[src / "esphome" / "b.c"]), + SimpleNamespace(env=made[2], sources=[src.parent / "lib" / "c.cpp"]), + ] + + projenv = _run_script(tmp_path, build_files=build_files) + assert made[0].prepended == pch.pch_consumer_flags() + assert made[1].prepended == made[2].prepended == projenv.prepended == [] + + +def test_copy_pch_script(tmp_path: Path) -> None: + from esphome.core import CORE + + CORE.build_path = tmp_path + toolchain.copy_pch_script() + assert (tmp_path / "pch.py").read_text() == _SCRIPT.read_text() + + +def _run_without_projenv(tmp_path: Path, targets: list[str]) -> Path: + proj = tmp_path / "dev" + (proj / "src").mkdir(parents=True) + + def strict_import(*names: str) -> None: + if "projenv" in names: + raise RuntimeError("Import of non-existent variable 'projenv'") + + env = _FakeSConsEnv(proj, proj / "src", "g++", ["-DX=1"]) + exec( # noqa: S102 + compile(_SCRIPT.read_text(), "pch.py", "exec"), + {"Import": strict_import, "env": env, "COMMAND_LINE_TARGETS": targets}, + ) + return proj + + +def test_pch_script_nobuild_without_projenv_is_noop(tmp_path: Path) -> None: + """-t nobuild never exports projenv; the script must not abort.""" + proj = _run_without_projenv(tmp_path, ["nobuild"]) + assert not (proj / "esphome_pch.h").exists() + + +def test_pch_script_build_without_projenv_stops(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="no project environment"): + _run_without_projenv(tmp_path, []) + + +def test_pch_script_stops_when_no_source_object_matches(tmp_path: Path) -> None: + """Flags on projenv would reach no compile while the log says the + precompiled header is in use.""" + + def build_files(args: tuple) -> list: + lib = args[1].parent / "lib" / "c.cpp" + return [SimpleNamespace(env=_FakeSConsEnv(*args), sources=[lib])] + + with pytest.raises(RuntimeError, match="no C\\+\\+ source takes"): + _run_script(tmp_path, build_files=build_files) + + +def test_pch_script_ignores_library_trees_and_non_headers(tmp_path: Path) -> None: + """.piolibdeps and non-header files must not enter the digest (or be + read at all); package versions already cover library identity.""" + proj = tmp_path / "dev" + libdeps = proj / ".piolibdeps" / "lib" / "src" + libdeps.mkdir(parents=True) + (libdeps / "lib.h").write_text("#define A 1\n") + override = proj / "lwip_override" + override.mkdir(parents=True) + (override / "lwipopts.h").write_text("#define TCP_MSS 1460\n") + (override / "notes.txt").write_text("v1\n") + flags = ["-DX=1", "-I", str(libdeps), "-I", str(override)] + _run_script(tmp_path, flags=flags) + first = (proj / "esphome_pch.h.gch.sum").read_text() + (libdeps / "lib.h").write_text("#define A 2\n") + (override / "notes.txt").write_text("v2\n") + (tmp_path / "fake-gxx.argv").unlink(missing_ok=True) + _run_script(tmp_path, flags=flags) + assert (proj / "esphome_pch.h.gch.sum").read_text() == first + + +def test_pch_script_hashes_project_local_include_dirs(tmp_path: Path) -> None: + """Generated headers in project-local -I dirs (e.g. rp2's lwip_override) + must invalidate the checksum when they change.""" + proj = tmp_path / "dev" + override = proj / "lwip_override" + override.mkdir(parents=True) + (override / "lwipopts.h").write_text("#define TCP_MSS 1460\n") + flags = ["-DX=1", "-I", str(override)] + _run_script(tmp_path, flags=flags) + first = (proj / "esphome_pch.h.gch.sum").read_text() + (override / "lwipopts.h").write_text("#define TCP_MSS 536\n") + (tmp_path / "fake-gxx.argv").unlink(missing_ok=True) + _run_script(tmp_path, flags=flags) + assert (proj / "esphome_pch.h.gch.sum").read_text() != first + + +def test_pch_script_folds_force_includes_found_on_the_include_path( + tmp_path: Path, +) -> None: + """A framework force-include (Arduino.h) is read before the core + headers, as it was without the precompiled header.""" + _run_script(tmp_path, flags=["-DX=1", "-include", "Arduino.h"]) + source = (tmp_path / "dev" / "esphome_pch_src.h").read_text() + assert source.splitlines() == [ + '#include "Arduino.h"', + '#include "esphome/core/pch_prefix.h"', + ] + + +def test_pch_script_skipped_for_an_absolute_force_include( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + outside = tmp_path / "outside.h" + outside.write_text("") + scons_env = _run_script(tmp_path, flags=["-DX=1", "-include", str(outside)]) + assert scons_env.prepended == [] + assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists() + assert "prevents the precompiled header" in capsys.readouterr().out + + +def _sum_after(tmp_path: Path, flags: list[str]) -> str: + (tmp_path / "fake-gxx.argv").unlink(missing_ok=True) + _run_script(tmp_path, flags=flags) + return (tmp_path / "dev" / "esphome_pch.h.gch.sum").read_text() + + +@pytest.mark.parametrize( + "option", [["-Iinclude"], ["-isystem", "include"], ["-iquoteinclude"]] +) +def test_pch_script_hashes_relative_include_dirs( + tmp_path: Path, option: list[str] +) -> None: + """Compiles run in the project root, so -Iinclude is a project dir.""" + include = tmp_path / "dev" / "include" + include.mkdir(parents=True) + (include / "user.h").write_text("#define A 1\n") + flags = ["-DX=1", *option, "-I."] + first = _sum_after(tmp_path, flags) + (include / "user.h").write_text("#define A 2\n") + assert _sum_after(tmp_path, flags) != first + + +def test_pch_script_hashes_the_sdkconfig(tmp_path: Path) -> None: + """sdkconfig.h is build output; the configuration it comes from is not.""" + proj = tmp_path / "dev" + proj.mkdir() + (proj / "sdkconfig.dev").write_text("CONFIG_X=y\n") + first = _sum_after(tmp_path, ["-DX=1"]) + (proj / "sdkconfig.dev").write_text("CONFIG_X=n\n") + assert _sum_after(tmp_path, ["-DX=1"]) != first + + +def test_pch_script_hashes_the_library_versions(tmp_path: Path) -> None: + sums = [] + for version in ("7.4.1", "7.4.2"): + (tmp_path / "fake-gxx.argv").unlink(missing_ok=True) + + with patch.object( + _FakeSConsEnv, + "GetProjectOption", + lambda self, name, default, version=version: [f"ArduinoJson@{version}"], + ): + _run_script(tmp_path) + sums.append((tmp_path / "dev" / "esphome_pch.h.gch.sum").read_text()) + assert sums[0] != sums[1] diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 6138d74524..d218f3201f 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -330,6 +330,30 @@ def test_idedata_null_section_raises_esphome_error(setup_core: Path) -> None: _ = toolchain.IDEData({"extra": None}).extra_flash_images +@pytest.mark.parametrize( + ("platform", "expected"), [("esp8266", "true"), ("nrf52", None)] +) +def test_run_platformio_cli_exports_the_pch_ccache_settings( + setup_core: Path, + mock_run_external_process: Mock, + platform: str, + expected: str | None, +) -> None: + """Only for a platform that takes the pch script.""" + CORE.build_path = str(setup_core / "build" / "test") + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: platform, + KEY_TARGET_FRAMEWORK: "arduino", + } + + with patch.dict(os.environ, {}, clear=True): + mock_run_external_process.return_value = 0 + toolchain.run_platformio_cli("test", "arg") + + env = mock_run_external_process.call_args[1]["env"] + assert env.get("CCACHE_PCH_EXTSUM") == expected + + @pytest.mark.parametrize( ("platform", "framework", "expected"), [ diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 7fd03062a2..432a2065bf 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -677,6 +677,29 @@ def test_clean_build_partial_exists( assert "dependencies.lock" not in caplog.text +@patch("esphome.writer.CORE") +def test_clean_build_partial_removes_pch_artifacts( + mock_core: MagicMock, + tmp_path: Path, +) -> None: + """The PlatformIO pch sidecars live at the project root and must go in + a partial clean, like the native backend's under .pioenvs.""" + from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES as names + + assert "esphome_pch.h.gch" in names + for name in names: + (tmp_path / name).write_text("x") + mock_core.relative_pioenvs_path.return_value = tmp_path / ".pioenvs" + mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps" + mock_core.relative_build_path.side_effect = lambda name: tmp_path / name + mock_core.relative_internal_path.side_effect = tmp_path.joinpath + + clean_build() + + for name in names: + assert not (tmp_path / name).exists() + + @patch("esphome.writer.CORE") def test_clean_build_nothing_exists( mock_core: MagicMock,