diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 639148b46b..cec6c4e273 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -2,21 +2,12 @@ import json import logging -import os from pathlib import Path -import subprocess -from esphome.build_helpers.ccache import effective_ccache_basedir -from esphome.build_helpers.idedata import ( - CXX_SOURCE_SUFFIXES, - expand_response_files, - is_launcher, - split_command, -) +from esphome.build_helpers import pch from esphome.build_helpers.pch import ( PCH_CORE_HEADER, PCH_HEADER_NAME, - pch_checksum, pch_enabled, pch_header_text, ) @@ -58,12 +49,6 @@ _PCH_HEADERS = ( # _pch_cmake() and prepare_pch() for the layout rationale _PCH_BUILD_HEADER = f"build/{PCH_HEADER_NAME}" -# Compile-command tokens dropped when retargeting a TU's flags at the -# prefix header: source/output/depfile flags with an argument, and the -# argument-less depfile flags (the pch compile must not touch depfiles) -_PCH_STRIP_FLAGS_WITH_ARG = frozenset({"-o", "-c", "-MT", "-MF", "-MQ"}) -_PCH_STRIP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MM", "-M"}) - # Replaces the IDF default C++ standard (-std=gnu++2b appended to # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(), @@ -348,102 +333,16 @@ set_source_files_properties(${{app_sources}} PROPERTIES """ -def _pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None: - """The exact src C++ flags from compile_commands.json, retargeted at - the header; None (logged) when no configured C++ TU is available yet.""" - try: - entries = json.loads( - (build_dir / "compile_commands.json").read_text(encoding="utf-8") - ) - except (OSError, json.JSONDecodeError) as err: - # Configure already succeeded, so an unusable DB is a real anomaly - _LOGGER.warning("No usable compile database, skipping pch: %s", err) - return None - if not isinstance(entries, list): - _LOGGER.warning("Malformed compile database, skipping pch") - return None - # Windows compile DBs use backslashes; normalize both sides - src_prefix = str(CORE.relative_src_path()).replace("\\", "/") - entry = next( - ( - e - for e in entries - if isinstance(e, dict) - and e.get("file", "").replace("\\", "/").startswith(src_prefix) - and e.get("file", "").endswith(CXX_SOURCE_SUFFIXES) - ), - None, - ) - if entry is None: - _LOGGER.warning("No src C++ entry in the compile database, skipping pch") - return None - tokens = expand_response_files( - split_command(entry.get("command", "")), Path(entry.get("directory", build_dir)) - ) - # A DB recorded with ccache enabled prefixes the compiler with the - # launcher; the .gch must be compiled directly - if tokens and is_launcher(tokens[0]): - tokens = tokens[1:] - if not tokens: - # An "arguments"-style or empty entry must skip cleanly, not spawn - # a compiler-less argv that warns on every build - _LOGGER.warning("Compile database entry has no usable command, skipping pch") - return None - args: list[str] = [] - arg_it = iter(tokens) - for tok in arg_it: - if tok in _PCH_STRIP_FLAGS_WITH_ARG: - next(arg_it, None) - continue - if tok in _PCH_STRIP_FLAGS: - continue - if tok == "-include": - # Drop only the injected prefix; user force-includes must reach - # the .gch compile or GCC rejects it over the macro mismatch - inc = next(arg_it, "") - if not inc.endswith(PCH_HEADER_NAME): - args.extend(("-include", inc)) - continue - args.append(tok) - return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)] - - def discard_pch() -> None: - """Remove the pch sidecars so a stale .gch is never consumed. - - Bumps the header only when a .gch was actually removed: TUs compiled - against it have incomplete depfiles, while a repeat failure with no - .gch must not force a full rebuild every build. - """ - header = CORE.relative_build_path(_PCH_BUILD_HEADER) - gch = Path(f"{header}.gch") - had_gch = gch.is_file() - gch.unlink(missing_ok=True) - Path(f"{gch}.sum").unlink(missing_ok=True) - if had_gch and header.is_file(): - os.utime(header) + """Drop the pch sidecars in the IDF build dir.""" + pch.discard_pch(CORE.relative_build_path("build")) def prepare_pch() -> None: - """Compile the prefix header's .gch and write its ccache .sum. - - Runs right before ninja, after every reconfigure, so the flags in - compile_commands.json and the sdkconfig are the settled ones. The .sum - doubles as the freshness stamp and folds in the compile command, so a - flag-only change rebuilds the .gch. A failed compile falls back to the - plain header include. - """ + """Build the .gch right before ninja, after every reconfigure, so the + compile_commands.json flags and the sdkconfig are the settled ones.""" if not pch_enabled(): return - build_dir = CORE.relative_build_path("build") - header = CORE.relative_build_path(_PCH_BUILD_HEADER) - gch = Path(f"{header}.gch") - sum_path = Path(f"{gch}.sum") - cmd = _pch_compile_command(build_dir, header, gch) - if cmd is None: - # Freshness cannot be validated; a leftover .gch must not be consumed - discard_pch() - return sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}") try: sdkconfig = sdkconfig_path.read_text(encoding="utf-8") @@ -454,72 +353,17 @@ def prepare_pch() -> None: "Could not read %s for the pch checksum: %s", sdkconfig_path, err ) sdkconfig = f"unreadable:{type(err).__name__}:{err.errno}" - # Stripped like ccache's own rewriting (a user CCACHE_BASEDIR wins) so - # identical configs hash identically across devices; the raw build path - # covers unresolved spellings in the compile DB - cmd_id = ( - " ".join(cmd) - .replace(effective_ccache_basedir(), "") - .replace(str(CORE.build_path), "") - ) - checksum = pch_checksum( - CORE.relative_src_path(), + pch.prepare_pch( + CORE.relative_build_path("build"), _PCH_HEADERS, ( - # The closure is sorted, so root order only enters via the text - pch_header_text(_PCH_HEADERS), str(idf_version()), CORE.cpp_standard or "", sdkconfig, *get_project_compile_flags(), *get_project_cxx_compile_flags(), - cmd_id, ), ) - if ( - gch.is_file() - and sum_path.is_file() - and sum_path.read_text(encoding="utf-8").strip() == checksum - ): - return - failed_marker = Path(f"{gch}.failed") - if ( - failed_marker.is_file() - and failed_marker.read_text(encoding="utf-8").strip() == checksum - ): - _LOGGER.info( - "Precompiled header disabled after an earlier failure; delete %s to retry", - failed_marker, - ) - return - try: - result = subprocess.run( - cmd, cwd=build_dir, capture_output=True, text=True, check=False, timeout=300 - ) - error = None - if result.returncode != 0: - error = result.stderr.strip() or f"exit code {result.returncode}" - elif not gch.is_file(): - error = "compiler produced no .gch" - except (OSError, subprocess.SubprocessError) as err: - # Transient (timeout, spawn/IO): warn and retry next build, no marker - _LOGGER.warning("Precompiled header compile did not run: %s", err) - discard_pch() - return - if error is not None: - _LOGGER.warning( - "Precompiled header failed; compiling without it: %s", error[:400] - ) - discard_pch() - # Skip retries until a header/flag/sdkconfig/command change - failed_marker.write_text(checksum + "\n", encoding="utf-8") - os.utime(header) - return - failed_marker.unlink(missing_ok=True) - sum_path.write_text(checksum + "\n", encoding="utf-8") - # The OBJECT_DEPENDS edge watches the header; bump it so consumers of - # the previous .gch recompile - os.utime(header) def write_project( diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index 4c8d0936b9..6fa09e7457 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -11,13 +11,21 @@ from __future__ import annotations from collections.abc import Iterable import hashlib +import json import logging import os from pathlib import Path import posixpath import re +import subprocess -from esphome.build_helpers.ccache import parse_enable_env +from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env +from esphome.build_helpers.idedata import ( + CXX_SOURCE_SUFFIXES, + expand_response_files, + is_launcher, + split_command, +) _LOGGER = logging.getLogger(__name__) @@ -122,3 +130,173 @@ def pch_checksum( digest.update(item.encode()) digest.update(b"\0") return digest.hexdigest() + + +# Compile-command tokens dropped when retargeting a TU's flags at the +# prefix header: source/output/depfile flags with an argument, and the +# argument-less depfile flags (the pch compile must not touch depfiles) +_PCH_STRIP_FLAGS_WITH_ARG = frozenset({"-o", "-c", "-MT", "-MF", "-MQ"}) +_PCH_STRIP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MM", "-M"}) + + +def pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None: + """The exact src C++ flags from compile_commands.json, retargeted at + the header; None (logged) when no configured C++ TU is available yet.""" + from esphome.core import CORE + + try: + entries = json.loads( + (build_dir / "compile_commands.json").read_text(encoding="utf-8") + ) + except (OSError, json.JSONDecodeError) as err: + # Configure already succeeded, so an unusable DB is a real anomaly + _LOGGER.warning("No usable compile database, skipping pch: %s", err) + return None + if not isinstance(entries, list): + _LOGGER.warning("Malformed compile database, skipping pch") + return None + # Windows compile DBs use backslashes; normalize both sides + src_prefix = str(CORE.relative_src_path()).replace("\\", "/") + entry = next( + ( + e + for e in entries + if isinstance(e, dict) + and e.get("file", "").replace("\\", "/").startswith(src_prefix) + and e.get("file", "").endswith(CXX_SOURCE_SUFFIXES) + ), + None, + ) + if entry is None: + _LOGGER.warning("No src C++ entry in the compile database, skipping pch") + return None + tokens = expand_response_files( + split_command(entry.get("command", "")), Path(entry.get("directory", build_dir)) + ) + # A DB recorded with ccache enabled prefixes the compiler with the + # launcher; the .gch must be compiled directly + if tokens and is_launcher(tokens[0]): + tokens = tokens[1:] + if not tokens: + # An "arguments"-style or empty entry must skip cleanly, not spawn + # a compiler-less argv that warns on every build + _LOGGER.warning("Compile database entry has no usable command, skipping pch") + return None + args: list[str] = [] + arg_it = iter(tokens) + for tok in arg_it: + if tok in _PCH_STRIP_FLAGS_WITH_ARG: + next(arg_it, None) + continue + if tok in _PCH_STRIP_FLAGS: + continue + if tok == "-include": + # Drop only the injected prefix; user force-includes must reach + # the .gch compile or GCC rejects it over the macro mismatch + inc = next(arg_it, "") + if not inc.endswith(PCH_HEADER_NAME): + args.extend(("-include", inc)) + continue + args.append(tok) + return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)] + + +def discard_pch(build_dir: Path) -> None: + """Remove the pch sidecars so a stale .gch is never consumed. + + Bumps the header only when a .gch was actually removed: TUs compiled + against it have incomplete depfiles, while a repeat failure with no + .gch must not force a full rebuild every build. + """ + header = build_dir / PCH_HEADER_NAME + gch = Path(f"{header}.gch") + had_gch = gch.is_file() + gch.unlink(missing_ok=True) + Path(f"{gch}.sum").unlink(missing_ok=True) + if had_gch and header.is_file(): + os.utime(header) + + +def prepare_pch( + build_dir: Path, include_headers: tuple[str, ...], extra: Iterable[str] +) -> None: + """Compile ``build_dir``'s .gch from compile_commands.json flags and + write its ccache .sum. + + The .sum doubles as the freshness stamp and folds in the compile + command, so a flag-only change rebuilds the .gch; ``extra`` carries + backend identity (framework version, sdkconfig, ...). A failed + compile falls back to the plain header include. + """ + from esphome.core import CORE + + header = build_dir / PCH_HEADER_NAME + gch = Path(f"{header}.gch") + sum_path = Path(f"{gch}.sum") + cmd = pch_compile_command(build_dir, header, gch) + if cmd is None: + # Freshness cannot be validated; a leftover .gch must not be consumed + discard_pch(build_dir) + return + # Stripped like ccache's own rewriting (a user CCACHE_BASEDIR wins) so + # identical configs hash identically across devices; the raw build path + # covers unresolved spellings in the compile DB + cmd_id = ( + " ".join(cmd) + .replace(effective_ccache_basedir(), "") + .replace(str(CORE.build_path), "") + ) + checksum = pch_checksum( + CORE.relative_src_path(), + include_headers, + ( + # The closure is sorted, so root order only enters via the text + pch_header_text(include_headers), + *extra, + cmd_id, + ), + ) + if ( + gch.is_file() + and sum_path.is_file() + and sum_path.read_text(encoding="utf-8").strip() == checksum + ): + return + failed_marker = Path(f"{gch}.failed") + if ( + failed_marker.is_file() + and failed_marker.read_text(encoding="utf-8").strip() == checksum + ): + _LOGGER.info( + "Precompiled header disabled after an earlier failure; delete %s to retry", + failed_marker, + ) + return + try: + result = subprocess.run( + cmd, cwd=build_dir, capture_output=True, text=True, check=False, timeout=300 + ) + error = None + if result.returncode != 0: + error = result.stderr.strip() or f"exit code {result.returncode}" + elif not gch.is_file(): + error = "compiler produced no .gch" + except (OSError, subprocess.SubprocessError) as err: + # Transient (timeout, spawn/IO): warn and retry next build, no marker + _LOGGER.warning("Precompiled header compile did not run: %s", err) + discard_pch(build_dir) + return + if error is not None: + _LOGGER.warning( + "Precompiled header failed; compiling without it: %s", error[:400] + ) + discard_pch(build_dir) + # Skip retries until a header/flag/backend-identity/command change + failed_marker.write_text(checksum + "\n", encoding="utf-8") + os.utime(header) + return + failed_marker.unlink(missing_ok=True) + sum_path.write_text(checksum + "\n", encoding="utf-8") + # Consumers depend on the header (depfiles cannot see through a .gch); + # bump it so users of the previous .gch recompile + os.utime(header) diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 9b7e78f4c1..a1dc30b958 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -549,7 +549,7 @@ def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None: with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile), ): prepare_pch() checksum = (dev / "build" / "esphome_pch.h.gch.sum").read_text().strip() @@ -557,7 +557,7 @@ def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None: # Unchanged inputs: the second call must not recompile with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError), ): prepare_pch() @@ -579,7 +579,7 @@ def test_pch_no_device_path_poison(tmp_path: Path) -> None: with ( patch.object(CORE, "name", name), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile), ): prepare_pch() content = get_component_cmakelists() @@ -600,13 +600,13 @@ def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None def test_pch_compile_command_variants(tmp_path: Path) -> None: """Missing DB, no matching entry, and launcher-prefixed commands.""" - from esphome.build_gen.espidf import _pch_compile_command + from esphome.build_helpers.pch import pch_compile_command build = tmp_path / "build" build.mkdir() header = build / "esphome_pch.h" gch = build / "esphome_pch.h.gch" - assert _pch_compile_command(build, header, gch) is None + assert pch_compile_command(build, header, gch) is None (build / "compile_commands.json").write_text( json.dumps( @@ -615,7 +615,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None: ] ) ) - assert _pch_compile_command(build, header, gch) is None + assert pch_compile_command(build, header, gch) is None src_file = str(tmp_path / "src" / "esphome" / "a.cpp") (build / "compile_commands.json").write_text( @@ -634,7 +634,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None: ) ) # Launcher stripped; -include/-o/-c and depfile flags removed - assert _pch_compile_command(build, header, gch) == [ + assert pch_compile_command(build, header, gch) == [ "g++", "-DX=1", "-x", @@ -649,7 +649,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None: def test_pch_compile_command_rejects_unusable_entries(tmp_path: Path) -> None: """Malformed DB shapes and command-less entries skip cleanly instead of producing a compiler-less argv retried every build.""" - from esphome.build_gen.espidf import _pch_compile_command + from esphome.build_helpers.pch import pch_compile_command build = tmp_path / "build" build.mkdir() @@ -659,16 +659,16 @@ def test_pch_compile_command_rejects_unusable_entries(tmp_path: Path) -> None: src_file = str(tmp_path / "src" / "esphome" / "a.cpp") db.write_text(json.dumps({"not": "a list"})) - assert _pch_compile_command(build, header, gch) is None + assert pch_compile_command(build, header, gch) is None db.write_text(json.dumps(["just a string"])) - assert _pch_compile_command(build, header, gch) is None + assert pch_compile_command(build, header, gch) is None # arguments-style entry (allowed by the spec, unused by CMake) db.write_text( json.dumps([{"arguments": ["g++", "-c", src_file], "file": src_file}]) ) - assert _pch_compile_command(build, header, gch) is None + assert pch_compile_command(build, header, gch) is None def test_pch_header_list_order_is_in_checksum( @@ -688,7 +688,7 @@ def test_pch_header_list_order_is_in_checksum( with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile), ): espidf_mod.prepare_pch() first = (dev / "build" / "esphome_pch.h.gch.sum").read_text() @@ -712,7 +712,7 @@ def test_prepare_pch_failure_writes_marker_and_skips_retry(tmp_path: Path) -> No with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=failing_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=failing_compile), ): prepare_pch() prepare_pch() @@ -737,7 +737,7 @@ def test_prepare_pch_spawn_oserror_is_transient(tmp_path: Path) -> None: before = header.stat().st_mtime_ns with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=raising), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=raising), ): prepare_pch() prepare_pch() @@ -762,7 +762,7 @@ def test_prepare_pch_transient_with_stale_gch_bumps_header(tmp_path: Path) -> No with ( patch.object(CORE, "name", "test"), patch( - "esphome.build_gen.espidf.subprocess.run", + "esphome.build_helpers.pch.subprocess.run", side_effect=OSError("no such compiler"), ), ): @@ -779,7 +779,7 @@ def test_prepare_pch_disabled_is_noop( monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") dev = _make_pch_device(tmp_path, "dev_d") CORE.build_path = dev - with patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError): + with patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError): prepare_pch() @@ -792,7 +792,7 @@ def test_prepare_pch_without_compile_commands(tmp_path: Path) -> None: CORE.build_path = dev with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError), ): prepare_pch() assert not (dev / "build" / "esphome_pch.h.gch.sum").exists() @@ -860,7 +860,7 @@ def test_prepare_pch_zero_exit_without_gch_is_failure(tmp_path: Path) -> None: with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=no_output), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=no_output), ): prepare_pch() assert not (dev / "build" / "esphome_pch.h.gch.sum").exists() @@ -887,7 +887,7 @@ def test_prepare_pch_bumps_header_for_object_depends(tmp_path: Path) -> None: with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile), ): prepare_pch() assert header.stat().st_mtime > before @@ -914,7 +914,7 @@ def test_prepare_pch_command_change_invalidates_sum(tmp_path: Path) -> None: with ( patch.object(CORE, "name", "test"), - patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile), + patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile), ): prepare_pch() first = (dev / "build" / "esphome_pch.h.gch.sum").read_text() @@ -925,7 +925,7 @@ def test_prepare_pch_command_change_invalidates_sum(tmp_path: Path) -> None: def test_prepare_pch_keeps_user_force_includes(tmp_path: Path) -> None: - from esphome.build_gen.espidf import _pch_compile_command + from esphome.build_helpers.pch import pch_compile_command dev = _make_pch_device(tmp_path, "dev_u") CORE.build_path = dev @@ -945,6 +945,6 @@ def test_prepare_pch_keeps_user_force_includes(tmp_path: Path) -> None: ] ) ) - cmd = _pch_compile_command(build, build / "esphome_pch.h", build / "x.gch") + cmd = pch_compile_command(build, build / "esphome_pch.h", build / "x.gch") assert "user.h" in cmd assert "esphome_pch.h" not in " ".join(cmd[:-3])