Merge branch 'platformio-pch-rp2' into platformio-pch-libretiny

This commit is contained in:
J. Nick Koston
2026-08-25 18:03:02 -05:00
3 changed files with 208 additions and 186 deletions
+7 -163
View File
@@ -2,21 +2,12 @@
import json
import logging
import os
from pathlib import Path
import subprocess
from esphome.build_helpers.ccache import effective_ccache_basedir
from esphome.build_helpers.idedata import (
CXX_SOURCE_SUFFIXES,
expand_response_files,
is_launcher,
split_command,
)
from esphome.build_helpers import pch
from esphome.build_helpers.pch import (
PCH_CORE_HEADER,
PCH_HEADER_NAME,
pch_checksum,
pch_enabled,
pch_header_text,
)
@@ -58,12 +49,6 @@ _PCH_HEADERS = (
# _pch_cmake() and prepare_pch() for the layout rationale
_PCH_BUILD_HEADER = f"build/{PCH_HEADER_NAME}"
# Compile-command tokens dropped when retargeting a TU's flags at the
# prefix header: source/output/depfile flags with an argument, and the
# argument-less depfile flags (the pch compile must not touch depfiles)
_PCH_STRIP_FLAGS_WITH_ARG = frozenset({"-o", "-c", "-MT", "-MF", "-MQ"})
_PCH_STRIP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MM", "-M"})
# Replaces the IDF default C++ standard (-std=gnu++2b appended to
# CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via
# cg.set_cpp_standard(). Emitted between include(project.cmake) and project(),
@@ -348,102 +333,16 @@ set_source_files_properties(${{app_sources}} PROPERTIES
"""
def _pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None:
"""The exact src C++ flags from compile_commands.json, retargeted at
the header; None (logged) when no configured C++ TU is available yet."""
try:
entries = json.loads(
(build_dir / "compile_commands.json").read_text(encoding="utf-8")
)
except (OSError, json.JSONDecodeError) as err:
# Configure already succeeded, so an unusable DB is a real anomaly
_LOGGER.warning("No usable compile database, skipping pch: %s", err)
return None
if not isinstance(entries, list):
_LOGGER.warning("Malformed compile database, skipping pch")
return None
# Windows compile DBs use backslashes; normalize both sides
src_prefix = str(CORE.relative_src_path()).replace("\\", "/")
entry = next(
(
e
for e in entries
if isinstance(e, dict)
and e.get("file", "").replace("\\", "/").startswith(src_prefix)
and e.get("file", "").endswith(CXX_SOURCE_SUFFIXES)
),
None,
)
if entry is None:
_LOGGER.warning("No src C++ entry in the compile database, skipping pch")
return None
tokens = expand_response_files(
split_command(entry.get("command", "")), Path(entry.get("directory", build_dir))
)
# A DB recorded with ccache enabled prefixes the compiler with the
# launcher; the .gch must be compiled directly
if tokens and is_launcher(tokens[0]):
tokens = tokens[1:]
if not tokens:
# An "arguments"-style or empty entry must skip cleanly, not spawn
# a compiler-less argv that warns on every build
_LOGGER.warning("Compile database entry has no usable command, skipping pch")
return None
args: list[str] = []
arg_it = iter(tokens)
for tok in arg_it:
if tok in _PCH_STRIP_FLAGS_WITH_ARG:
next(arg_it, None)
continue
if tok in _PCH_STRIP_FLAGS:
continue
if tok == "-include":
# Drop only the injected prefix; user force-includes must reach
# the .gch compile or GCC rejects it over the macro mismatch
inc = next(arg_it, "")
if not inc.endswith(PCH_HEADER_NAME):
args.extend(("-include", inc))
continue
args.append(tok)
return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)]
def discard_pch() -> None:
"""Remove the pch sidecars so a stale .gch is never consumed.
Bumps the header only when a .gch was actually removed: TUs compiled
against it have incomplete depfiles, while a repeat failure with no
.gch must not force a full rebuild every build.
"""
header = CORE.relative_build_path(_PCH_BUILD_HEADER)
gch = Path(f"{header}.gch")
had_gch = gch.is_file()
gch.unlink(missing_ok=True)
Path(f"{gch}.sum").unlink(missing_ok=True)
if had_gch and header.is_file():
os.utime(header)
"""Drop the pch sidecars in the IDF build dir."""
pch.discard_pch(CORE.relative_build_path("build"))
def prepare_pch() -> None:
"""Compile the prefix header's .gch and write its ccache .sum.
Runs right before ninja, after every reconfigure, so the flags in
compile_commands.json and the sdkconfig are the settled ones. The .sum
doubles as the freshness stamp and folds in the compile command, so a
flag-only change rebuilds the .gch. A failed compile falls back to the
plain header include.
"""
"""Build the .gch right before ninja, after every reconfigure, so the
compile_commands.json flags and the sdkconfig are the settled ones."""
if not pch_enabled():
return
build_dir = CORE.relative_build_path("build")
header = CORE.relative_build_path(_PCH_BUILD_HEADER)
gch = Path(f"{header}.gch")
sum_path = Path(f"{gch}.sum")
cmd = _pch_compile_command(build_dir, header, gch)
if cmd is None:
# Freshness cannot be validated; a leftover .gch must not be consumed
discard_pch()
return
sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}")
try:
sdkconfig = sdkconfig_path.read_text(encoding="utf-8")
@@ -454,72 +353,17 @@ def prepare_pch() -> None:
"Could not read %s for the pch checksum: %s", sdkconfig_path, err
)
sdkconfig = f"unreadable:{type(err).__name__}:{err.errno}"
# Stripped like ccache's own rewriting (a user CCACHE_BASEDIR wins) so
# identical configs hash identically across devices; the raw build path
# covers unresolved spellings in the compile DB
cmd_id = (
" ".join(cmd)
.replace(effective_ccache_basedir(), "")
.replace(str(CORE.build_path), "")
)
checksum = pch_checksum(
CORE.relative_src_path(),
pch.prepare_pch(
CORE.relative_build_path("build"),
_PCH_HEADERS,
(
# The closure is sorted, so root order only enters via the text
pch_header_text(_PCH_HEADERS),
str(idf_version()),
CORE.cpp_standard or "",
sdkconfig,
*get_project_compile_flags(),
*get_project_cxx_compile_flags(),
cmd_id,
),
)
if (
gch.is_file()
and sum_path.is_file()
and sum_path.read_text(encoding="utf-8").strip() == checksum
):
return
failed_marker = Path(f"{gch}.failed")
if (
failed_marker.is_file()
and failed_marker.read_text(encoding="utf-8").strip() == checksum
):
_LOGGER.info(
"Precompiled header disabled after an earlier failure; delete %s to retry",
failed_marker,
)
return
try:
result = subprocess.run(
cmd, cwd=build_dir, capture_output=True, text=True, check=False, timeout=300
)
error = None
if result.returncode != 0:
error = result.stderr.strip() or f"exit code {result.returncode}"
elif not gch.is_file():
error = "compiler produced no .gch"
except (OSError, subprocess.SubprocessError) as err:
# Transient (timeout, spawn/IO): warn and retry next build, no marker
_LOGGER.warning("Precompiled header compile did not run: %s", err)
discard_pch()
return
if error is not None:
_LOGGER.warning(
"Precompiled header failed; compiling without it: %s", error[:400]
)
discard_pch()
# Skip retries until a header/flag/sdkconfig/command change
failed_marker.write_text(checksum + "\n", encoding="utf-8")
os.utime(header)
return
failed_marker.unlink(missing_ok=True)
sum_path.write_text(checksum + "\n", encoding="utf-8")
# The OBJECT_DEPENDS edge watches the header; bump it so consumers of
# the previous .gch recompile
os.utime(header)
def write_project(
+179 -1
View File
@@ -11,13 +11,21 @@ from __future__ import annotations
from collections.abc import Iterable
import hashlib
import json
import logging
import os
from pathlib import Path
import posixpath
import re
import subprocess
from esphome.build_helpers.ccache import parse_enable_env
from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env
from esphome.build_helpers.idedata import (
CXX_SOURCE_SUFFIXES,
expand_response_files,
is_launcher,
split_command,
)
_LOGGER = logging.getLogger(__name__)
@@ -122,3 +130,173 @@ def pch_checksum(
digest.update(item.encode())
digest.update(b"\0")
return digest.hexdigest()
# Compile-command tokens dropped when retargeting a TU's flags at the
# prefix header: source/output/depfile flags with an argument, and the
# argument-less depfile flags (the pch compile must not touch depfiles)
_PCH_STRIP_FLAGS_WITH_ARG = frozenset({"-o", "-c", "-MT", "-MF", "-MQ"})
_PCH_STRIP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MM", "-M"})
def pch_compile_command(build_dir: Path, header: Path, gch: Path) -> list[str] | None:
"""The exact src C++ flags from compile_commands.json, retargeted at
the header; None (logged) when no configured C++ TU is available yet."""
from esphome.core import CORE
try:
entries = json.loads(
(build_dir / "compile_commands.json").read_text(encoding="utf-8")
)
except (OSError, json.JSONDecodeError) as err:
# Configure already succeeded, so an unusable DB is a real anomaly
_LOGGER.warning("No usable compile database, skipping pch: %s", err)
return None
if not isinstance(entries, list):
_LOGGER.warning("Malformed compile database, skipping pch")
return None
# Windows compile DBs use backslashes; normalize both sides
src_prefix = str(CORE.relative_src_path()).replace("\\", "/")
entry = next(
(
e
for e in entries
if isinstance(e, dict)
and e.get("file", "").replace("\\", "/").startswith(src_prefix)
and e.get("file", "").endswith(CXX_SOURCE_SUFFIXES)
),
None,
)
if entry is None:
_LOGGER.warning("No src C++ entry in the compile database, skipping pch")
return None
tokens = expand_response_files(
split_command(entry.get("command", "")), Path(entry.get("directory", build_dir))
)
# A DB recorded with ccache enabled prefixes the compiler with the
# launcher; the .gch must be compiled directly
if tokens and is_launcher(tokens[0]):
tokens = tokens[1:]
if not tokens:
# An "arguments"-style or empty entry must skip cleanly, not spawn
# a compiler-less argv that warns on every build
_LOGGER.warning("Compile database entry has no usable command, skipping pch")
return None
args: list[str] = []
arg_it = iter(tokens)
for tok in arg_it:
if tok in _PCH_STRIP_FLAGS_WITH_ARG:
next(arg_it, None)
continue
if tok in _PCH_STRIP_FLAGS:
continue
if tok == "-include":
# Drop only the injected prefix; user force-includes must reach
# the .gch compile or GCC rejects it over the macro mismatch
inc = next(arg_it, "")
if not inc.endswith(PCH_HEADER_NAME):
args.extend(("-include", inc))
continue
args.append(tok)
return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)]
def discard_pch(build_dir: Path) -> None:
"""Remove the pch sidecars so a stale .gch is never consumed.
Bumps the header only when a .gch was actually removed: TUs compiled
against it have incomplete depfiles, while a repeat failure with no
.gch must not force a full rebuild every build.
"""
header = build_dir / PCH_HEADER_NAME
gch = Path(f"{header}.gch")
had_gch = gch.is_file()
gch.unlink(missing_ok=True)
Path(f"{gch}.sum").unlink(missing_ok=True)
if had_gch and header.is_file():
os.utime(header)
def prepare_pch(
build_dir: Path, include_headers: tuple[str, ...], extra: Iterable[str]
) -> None:
"""Compile ``build_dir``'s .gch from compile_commands.json flags and
write its ccache .sum.
The .sum doubles as the freshness stamp and folds in the compile
command, so a flag-only change rebuilds the .gch; ``extra`` carries
backend identity (framework version, sdkconfig, ...). A failed
compile falls back to the plain header include.
"""
from esphome.core import CORE
header = build_dir / PCH_HEADER_NAME
gch = Path(f"{header}.gch")
sum_path = Path(f"{gch}.sum")
cmd = pch_compile_command(build_dir, header, gch)
if cmd is None:
# Freshness cannot be validated; a leftover .gch must not be consumed
discard_pch(build_dir)
return
# Stripped like ccache's own rewriting (a user CCACHE_BASEDIR wins) so
# identical configs hash identically across devices; the raw build path
# covers unresolved spellings in the compile DB
cmd_id = (
" ".join(cmd)
.replace(effective_ccache_basedir(), "")
.replace(str(CORE.build_path), "")
)
checksum = pch_checksum(
CORE.relative_src_path(),
include_headers,
(
# The closure is sorted, so root order only enters via the text
pch_header_text(include_headers),
*extra,
cmd_id,
),
)
if (
gch.is_file()
and sum_path.is_file()
and sum_path.read_text(encoding="utf-8").strip() == checksum
):
return
failed_marker = Path(f"{gch}.failed")
if (
failed_marker.is_file()
and failed_marker.read_text(encoding="utf-8").strip() == checksum
):
_LOGGER.info(
"Precompiled header disabled after an earlier failure; delete %s to retry",
failed_marker,
)
return
try:
result = subprocess.run(
cmd, cwd=build_dir, capture_output=True, text=True, check=False, timeout=300
)
error = None
if result.returncode != 0:
error = result.stderr.strip() or f"exit code {result.returncode}"
elif not gch.is_file():
error = "compiler produced no .gch"
except (OSError, subprocess.SubprocessError) as err:
# Transient (timeout, spawn/IO): warn and retry next build, no marker
_LOGGER.warning("Precompiled header compile did not run: %s", err)
discard_pch(build_dir)
return
if error is not None:
_LOGGER.warning(
"Precompiled header failed; compiling without it: %s", error[:400]
)
discard_pch(build_dir)
# Skip retries until a header/flag/backend-identity/command change
failed_marker.write_text(checksum + "\n", encoding="utf-8")
os.utime(header)
return
failed_marker.unlink(missing_ok=True)
sum_path.write_text(checksum + "\n", encoding="utf-8")
# Consumers depend on the header (depfiles cannot see through a .gch);
# bump it so users of the previous .gch recompile
os.utime(header)
+22 -22
View File
@@ -549,7 +549,7 @@ def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None:
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile),
):
prepare_pch()
checksum = (dev / "build" / "esphome_pch.h.gch.sum").read_text().strip()
@@ -557,7 +557,7 @@ def test_prepare_pch_writes_header_and_sum(tmp_path: Path) -> None:
# Unchanged inputs: the second call must not recompile
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError),
):
prepare_pch()
@@ -579,7 +579,7 @@ def test_pch_no_device_path_poison(tmp_path: Path) -> None:
with (
patch.object(CORE, "name", name),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile),
):
prepare_pch()
content = get_component_cmakelists()
@@ -600,13 +600,13 @@ def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None
def test_pch_compile_command_variants(tmp_path: Path) -> None:
"""Missing DB, no matching entry, and launcher-prefixed commands."""
from esphome.build_gen.espidf import _pch_compile_command
from esphome.build_helpers.pch import pch_compile_command
build = tmp_path / "build"
build.mkdir()
header = build / "esphome_pch.h"
gch = build / "esphome_pch.h.gch"
assert _pch_compile_command(build, header, gch) is None
assert pch_compile_command(build, header, gch) is None
(build / "compile_commands.json").write_text(
json.dumps(
@@ -615,7 +615,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None:
]
)
)
assert _pch_compile_command(build, header, gch) is None
assert pch_compile_command(build, header, gch) is None
src_file = str(tmp_path / "src" / "esphome" / "a.cpp")
(build / "compile_commands.json").write_text(
@@ -634,7 +634,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None:
)
)
# Launcher stripped; -include/-o/-c and depfile flags removed
assert _pch_compile_command(build, header, gch) == [
assert pch_compile_command(build, header, gch) == [
"g++",
"-DX=1",
"-x",
@@ -649,7 +649,7 @@ def test_pch_compile_command_variants(tmp_path: Path) -> None:
def test_pch_compile_command_rejects_unusable_entries(tmp_path: Path) -> None:
"""Malformed DB shapes and command-less entries skip cleanly instead of
producing a compiler-less argv retried every build."""
from esphome.build_gen.espidf import _pch_compile_command
from esphome.build_helpers.pch import pch_compile_command
build = tmp_path / "build"
build.mkdir()
@@ -659,16 +659,16 @@ def test_pch_compile_command_rejects_unusable_entries(tmp_path: Path) -> None:
src_file = str(tmp_path / "src" / "esphome" / "a.cpp")
db.write_text(json.dumps({"not": "a list"}))
assert _pch_compile_command(build, header, gch) is None
assert pch_compile_command(build, header, gch) is None
db.write_text(json.dumps(["just a string"]))
assert _pch_compile_command(build, header, gch) is None
assert pch_compile_command(build, header, gch) is None
# arguments-style entry (allowed by the spec, unused by CMake)
db.write_text(
json.dumps([{"arguments": ["g++", "-c", src_file], "file": src_file}])
)
assert _pch_compile_command(build, header, gch) is None
assert pch_compile_command(build, header, gch) is None
def test_pch_header_list_order_is_in_checksum(
@@ -688,7 +688,7 @@ def test_pch_header_list_order_is_in_checksum(
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile),
):
espidf_mod.prepare_pch()
first = (dev / "build" / "esphome_pch.h.gch.sum").read_text()
@@ -712,7 +712,7 @@ def test_prepare_pch_failure_writes_marker_and_skips_retry(tmp_path: Path) -> No
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=failing_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=failing_compile),
):
prepare_pch()
prepare_pch()
@@ -737,7 +737,7 @@ def test_prepare_pch_spawn_oserror_is_transient(tmp_path: Path) -> None:
before = header.stat().st_mtime_ns
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=raising),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=raising),
):
prepare_pch()
prepare_pch()
@@ -762,7 +762,7 @@ def test_prepare_pch_transient_with_stale_gch_bumps_header(tmp_path: Path) -> No
with (
patch.object(CORE, "name", "test"),
patch(
"esphome.build_gen.espidf.subprocess.run",
"esphome.build_helpers.pch.subprocess.run",
side_effect=OSError("no such compiler"),
),
):
@@ -779,7 +779,7 @@ def test_prepare_pch_disabled_is_noop(
monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
dev = _make_pch_device(tmp_path, "dev_d")
CORE.build_path = dev
with patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError):
with patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError):
prepare_pch()
@@ -792,7 +792,7 @@ def test_prepare_pch_without_compile_commands(tmp_path: Path) -> None:
CORE.build_path = dev
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=AssertionError),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=AssertionError),
):
prepare_pch()
assert not (dev / "build" / "esphome_pch.h.gch.sum").exists()
@@ -860,7 +860,7 @@ def test_prepare_pch_zero_exit_without_gch_is_failure(tmp_path: Path) -> None:
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=no_output),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=no_output),
):
prepare_pch()
assert not (dev / "build" / "esphome_pch.h.gch.sum").exists()
@@ -887,7 +887,7 @@ def test_prepare_pch_bumps_header_for_object_depends(tmp_path: Path) -> None:
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile),
):
prepare_pch()
assert header.stat().st_mtime > before
@@ -914,7 +914,7 @@ def test_prepare_pch_command_change_invalidates_sum(tmp_path: Path) -> None:
with (
patch.object(CORE, "name", "test"),
patch("esphome.build_gen.espidf.subprocess.run", side_effect=fake_compile),
patch("esphome.build_helpers.pch.subprocess.run", side_effect=fake_compile),
):
prepare_pch()
first = (dev / "build" / "esphome_pch.h.gch.sum").read_text()
@@ -925,7 +925,7 @@ def test_prepare_pch_command_change_invalidates_sum(tmp_path: Path) -> None:
def test_prepare_pch_keeps_user_force_includes(tmp_path: Path) -> None:
from esphome.build_gen.espidf import _pch_compile_command
from esphome.build_helpers.pch import pch_compile_command
dev = _make_pch_device(tmp_path, "dev_u")
CORE.build_path = dev
@@ -945,6 +945,6 @@ def test_prepare_pch_keeps_user_force_includes(tmp_path: Path) -> None:
]
)
)
cmd = _pch_compile_command(build, build / "esphome_pch.h", build / "x.gch")
cmd = pch_compile_command(build, build / "esphome_pch.h", build / "x.gch")
assert "user.h" in cmd
assert "esphome_pch.h" not in " ".join(cmd[:-3])