diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index bf099d4ebf..2784f13511 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -196,6 +196,7 @@ jobs: # the default. id: - esp8266-arduino + - esp8266-arduino-native - esp32-arduino-platformio - esp32-arduino-esp-idf - esp32-idf-platformio diff --git a/docker/test_configs/esp8266-arduino-native.yaml b/docker/test_configs/esp8266-arduino-native.yaml new file mode 100644 index 0000000000..99eab33ecb --- /dev/null +++ b/docker/test_configs/esp8266-arduino-native.yaml @@ -0,0 +1,8 @@ +esphome: + name: docker-test-esp8266-native + +esp8266: + board: d1_mini + toolchain: arduino + +logger: diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 4a0e29ab3c..c66292a836 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -25,6 +25,7 @@ import subprocess from typing import TYPE_CHECKING, NamedTuple from esphome.arduino8266.framework import toolchain_tool +from esphome.build_helpers.idedata import is_joined_include from esphome.build_helpers.ninja import ( escape as _e, quote_path as _q, @@ -36,8 +37,11 @@ from esphome.build_helpers.ninja_gen import ( compile_edges, compile_rule_lines, library_edges, + pch_edges, + pch_rule_lines, tool_lines, ) +from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS from esphome.components.esp8266 import build_surgery from esphome.components.esp8266.boards import ( BOARDS, @@ -1057,6 +1061,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: ), *compile_rule_lines(), *ar_rule_lines(toolchain_tool(paths.toolchain, "ar")), + *pch_rule_lines(), "rule link", " command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group", " rspfile = $out.rsp", @@ -1111,7 +1116,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # One source of truth with the PlatformIO path: esp8266/__init__ pins # build_src_flags (the throw_stubs force-include); -include paths # resolve against the source root - src_parts: list[str] = [] + src_other: list[str] = [] + src_includes: list[str] = [] src_it = iter( lex_build_flags(_pio_option("build_src_flags", ""), "build_src_flags") ) @@ -1122,15 +1128,32 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: raise EsphomeError( "build_src_flags has a trailing '-include' with no header" ) - src_parts.append(f"-include {_q(src_dir / header)}") + src_includes.append(header) + elif is_joined_include(tok): + # Left in src_other it would precede the pch include + src_includes.append(tok[len("-include") :]) else: - src_parts.append(_shell_token(tok)) - src_extra = " ".join(src_parts) + src_other.append(_shell_token(tok)) + include_flags = [f"-include {_q(src_dir / h)}" for h in src_includes] # One shared variable instead of repeating the flags line on every src # edge (hundreds of edges in a real project) - lines.append(f"srcflags = {src_extra}") + lines.append(f"srcflags = {' '.join(src_other + include_flags)}") + src_cxx_override = pch_edges( + lines, + build_dir, + src_dir, + (*src_includes, *PCH_DEFAULT_HEADERS), + flag_sets.cxxflags, + src_other, + (str(paths.framework), str(paths.toolchain)), + ) src_objs = compile_edges( - lines, collect_sources(src_dir), src_dir, "src", flags="$srcflags" + lines, + collect_sources(src_dir), + src_dir, + "src", + flags="$srcflags", + cxx_override=src_cxx_override, ) ld_deps = [f"ld/{_COMMON_LD_NAME}"] diff --git a/esphome/build_helpers/ccache.py b/esphome/build_helpers/ccache.py index 541dcbe3d3..df9aa8c9a0 100644 --- a/esphome/build_helpers/ccache.py +++ b/esphome/build_helpers/ccache.py @@ -102,15 +102,30 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: return {k: v for k, v in defaults.items() if k not in os.environ} +def effective_ccache_basedir() -> str: + """The prefix ccache strips from hashed paths: a usable user + CCACHE_BASEDIR, else the resolved build path.""" + from esphome.core import CORE + + raw = os.environ.get("CCACHE_BASEDIR") + if raw is not None and Path(raw).is_absolute() and len(Path(raw).parts) > 1: + return raw + return str(Path(CORE.build_path).resolve()) + + def ccache_env(ccache: str | None, tools_cache: tuple[str, str]) -> dict[str, str]: """The ccache settings for a build subprocess (not os.environ). ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None when disabled; ``tools_cache`` is the backend's tools cache spec, which holds - its ccache dir. + its ccache dir. The pch settings include ``time_macros``: a cached + object can keep an older ``__DATE__`` or ``__TIME__``. """ if ccache is None: return {} + from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import tools_cache_path - return ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache") + env = ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache") + env.update(ccache_pch_env()) + return env diff --git a/esphome/build_helpers/idedata.py b/esphome/build_helpers/idedata.py index 98ee804403..62bea83022 100644 --- a/esphome/build_helpers/idedata.py +++ b/esphome/build_helpers/idedata.py @@ -152,6 +152,15 @@ def _is_launcher(token: str) -> bool: return Path(token).stem.lower() in _LAUNCHER_STEMS +def is_joined_include(tok: str) -> bool: + """The joined ``-includefoo.h`` spelling; excludes clang's -include-pch.""" + return ( + tok.startswith("-include") + and tok != "-include" + and not tok.startswith("-include-") + ) + + def parse_entry( entry: dict, launcher: str | None = None ) -> tuple[str, list[str], list[str], list[str]]: diff --git a/esphome/build_helpers/ninja_gen.py b/esphome/build_helpers/ninja_gen.py index d4cfc301ac..40cc8e7400 100644 --- a/esphome/build_helpers/ninja_gen.py +++ b/esphome/build_helpers/ninja_gen.py @@ -13,12 +13,23 @@ from pathlib import Path import sys from typing import TYPE_CHECKING +from esphome.build_helpers.idedata import is_joined_include from esphome.build_helpers.ninja import ( escape as _e, quote_path as _q, shell_token as _shell_token, ) +from esphome.build_helpers.pch import ( + PCH_GCH_NAME, + PCH_SUM_NAME, + log_pch_in_use, + pch_consumer_flags, + pch_enabled, + pch_identity, + write_pch_headers, +) from esphome.framework_helpers import strip_win_long_path_prefix +from esphome.helpers import write_file_if_changed from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX if TYPE_CHECKING: @@ -94,6 +105,62 @@ def compile_rule_lines() -> list[str]: ] +def pch_rule_lines() -> list[str]: + """The precompiled header rule, for a generator that emits one.""" + return [ + # No $ccache: the .gch embeds build dir paths + "rule pch", + " command = $cxx -MMD -MF $out.d -x c++-header $cxxflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = PCH $out", + ] + + +def pch_edges( + lines: list[str], + build_dir: Path, + src_dir: Path, + headers: Sequence[str], + cxxflags: Sequence[str], + src_flags: Sequence[str], + identity: Sequence[str], +) -> tuple[str, str] | None: + """Emit the precompiled header for the C++ src edges. + + ``headers`` are folded into one prefix header, ``src_flags`` are the + flags every src edge carries and ``identity`` names what else the + compile depends on, the compiler included. Returns the ``cxx_override`` + for ``compile_edges``, or None without a pch. + """ + if not pch_enabled(): + return None + if any( + tok == "-include" or tok.startswith("--include") or is_joined_include(tok) + for tok in cxxflags + ): + # $cxxflags expands first and GCC only loads a .gch for the first + # -include + _LOGGER.warning( + "A -include in the compiler flags prevents the precompiled header from " + "loading; compiling without it" + ) + return None + log_pch_in_use() + source = write_pch_headers(build_dir, headers) + write_file_if_changed( + build_dir / PCH_SUM_NAME, + pch_identity([*cxxflags, *src_flags], src_dir, tuple(headers), identity) + "\n", + ) + gch = _e(PCH_GCH_NAME) + # The checksum file changes with anything the .gch depends on + lines.append(f"build {gch}: pch {_e(source)} | {_e(PCH_SUM_NAME)}") + if src_flags: + lines.append(f" flags = {' '.join(src_flags)}") + lines.append(f"srccxxflags = {' '.join([*src_flags, *pch_consumer_flags()])}") + return ("$srccxxflags", gch) + + def ar_rule_lines(ar: Path | str) -> list[str]: return [ "rule ar", @@ -111,20 +178,26 @@ def compile_edges( group: str, flags: str = "", own_includes: str = "", + cxx_override: tuple[str, str] | None = None, ) -> list[str]: - """Emit compile edges for ``sources``; return the object paths.""" + """Emit compile edges for ``sources``; return the object paths. + + ``cxx_override`` is a (flags, implicit-dep) pair applied to C++ edges + only, replacing ``flags`` (used for the precompiled header). + """ objects = [] for src in sources: rel = src.relative_to(root).as_posix() obj = f"obj/{group}/{rel}.o" escaped_obj = _e(obj) - lines.append( - f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}" - ) + kind = SOURCE_KIND_FOR_SUFFIX[src.suffix] + override = cxx_override if kind == "cxx" else None + implicit = f" | {override[1]}" if override else "" + lines.append(f"build {escaped_obj}: {kind} {_e(src)}{implicit}") if own_includes: lines.append(f" own_includes = {own_includes}") - if flags: - lines.append(f" flags = {flags}") + if edge_flags := override[0] if override else flags: + lines.append(f" flags = {edge_flags}") # Escaped once here: the returned paths only ever appear in build # statements (archive/link inputs), which use ninja escaping objects.append(escaped_obj) diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index d93c186c7c..bfe81d8504 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -10,10 +10,29 @@ from pathlib import Path import posixpath import re -from esphome.build_helpers.ccache import parse_enable_env +from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env +from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) +# The header and its sidecars live in the build directory +PCH_HEADER_NAME = "esphome_pch.h" +PCH_GCH_NAME = f"{PCH_HEADER_NAME}.gch" +# ccache hashes this instead of the .gch; also the freshness stamp +PCH_SUM_NAME = f"{PCH_GCH_NAME}.sum" +# The include list the .gch is compiled from +PCH_SOURCE_NAME = "esphome_pch_src.h" + +# GCC can skip a .gch without a diagnostic and read the header of the same +# name, so that header is an error. Other tools get the include list. +PCH_GUARD_TEXT = f"""\ +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTELLISENSE__) +#error "The precompiled header was not loaded" +#else +#include "{PCH_SOURCE_NAME}" +#endif +""" + # The core headers every backend precompiles PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) @@ -29,6 +48,12 @@ def pch_enabled() -> bool: return parse_enable_env("ESPHOME_PCH_ENABLE") is not False +def pch_consumer_flags() -> list[str]: + """Flags a C++ src compile loads the pch with. The -include stays + relative: an absolute path would enter the ccache key.""" + return ["-Winvalid-pch", "-Werror=invalid-pch", "-include", PCH_HEADER_NAME] + + def ccache_pch_env() -> dict[str, str]: """What ccache needs to cache compiles that load a .gch, added to what the user already set.""" @@ -46,6 +71,20 @@ def ccache_pch_env() -> dict[str, str]: return env +def pch_header_text(include_headers: Iterable[str]) -> str: + """The prefix-header source: exactly these includes, in order.""" + return "".join(f'#include "{name}"\n' for name in include_headers) + + +def write_pch_headers(build_dir: Path, include_headers: Iterable[str]) -> Path: + """Write the guard header and the include list; return the latter, + which is what the .gch compiles from.""" + write_file_if_changed(build_dir / PCH_HEADER_NAME, PCH_GUARD_TEXT) + source = build_dir / PCH_SOURCE_NAME + write_file_if_changed(source, pch_header_text(include_headers)) + return source + + def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]: """Include closure of ``roots``: src-relative name -> contents. @@ -87,6 +126,27 @@ def pch_checksum( return digest.hexdigest() +def pch_identity( + tokens: Iterable[str], + src_dir: Path, + include_headers: tuple[str, ...], + extra: Iterable[str], +) -> str: + """The .sum digest: include closure, header text, ``extra`` and the + compile flags with the build path stripped, as ccache does.""" + from esphome.core import CORE + + flags = ( + " ".join(tokens) + .replace(str(CORE.build_path), "") + .replace(effective_ccache_basedir(), "") + ) + # The closure is sorted, so header order only enters via the text + return pch_checksum( + src_dir, include_headers, (pch_header_text(include_headers), *extra, flags) + ) + + _DISABLE_HINT = " (set ESPHOME_PCH_ENABLE=0 to disable)" diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 65a78c0253..8bd304b42b 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -364,7 +364,87 @@ def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None: line for line in content.splitlines() if line.startswith(" flags = ") ] assert flags_lines - assert all(line == " flags = $srcflags" for line in flags_lines) + # C++ src edges consume the precompiled header; C/assembly keep srcflags + assert set(flags_lines) == {" flags = $srcflags", " flags = $srccxxflags"} + + +def test_write_project_pch(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths, ccache="/usr/bin/ccache") + build_dir = CORE.relative_pioenvs_path(CORE.name) + assert "rule pch" in content + # Compiled from the include list, and again when the checksum changes + gch_edge = next( + line + for line in content.splitlines() + if line.startswith("build esphome_pch.h.gch: pch ") + ) + assert gch_edge.endswith("esphome_pch_src.h | esphome_pch.h.gch.sum") + for line in content.splitlines(): + # C++ edges wait on the .gch; the C edge must not reference it + if line.startswith("build obj/src/main.cpp.o:"): + assert line.endswith("| esphome_pch.h.gch") + if line.startswith("build obj/src/esphome/vendor.c.o:"): + assert "esphome_pch" not in line + assert (build_dir / "esphome_pch_src.h").read_text().splitlines() == [ + '#include "esphome/components/esp8266/throw_stubs.h"', + '#include "esphome/core/pch_prefix.h"', + ] + assert "#error" in (build_dir / "esphome_pch.h").read_text() + assert (build_dir / "esphome_pch.h.gch.sum").read_text().strip() + + +def test_write_project_pch_folds_joined_src_force_include( + tmp_path: Path, +) -> None: + """-includefoo.h in build_src_flags must fold into the pch like the + separated spelling, not precede and defeat it.""" + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + CORE.platformio_options["build_src_flags"] = "-includeesphome/core/defines.h" + content = _write_ninja(paths, ccache="/usr/bin/ccache") + assert "build esphome_pch.h.gch: pch" in content + assert "srccxxflags" in content + assert "-includeesphome" not in content + + +@pytest.mark.parametrize("flag", ["-includefoo.h", "--include=foo.h"]) +def test_write_project_pch_skipped_for_other_force_include_spellings( + tmp_path: Path, caplog: pytest.LogCaptureFixture, flag: str +) -> None: + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", flag) + content = _write_ninja(paths, ccache="/usr/bin/ccache") + assert "esphome_pch" not in content + assert "prevents the precompiled header" in caplog.text + + +def test_write_project_pch_skipped_when_user_force_include_precedes( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A -include in build_flags lands ahead of the pch include, so GCC + would never load the .gch; skip it and say so.""" + paths = _make_framework(tmp_path) + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", "-include foo.h" + ) + content = _write_ninja(paths, ccache="/usr/bin/ccache") + assert "esphome_pch" not in content + assert "srccxxflags" not in content + assert "prevents the precompiled header" in caplog.text + + +def test_write_project_pch_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths) + assert "esphome_pch" not in content + assert "srccxxflags" not in content + assert " flags = $srcflags" in content def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None: @@ -1716,3 +1796,37 @@ def test_write_project_rejects_spaced_ldscript_override(tmp_path: Path) -> None: CORE.relative_src_path().mkdir(parents=True, exist_ok=True) with pytest.raises(EsphomeError, match="Invalid flash linker script name"): arduino8266.write_project(paths, None) + + +def test_write_project_pch_no_device_path_poison(tmp_path: Path) -> None: + """Regression: the -include stays relative and the .sum carries no + per-device path, or cross-device ccache sharing breaks.""" + paths = _make_framework(tmp_path / "shared") + sums = [] + for name in ("dev_a", "dev_b"): + CORE.name = name + CORE.build_path = tmp_path / name + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths, ccache="/usr/bin/ccache") + assert ( + "srccxxflags = -Winvalid-pch -Werror=invalid-pch " + "-include esphome_pch.h" in content + ) + sums.append( + (CORE.relative_pioenvs_path(name) / "esphome_pch.h.gch.sum").read_text() + ) + assert sums[0] == sums[1] + + +def test_write_project_pch_sum_tracks_src_flags(tmp_path: Path) -> None: + """The header compiles with the src flags too, so they are part of what + ccache is told about it.""" + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + sum_path = CORE.relative_pioenvs_path(CORE.name) / "esphome_pch.h.gch.sum" + sums = [] + for value in ("1", "2"): + CORE.platformio_options["build_src_flags"] = f"-DSRC_ONLY={value}" + _write_ninja(paths) + sums.append(sum_path.read_text()) + assert sums[0] != sums[1] diff --git a/tests/unit_tests/build_helpers/test_ccache.py b/tests/unit_tests/build_helpers/test_ccache.py index 468a97daab..17d79b1ddb 100644 --- a/tests/unit_tests/build_helpers/test_ccache.py +++ b/tests/unit_tests/build_helpers/test_ccache.py @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from esphome.build_helpers import ccache +from esphome.core import CORE def test_resolve_opt_out() -> None: @@ -122,6 +123,20 @@ def test_parse_enable_env_spelling_tables( assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected +def test_effective_ccache_basedir_prefers_user_value(tmp_path: Path) -> None: + CORE.build_path = tmp_path + # Drive-qualified on Windows: "/custom/base" is not absolute there + base = "C:\\custom\\base" if os.name == "nt" else "/custom/base" + with patch.dict(os.environ, {"CCACHE_BASEDIR": base}, clear=True): + assert ccache.effective_ccache_basedir() == base + with patch.dict(os.environ, {}, clear=True): + assert ccache.effective_ccache_basedir() == str(tmp_path.resolve()) + # Degenerate values would strip substrings ccache never rewrites + for bad in ("", "/", "a/b"): + with patch.dict(os.environ, {"CCACHE_BASEDIR": bad}, clear=True): + assert ccache.effective_ccache_basedir() == str(tmp_path.resolve()) + + def test_resolve_absolute_ccache_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -165,3 +180,18 @@ def test_ccache_defaults_env_escapes_a_dollar( env = ccache.ccache_defaults_env(tmp_path / "c$d") assert env["CCACHE_DIR"].endswith("c$$d") assert env["CCACHE_BASEDIR"].endswith("a$$b") + + +def _pch_ccache_env(tmp_path: Path, environ: dict[str, str]) -> dict[str, str]: + CORE.build_path = tmp_path / "build" + spec = ("ESPHOME_TEST_PREFIX", "test") + environ = {"ESPHOME_TEST_PREFIX": str(tmp_path / "cache"), **environ} + with patch.dict(os.environ, environ, clear=True): + return ccache.ccache_env("/usr/bin/ccache", spec) + + +def test_ccache_env_includes_pch_settings(tmp_path: Path) -> None: + """A native build exports the ccache settings the pch needs.""" + env = _pch_ccache_env(tmp_path, {}) + assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros" + assert env["CCACHE_PCH_EXTSUM"] == "true" diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index 5a3ba2cdd7..99109bec5e 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -57,6 +57,11 @@ def test_ccache_pch_env_disabled() -> None: assert pch.ccache_pch_env() == {} +def test_pch_header_text_preserves_order() -> None: + text = pch.pch_header_text(["b.h", "a.h"]) + assert text == '#include "b.h"\n#include "a.h"\n' + + def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None: """Sibling includes resolve against the includer's directory first, full paths against the src root; unresolvable names end the walk."""