diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index bd13726d85..ec0ff416e5 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -9,15 +9,9 @@ import os from pathlib import Path import posixpath import re -import sys from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env -from esphome.const import ( - PLATFORM_BK72XX, - PLATFORM_LN882X, - PLATFORM_NRF52, - PLATFORM_RTL87XX, -) +from esphome.const import PLATFORM_NRF52 from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -40,8 +34,12 @@ PCH_GUARD_TEXT = f"""\ #endif """ +# The cc1plus wrapper the PlatformIO script writes on arm64 macOS +PCH_CC1_DIR = "pch_cc1" + # What the PlatformIO script leaves in the project root, for cleanup PCH_ARTIFACT_NAMES = (PCH_HEADER_NAME, PCH_GCH_NAME, PCH_SUM_NAME, PCH_SOURCE_NAME) +PCH_ARTIFACT_DIRS = (PCH_CC1_DIR,) # The core headers every backend precompiles PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) @@ -53,17 +51,6 @@ PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset( } ) -# The GCC 10.3 of LibreTiny only loads its own .gch back on Linux: elsewhere -# the compiler is loaded at a new address each run, which GCC accepts from -# version 12. Remove once LibreTiny ships GCC 12 or newer -PCH_SCRIPT_LINUX_ONLY_PLATFORMS = frozenset( - { - PLATFORM_BK72XX, - PLATFORM_LN882X, - PLATFORM_RTL87XX, - } -) - # What ccache needs to cache compiles that load a .gch _CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros") @@ -103,10 +90,7 @@ def pch_script_enabled() -> bool: """Whether this PlatformIO build takes the pch script.""" from esphome.core import CORE - platform = CORE.target_platform - if platform in PCH_SCRIPT_LINUX_ONLY_PLATFORMS and sys.platform != "linux": - return False - return pch_enabled() and platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS + return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS def pch_header_text(include_headers: Iterable[str]) -> str: diff --git a/esphome/platformio/pch.py.script b/esphome/platformio/pch.py.script index 962f5c3f2c..960d5bd7d9 100644 --- a/esphome/platformio/pch.py.script +++ b/esphome/platformio/pch.py.script @@ -1,10 +1,12 @@ import hashlib import os from pathlib import Path +import platform import posixpath import re import shlex import subprocess +import sys # pylint: disable=E0602 Import("env") # noqa: F821 @@ -29,6 +31,35 @@ _GUARD_TEXT = f"""\ #endif """ _INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE) +# Before GCC 12 a .gch only loads at the address it was saved from. Checked +# for this PlatformIO platform, whose compiler is built for a random address +_OLD_GCC_PIO_PLATFORM = "libretiny" +_MOVABLE_PCH_GCC = 12 +# Relative: compiles run in the project root +_CC1_DIR = "pch_cc1" +_CC1PLUS_WRAPPER = """\ +#!/usr/bin/env -S "{python}" -IS +# Written by ESPHome: starts cc1plus without address randomisation, so it +# loads the precompiled header. See esphome/platformio/pch.py.script. +import ctypes +import sys + +real = {real!r} +libc = ctypes.CDLL("/usr/lib/libSystem.B.dylib") +attr = ctypes.create_string_buffer(64) +libc.posix_spawnattr_init(attr) +# POSIX_SPAWN_SETEXEC | _POSIX_SPAWN_DISABLE_ASLR +libc.posix_spawnattr_setflags(attr, ctypes.c_short(0x0040 | 0x0100)) +# What os.fsencode does, without the import +encoding = sys.getfilesystemencoding() +argv = [real.encode(), *(a.encode(encoding, "surrogateescape") for a in sys.argv[1:]), None] +libc._NSGetEnviron.restype = ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)) +# Replaces this process on success; the return value is the error number +rc = libc.posix_spawn(None, argv[0], None, attr, (ctypes.c_char_p * len(argv))(*argv), + libc._NSGetEnviron().contents) +libc.strerror.restype = ctypes.c_char_p +sys.exit("ESPHome: cannot start " + real + ": " + libc.strerror(rc).decode()) +""" _CXX_SUFFIXES = (".cpp", ".cc", ".cxx") _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".inc") _INCLUDE_DIR_OPTIONS = ("-I", "-isystem", "-iquote", "-idirafter") @@ -81,6 +112,35 @@ def _src_envs(src_dir: Path) -> list: return list(found.values()) or [projenv] # noqa: F821 +def _driver_output(cxx: str, option: str) -> str: + result = subprocess.run([cxx, option], capture_output=True, text=True) # noqa: PLW1510 + return result.stdout.strip() + + +def _old_gcc_flags(proj_dir: Path, pio_platform: str, cxx: str): + """Extra driver flags for an old GCC, or None where its .gch cannot be + used. Linux loads it as is. arm64 macOS loads every program at a random + address, so there the driver gets a -B directory whose cc1plus starts + the real one without that randomisation.""" + if pio_platform != _OLD_GCC_PIO_PLATFORM or sys.platform == "linux": + return [] + major = _driver_output(cxx, "-dumpversion").partition(".")[0] + if major.isdigit() and int(major) >= _MOVABLE_PCH_GCC: + return [] + if sys.platform != "darwin" or platform.machine() != "arm64": + return None + # The driver prints the bare name when it has no cc1plus of its own + real = _driver_output(cxx, "-print-prog-name=cc1plus") + if not os.path.isabs(real): + return None + wrapper = proj_dir / _CC1_DIR / "cc1plus" + wrapper.parent.mkdir(exist_ok=True) + real = os.path.normpath(real) + wrapper.write_text(_CC1PLUS_WRAPPER.format(python=sys.executable, real=real)) + wrapper.chmod(0o755) + return [f"-B{_CC1_DIR}/"] + + def _setup_pch() -> None: if projenv is None: if "nobuild" in COMMAND_LINE_TARGETS: # noqa: F821 @@ -97,6 +157,11 @@ def _setup_pch() -> None: src_envs = _src_envs(src_dir) cxx = src_envs[0].subst("$CXX") + pio_platform = env.PioPlatform() # noqa: F821 + extra = _old_gcc_flags(proj_dir, pio_platform.name, cxx) + if extra is None: + print("ESPHome: this compiler cannot load a precompiled header here; compiling without it") + return # The header holds the -include entries itself, so the .gch compile must # not see them; consumers keep theirs, which the .gch then satisfies flags = [] @@ -133,9 +198,8 @@ def _setup_pch() -> None: digest.update(flags_id.encode()) # GCC does not check a .gch against its headers and PlatformIO package # paths carry no version - platform = env.PioPlatform() # noqa: F821 - for package in sorted(platform.packages): - digest.update(f"{package}={platform.get_package_version(package)}\0".encode()) + for package in sorted(pio_platform.packages): + digest.update(f"{package}={pio_platform.get_package_version(package)}\0".encode()) # Libraries carry their version in lib_deps digest.update(repr(env.GetProjectOption("lib_deps", [])).encode()) # noqa: F821 closure = _include_closure(src_dir, headers) @@ -183,7 +247,7 @@ def _setup_pch() -> None: header.write_text(_GUARD_TEXT, encoding="utf-8") source.write_text(content, encoding="utf-8") result = subprocess.run( # noqa: PLW1510 - [cxx, "-x", "c++-header", *flags, "-c", str(source), "-o", str(gch)], + [cxx, "-x", "c++-header", *flags, *extra, "-c", str(source), "-o", str(gch)], cwd=proj_dir, capture_output=True, text=True, @@ -196,7 +260,7 @@ def _setup_pch() -> None: # Prepended: GCC only loads a .gch for the first -include for src_env in src_envs: - src_env.Prepend(CXXFLAGS=_CONSUMER_FLAGS) + src_env.Prepend(CXXFLAGS=[*extra, *_CONSUMER_FLAGS]) print( "ESPHome: Compiling with a precompiled header " "(set ESPHOME_PCH_ENABLE=0 to disable)" diff --git a/esphome/writer.py b/esphome/writer.py index 9f2ff85cff..0bd485bfd3 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -611,10 +611,13 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False): _LOGGER.info("Deleting %s", idf_path) rmtree(idf_path) # The PlatformIO pch files live in the project root - from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES + from esphome.build_helpers.pch import PCH_ARTIFACT_DIRS, PCH_ARTIFACT_NAMES for name in PCH_ARTIFACT_NAMES: CORE.relative_build_path(name).unlink(missing_ok=True) + for name in PCH_ARTIFACT_DIRS: + if (pch_dir := CORE.relative_build_path(name)).is_dir(): + rmtree(pch_dir) # idedata caches live under the data dir, not the build path; globbed # so a future backend suffix cannot drift out of clean-all diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index 0ea8215738..a03b9c3a11 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -113,23 +113,12 @@ def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None: "platform", ["bk72xx", "esp32", "esp8266", "ln882x", "rp2", "rtl87xx"] ) def test_pch_script_enabled(platform: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(pch.sys, "platform", "linux") CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} assert pch.pch_script_enabled() monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") assert not pch.pch_script_enabled() -@pytest.mark.parametrize("host", ["darwin", "win32"]) -@pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_LINUX_ONLY_PLATFORMS)) -def test_pch_script_linux_only_platform( - platform: str, host: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(pch.sys, "platform", host) - CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} - assert not pch.pch_script_enabled() - - @pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_EXCLUDED_PLATFORMS)) def test_pch_script_excluded_platform(platform: str) -> None: CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} diff --git a/tests/unit_tests/test_platformio_pch_script.py b/tests/unit_tests/test_platformio_pch_script.py index bccfc17477..e7ea1a3420 100644 --- a/tests/unit_tests/test_platformio_pch_script.py +++ b/tests/unit_tests/test_platformio_pch_script.py @@ -6,6 +6,8 @@ from collections.abc import Callable import os from pathlib import Path import stat +import subprocess +import sys from types import SimpleNamespace from unittest.mock import patch @@ -22,6 +24,7 @@ _SCRIPT = Path(toolchain.__file__).parent / "pch.py.script" class _FakePlatform: + name = "fake" packages = {"framework-x": {}, "toolchain-y": {}} def get_package_version(self, name: str) -> str | None: @@ -72,7 +75,13 @@ class _FakeSConsEnv(dict): def _fake_cxx(tmp_path: Path, fail: bool = False) -> Path: """A compiler stand-in that records its argv and writes the -o target.""" cxx = tmp_path / "fake-gxx" + # As GCC: the program's path when it exists next to the driver, else its + # name; the version from FAKE_GCC_VERSION body = ( + 'case "$1" in -print-prog-name=*) n=${1#*=};' + ' p="$(dirname "$0")/../libexec/gcc/arm-none-eabi/10.3.1/$n";' + ' [ -x "$p" ] && echo "$p" || echo "$n"; exit 0;;' + ' -dumpversion) echo "${FAKE_GCC_VERSION:-10.3.1}"; exit 0;; esac\n' 'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n' ) if fail: @@ -93,6 +102,7 @@ def _run_script( name: str = "dev", platform_cls: type[_FakePlatform] = _FakePlatform, build_files: Callable[[tuple], list] | None = None, + cxx: Path | None = None, ) -> _FakeSConsEnv: proj = tmp_path / name src = proj / "src" @@ -101,7 +111,8 @@ def _run_script( (src / "esphome" / "core" / "pch_prefix.h").write_text( '#include "esphome/core/defines.h"\n' ) - cxx = _fake_cxx(tmp_path, fail=fail) + if cxx is None: + cxx = _fake_cxx(tmp_path, fail=fail) args = (proj, src, str(cxx), flags or ["-DX=1"], platform_cls) # Distinct objects: the -include flags must land on projenv only global_env = _FakeSConsEnv(*args) @@ -137,6 +148,114 @@ def test_pch_script_builds_and_prepends_relative_include(tmp_path: Path) -> None assert scons_env.global_env.prepended == [] +class _LibreTinyPlatform(_FakePlatform): + name = "libretiny" + + +def _run_on_host( + tmp_path: Path, + host: str, + machine: str, + platform_cls: type[_FakePlatform], + cxx: Path | None = None, + gcc_version: str = "10.3.1", +) -> _FakeSConsEnv: + with ( + patch.object(sys, "platform", host), + patch("platform.machine", return_value=machine), + ): + return _run_script( + tmp_path, + platform_cls=platform_cls, + cxx=cxx, + env_vars={"FAKE_GCC_VERSION": gcc_version}, + ) + + +def _fake_toolchain(tmp_path: Path) -> tuple[Path, Path]: + """A driver in bin/ and a cc1plus that records its argv in libexec/.""" + toolchain = tmp_path / "toolchain" + (toolchain / "bin").mkdir(parents=True) + cxx = _fake_cxx(toolchain / "bin") + real = toolchain / "libexec" / "gcc" / "arm-none-eabi" / "10.3.1" / "cc1plus" + real.parent.mkdir(parents=True) + real.write_text('#!/bin/sh\nprintf \'%s\\n\' "$@" > "$0.argv"\n') + real.chmod(0o755) + return cxx, real + + +def test_pch_script_gcc10_wrapper_on_apple_silicon(tmp_path: Path) -> None: + """The LibreTiny .gch compile and the consumers get a -B directory + holding a cc1plus that starts the real one.""" + cxx, real = _fake_toolchain(tmp_path) + scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx) + wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus" + assert wrapper.stat().st_mode & stat.S_IXUSR + assert repr(str(real)) in wrapper.read_text(encoding="utf-8") + argv = Path(f"{cxx}.argv").read_text(encoding="utf-8").split("\n") + assert f"-B{pch.PCH_CC1_DIR}/" in argv + assert scons_env.prepended == [f"-B{pch.PCH_CC1_DIR}/", *pch.pch_consumer_flags()] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="the wrapper is macOS only") +def test_pch_script_gcc10_wrapper_starts_the_real_cc1plus(tmp_path: Path) -> None: + cxx, real = _fake_toolchain(tmp_path) + _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx) + wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus" + result = subprocess.run( + [str(wrapper), "-quiet", "x.cpp"], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + assert Path(f"{real}.argv").read_text(encoding="utf-8") == "-quiet\nx.cpp\n" + + +def test_pch_script_gcc10_without_cc1plus_builds_plainly( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A driver without a cc1plus of its own gets no wrapper and no header.""" + scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform) + assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists() + assert scons_env.prepended == [] + assert "compiling without it" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("host", "machine"), [("darwin", "x86_64"), ("win32", "AMD64")] +) +def test_pch_script_gcc10_skipped_elsewhere( + tmp_path: Path, host: str, machine: str, capsys: pytest.CaptureFixture[str] +) -> None: + """Where the GCC 10 .gch cannot load, the build runs without it.""" + scons_env = _run_on_host(tmp_path, host, machine, _LibreTinyPlatform) + assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists() + assert scons_env.prepended == [] + assert "compiling without it" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("host", "machine", "platform_cls", "gcc_version"), + [ + ("darwin", "arm64", _FakePlatform, "10.3.1"), + ("linux", "aarch64", _LibreTinyPlatform, "10.3.1"), + # From GCC 12 the .gch loads at any address, so the rule retires itself + ("darwin", "arm64", _LibreTinyPlatform, "12.2.0"), + ("win32", "AMD64", _LibreTinyPlatform, "14.2.0"), + ], +) +def test_pch_script_no_wrapper_where_the_gch_loads( + tmp_path: Path, + host: str, + machine: str, + platform_cls: type[_FakePlatform], + gcc_version: str, +) -> None: + scons_env = _run_on_host( + tmp_path, host, machine, platform_cls, gcc_version=gcc_version + ) + assert not (tmp_path / "dev" / pch.PCH_CC1_DIR).exists() + assert scons_env.prepended == pch.pch_consumer_flags() + + def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: """The script cannot import esphome, so its copies are pinned.""" namespace: dict[str, object] = { @@ -152,6 +271,7 @@ def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: assert namespace["_CONSUMER_FLAGS"] == pch.pch_consumer_flags() assert namespace["_GUARD_TEXT"] == pch.PCH_GUARD_TEXT assert namespace["_INCLUDE_RE"].pattern == pch._INCLUDE_RE.pattern + assert namespace["_CC1_DIR"] == pch.PCH_CC1_DIR def test_pch_script_compile_failure_stops_the_build(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 432a2065bf..a1ae7f9ec7 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -684,11 +684,17 @@ def test_clean_build_partial_removes_pch_artifacts( ) -> None: """The PlatformIO pch sidecars live at the project root and must go in a partial clean, like the native backend's under .pioenvs.""" - from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES as names + from esphome.build_helpers.pch import ( + PCH_ARTIFACT_DIRS as dirs, + PCH_ARTIFACT_NAMES as names, + ) assert "esphome_pch.h.gch" in names for name in names: (tmp_path / name).write_text("x") + for name in dirs: + (tmp_path / name).mkdir() + (tmp_path / name / "cc1plus").write_text("x") mock_core.relative_pioenvs_path.return_value = tmp_path / ".pioenvs" mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps" mock_core.relative_build_path.side_effect = lambda name: tmp_path / name @@ -696,7 +702,7 @@ def test_clean_build_partial_removes_pch_artifacts( clean_build() - for name in names: + for name in (*names, *dirs): assert not (tmp_path / name).exists()