[libretiny] Load the precompiled header on Apple Silicon Macs (#19857)

esphome/build_helpers/pch.py

@@ -9,15 +9,9 @@ import os
 from pathlib import Path
 import posixpath
 import re
-import sys
 
 from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env
-from esphome.const import (
-    PLATFORM_BK72XX,
-    PLATFORM_LN882X,
-    PLATFORM_NRF52,
-    PLATFORM_RTL87XX,
-)
+from esphome.const import PLATFORM_NRF52
 from esphome.helpers import write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
@@ -40,8 +34,12 @@ PCH_GUARD_TEXT = f"""\
 #endif
 """
 
+# The cc1plus wrapper the PlatformIO script writes on arm64 macOS
+PCH_CC1_DIR = "pch_cc1"
+
 # What the PlatformIO script leaves in the project root, for cleanup
 PCH_ARTIFACT_NAMES = (PCH_HEADER_NAME, PCH_GCH_NAME, PCH_SUM_NAME, PCH_SOURCE_NAME)
+PCH_ARTIFACT_DIRS = (PCH_CC1_DIR,)
 
 # The core headers every backend precompiles
 PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",)
@@ -53,17 +51,6 @@ PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset(
     }
 )
 
-# The GCC 10.3 of LibreTiny only loads its own .gch back on Linux: elsewhere
-# the compiler is loaded at a new address each run, which GCC accepts from
-# version 12. Remove once LibreTiny ships GCC 12 or newer
-PCH_SCRIPT_LINUX_ONLY_PLATFORMS = frozenset(
-    {
-        PLATFORM_BK72XX,
-        PLATFORM_LN882X,
-        PLATFORM_RTL87XX,
-    }
-)
-
 # What ccache needs to cache compiles that load a .gch
 _CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros")
 
@@ -103,10 +90,7 @@ def pch_script_enabled() -> bool:
     """Whether this PlatformIO build takes the pch script."""
     from esphome.core import CORE
 
-    platform = CORE.target_platform
-    if platform in PCH_SCRIPT_LINUX_ONLY_PLATFORMS and sys.platform != "linux":
-        return False
-    return pch_enabled() and platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS
+    return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS
 
 
 def pch_header_text(include_headers: Iterable[str]) -> str:

esphome/platformio/pch.py.script

@@ -1,10 +1,12 @@
 import hashlib
 import os
 from pathlib import Path
+import platform
 import posixpath
 import re
 import shlex
 import subprocess
+import sys
 
 # pylint: disable=E0602
 Import("env")  # noqa: F821
@@ -29,6 +31,35 @@ _GUARD_TEXT = f"""\
 #endif
 """
 _INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE)
+# Before GCC 12 a .gch only loads at the address it was saved from. Checked
+# for this PlatformIO platform, whose compiler is built for a random address
+_OLD_GCC_PIO_PLATFORM = "libretiny"
+_MOVABLE_PCH_GCC = 12
+# Relative: compiles run in the project root
+_CC1_DIR = "pch_cc1"
+_CC1PLUS_WRAPPER = """\
+#!/usr/bin/env -S "{python}" -IS
+# Written by ESPHome: starts cc1plus without address randomisation, so it
+# loads the precompiled header. See esphome/platformio/pch.py.script.
+import ctypes
+import sys
+
+real = {real!r}
+libc = ctypes.CDLL("/usr/lib/libSystem.B.dylib")
+attr = ctypes.create_string_buffer(64)
+libc.posix_spawnattr_init(attr)
+# POSIX_SPAWN_SETEXEC | _POSIX_SPAWN_DISABLE_ASLR
+libc.posix_spawnattr_setflags(attr, ctypes.c_short(0x0040 | 0x0100))
+# What os.fsencode does, without the import
+encoding = sys.getfilesystemencoding()
+argv = [real.encode(), *(a.encode(encoding, "surrogateescape") for a in sys.argv[1:]), None]
+libc._NSGetEnviron.restype = ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p))
+# Replaces this process on success; the return value is the error number
+rc = libc.posix_spawn(None, argv[0], None, attr, (ctypes.c_char_p * len(argv))(*argv),
+                      libc._NSGetEnviron().contents)
+libc.strerror.restype = ctypes.c_char_p
+sys.exit("ESPHome: cannot start " + real + ": " + libc.strerror(rc).decode())
+"""
 _CXX_SUFFIXES = (".cpp", ".cc", ".cxx")
 _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".inc")
 _INCLUDE_DIR_OPTIONS = ("-I", "-isystem", "-iquote", "-idirafter")
@@ -81,6 +112,35 @@ def _src_envs(src_dir: Path) -> list:
     return list(found.values()) or [projenv]  # noqa: F821
 
 
+def _driver_output(cxx: str, option: str) -> str:
+    result = subprocess.run([cxx, option], capture_output=True, text=True)  # noqa: PLW1510
+    return result.stdout.strip()
+
+
+def _old_gcc_flags(proj_dir: Path, pio_platform: str, cxx: str):
+    """Extra driver flags for an old GCC, or None where its .gch cannot be
+    used. Linux loads it as is. arm64 macOS loads every program at a random
+    address, so there the driver gets a -B directory whose cc1plus starts
+    the real one without that randomisation."""
+    if pio_platform != _OLD_GCC_PIO_PLATFORM or sys.platform == "linux":
+        return []
+    major = _driver_output(cxx, "-dumpversion").partition(".")[0]
+    if major.isdigit() and int(major) >= _MOVABLE_PCH_GCC:
+        return []
+    if sys.platform != "darwin" or platform.machine() != "arm64":
+        return None
+    # The driver prints the bare name when it has no cc1plus of its own
+    real = _driver_output(cxx, "-print-prog-name=cc1plus")
+    if not os.path.isabs(real):
+        return None
+    wrapper = proj_dir / _CC1_DIR / "cc1plus"
+    wrapper.parent.mkdir(exist_ok=True)
+    real = os.path.normpath(real)
+    wrapper.write_text(_CC1PLUS_WRAPPER.format(python=sys.executable, real=real))
+    wrapper.chmod(0o755)
+    return [f"-B{_CC1_DIR}/"]
+
+
 def _setup_pch() -> None:
     if projenv is None:
         if "nobuild" in COMMAND_LINE_TARGETS:  # noqa: F821
@@ -97,6 +157,11 @@ def _setup_pch() -> None:
     src_envs = _src_envs(src_dir)
 
     cxx = src_envs[0].subst("$CXX")
+    pio_platform = env.PioPlatform()  # noqa: F821
+    extra = _old_gcc_flags(proj_dir, pio_platform.name, cxx)
+    if extra is None:
+        print("ESPHome: this compiler cannot load a precompiled header here; compiling without it")
+        return
     # The header holds the -include entries itself, so the .gch compile must
     # not see them; consumers keep theirs, which the .gch then satisfies
     flags = []
@@ -133,9 +198,8 @@ def _setup_pch() -> None:
     digest.update(flags_id.encode())
     # GCC does not check a .gch against its headers and PlatformIO package
     # paths carry no version
-    platform = env.PioPlatform()  # noqa: F821
-    for package in sorted(platform.packages):
-        digest.update(f"{package}={platform.get_package_version(package)}\0".encode())
+    for package in sorted(pio_platform.packages):
+        digest.update(f"{package}={pio_platform.get_package_version(package)}\0".encode())
     # Libraries carry their version in lib_deps
     digest.update(repr(env.GetProjectOption("lib_deps", [])).encode())  # noqa: F821
     closure = _include_closure(src_dir, headers)
@@ -183,7 +247,7 @@ def _setup_pch() -> None:
         header.write_text(_GUARD_TEXT, encoding="utf-8")
         source.write_text(content, encoding="utf-8")
         result = subprocess.run(  # noqa: PLW1510
-            [cxx, "-x", "c++-header", *flags, "-c", str(source), "-o", str(gch)],
+            [cxx, "-x", "c++-header", *flags, *extra, "-c", str(source), "-o", str(gch)],
             cwd=proj_dir,
             capture_output=True,
             text=True,
@@ -196,7 +260,7 @@ def _setup_pch() -> None:
 
     # Prepended: GCC only loads a .gch for the first -include
     for src_env in src_envs:
-        src_env.Prepend(CXXFLAGS=_CONSUMER_FLAGS)
+        src_env.Prepend(CXXFLAGS=[*extra, *_CONSUMER_FLAGS])
     print(
         "ESPHome: Compiling with a precompiled header "
         "(set ESPHOME_PCH_ENABLE=0 to disable)"

esphome/writer.py

@@ -611,10 +611,13 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False):
                 _LOGGER.info("Deleting %s", idf_path)
                 rmtree(idf_path)
         # The PlatformIO pch files live in the project root
-        from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES
+        from esphome.build_helpers.pch import PCH_ARTIFACT_DIRS, PCH_ARTIFACT_NAMES
 
         for name in PCH_ARTIFACT_NAMES:
             CORE.relative_build_path(name).unlink(missing_ok=True)
+        for name in PCH_ARTIFACT_DIRS:
+            if (pch_dir := CORE.relative_build_path(name)).is_dir():
+                rmtree(pch_dir)
 
     # idedata caches live under the data dir, not the build path; globbed
     # so a future backend suffix cannot drift out of clean-all

tests/unit_tests/build_helpers/test_pch.py

@@ -113,23 +113,12 @@ def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None:
     "platform", ["bk72xx", "esp32", "esp8266", "ln882x", "rp2", "rtl87xx"]
 )
 def test_pch_script_enabled(platform: str, monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(pch.sys, "platform", "linux")
     CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform}
     assert pch.pch_script_enabled()
     monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
     assert not pch.pch_script_enabled()
 
 
-@pytest.mark.parametrize("host", ["darwin", "win32"])
-@pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_LINUX_ONLY_PLATFORMS))
-def test_pch_script_linux_only_platform(
-    platform: str, host: str, monkeypatch: pytest.MonkeyPatch
-) -> None:
-    monkeypatch.setattr(pch.sys, "platform", host)
-    CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform}
-    assert not pch.pch_script_enabled()
-
-
 @pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_EXCLUDED_PLATFORMS))
 def test_pch_script_excluded_platform(platform: str) -> None:
     CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform}

tests/unit_tests/test_platformio_pch_script.py

@@ -6,6 +6,8 @@ from collections.abc import Callable
 import os
 from pathlib import Path
 import stat
+import subprocess
+import sys
 from types import SimpleNamespace
 from unittest.mock import patch
 
@@ -22,6 +24,7 @@ _SCRIPT = Path(toolchain.__file__).parent / "pch.py.script"
 
 
 class _FakePlatform:
+    name = "fake"
     packages = {"framework-x": {}, "toolchain-y": {}}
 
     def get_package_version(self, name: str) -> str | None:
@@ -72,7 +75,13 @@ class _FakeSConsEnv(dict):
 def _fake_cxx(tmp_path: Path, fail: bool = False) -> Path:
     """A compiler stand-in that records its argv and writes the -o target."""
     cxx = tmp_path / "fake-gxx"
+    # As GCC: the program's path when it exists next to the driver, else its
+    # name; the version from FAKE_GCC_VERSION
     body = (
+        'case "$1" in -print-prog-name=*) n=${1#*=};'
+        ' p="$(dirname "$0")/../libexec/gcc/arm-none-eabi/10.3.1/$n";'
+        ' [ -x "$p" ] && echo "$p" || echo "$n"; exit 0;;'
+        ' -dumpversion) echo "${FAKE_GCC_VERSION:-10.3.1}"; exit 0;; esac\n'
         'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n'
     )
     if fail:
@@ -93,6 +102,7 @@ def _run_script(
     name: str = "dev",
     platform_cls: type[_FakePlatform] = _FakePlatform,
     build_files: Callable[[tuple], list] | None = None,
+    cxx: Path | None = None,
 ) -> _FakeSConsEnv:
     proj = tmp_path / name
     src = proj / "src"
@@ -101,7 +111,8 @@ def _run_script(
     (src / "esphome" / "core" / "pch_prefix.h").write_text(
         '#include "esphome/core/defines.h"\n'
     )
-    cxx = _fake_cxx(tmp_path, fail=fail)
+    if cxx is None:
+        cxx = _fake_cxx(tmp_path, fail=fail)
     args = (proj, src, str(cxx), flags or ["-DX=1"], platform_cls)
     # Distinct objects: the -include flags must land on projenv only
     global_env = _FakeSConsEnv(*args)
@@ -137,6 +148,114 @@ def test_pch_script_builds_and_prepends_relative_include(tmp_path: Path) -> None
     assert scons_env.global_env.prepended == []
 
 
+class _LibreTinyPlatform(_FakePlatform):
+    name = "libretiny"
+
+
+def _run_on_host(
+    tmp_path: Path,
+    host: str,
+    machine: str,
+    platform_cls: type[_FakePlatform],
+    cxx: Path | None = None,
+    gcc_version: str = "10.3.1",
+) -> _FakeSConsEnv:
+    with (
+        patch.object(sys, "platform", host),
+        patch("platform.machine", return_value=machine),
+    ):
+        return _run_script(
+            tmp_path,
+            platform_cls=platform_cls,
+            cxx=cxx,
+            env_vars={"FAKE_GCC_VERSION": gcc_version},
+        )
+
+
+def _fake_toolchain(tmp_path: Path) -> tuple[Path, Path]:
+    """A driver in bin/ and a cc1plus that records its argv in libexec/."""
+    toolchain = tmp_path / "toolchain"
+    (toolchain / "bin").mkdir(parents=True)
+    cxx = _fake_cxx(toolchain / "bin")
+    real = toolchain / "libexec" / "gcc" / "arm-none-eabi" / "10.3.1" / "cc1plus"
+    real.parent.mkdir(parents=True)
+    real.write_text('#!/bin/sh\nprintf \'%s\\n\' "$@" > "$0.argv"\n')
+    real.chmod(0o755)
+    return cxx, real
+
+
+def test_pch_script_gcc10_wrapper_on_apple_silicon(tmp_path: Path) -> None:
+    """The LibreTiny .gch compile and the consumers get a -B directory
+    holding a cc1plus that starts the real one."""
+    cxx, real = _fake_toolchain(tmp_path)
+    scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx)
+    wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus"
+    assert wrapper.stat().st_mode & stat.S_IXUSR
+    assert repr(str(real)) in wrapper.read_text(encoding="utf-8")
+    argv = Path(f"{cxx}.argv").read_text(encoding="utf-8").split("\n")
+    assert f"-B{pch.PCH_CC1_DIR}/" in argv
+    assert scons_env.prepended == [f"-B{pch.PCH_CC1_DIR}/", *pch.pch_consumer_flags()]
+
+
+@pytest.mark.skipif(sys.platform != "darwin", reason="the wrapper is macOS only")
+def test_pch_script_gcc10_wrapper_starts_the_real_cc1plus(tmp_path: Path) -> None:
+    cxx, real = _fake_toolchain(tmp_path)
+    _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx)
+    wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus"
+    result = subprocess.run(
+        [str(wrapper), "-quiet", "x.cpp"], capture_output=True, text=True, check=False
+    )
+    assert result.returncode == 0, result.stderr
+    assert Path(f"{real}.argv").read_text(encoding="utf-8") == "-quiet\nx.cpp\n"
+
+
+def test_pch_script_gcc10_without_cc1plus_builds_plainly(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """A driver without a cc1plus of its own gets no wrapper and no header."""
+    scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform)
+    assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists()
+    assert scons_env.prepended == []
+    assert "compiling without it" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    ("host", "machine"), [("darwin", "x86_64"), ("win32", "AMD64")]
+)
+def test_pch_script_gcc10_skipped_elsewhere(
+    tmp_path: Path, host: str, machine: str, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """Where the GCC 10 .gch cannot load, the build runs without it."""
+    scons_env = _run_on_host(tmp_path, host, machine, _LibreTinyPlatform)
+    assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists()
+    assert scons_env.prepended == []
+    assert "compiling without it" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    ("host", "machine", "platform_cls", "gcc_version"),
+    [
+        ("darwin", "arm64", _FakePlatform, "10.3.1"),
+        ("linux", "aarch64", _LibreTinyPlatform, "10.3.1"),
+        # From GCC 12 the .gch loads at any address, so the rule retires itself
+        ("darwin", "arm64", _LibreTinyPlatform, "12.2.0"),
+        ("win32", "AMD64", _LibreTinyPlatform, "14.2.0"),
+    ],
+)
+def test_pch_script_no_wrapper_where_the_gch_loads(
+    tmp_path: Path,
+    host: str,
+    machine: str,
+    platform_cls: type[_FakePlatform],
+    gcc_version: str,
+) -> None:
+    scons_env = _run_on_host(
+        tmp_path, host, machine, platform_cls, gcc_version=gcc_version
+    )
+    assert not (tmp_path / "dev" / pch.PCH_CC1_DIR).exists()
+    assert scons_env.prepended == pch.pch_consumer_flags()
+
+
 def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None:
     """The script cannot import esphome, so its copies are pinned."""
     namespace: dict[str, object] = {
@@ -152,6 +271,7 @@ def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None:
     assert namespace["_CONSUMER_FLAGS"] == pch.pch_consumer_flags()
     assert namespace["_GUARD_TEXT"] == pch.PCH_GUARD_TEXT
     assert namespace["_INCLUDE_RE"].pattern == pch._INCLUDE_RE.pattern
+    assert namespace["_CC1_DIR"] == pch.PCH_CC1_DIR
 
 
 def test_pch_script_compile_failure_stops_the_build(tmp_path: Path) -> None:

tests/unit_tests/test_writer.py

@@ -684,11 +684,17 @@ def test_clean_build_partial_removes_pch_artifacts(
 ) -> None:
     """The PlatformIO pch sidecars live at the project root and must go in
     a partial clean, like the native backend's under .pioenvs."""
-    from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES as names
+    from esphome.build_helpers.pch import (
+        PCH_ARTIFACT_DIRS as dirs,
+        PCH_ARTIFACT_NAMES as names,
+    )
 
     assert "esphome_pch.h.gch" in names
     for name in names:
         (tmp_path / name).write_text("x")
+    for name in dirs:
+        (tmp_path / name).mkdir()
+        (tmp_path / name / "cc1plus").write_text("x")
     mock_core.relative_pioenvs_path.return_value = tmp_path / ".pioenvs"
     mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps"
     mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
@@ -696,7 +702,7 @@ def test_clean_build_partial_removes_pch_artifacts(
 
     clean_build()
 
-    for name in names:
+    for name in (*names, *dirs):
         assert not (tmp_path / name).exists()