Merge branch 'dev' into ccache-pch-old-gate

ccache before 4.7 stores a compile whose included header is less than a
second old without that header, so after the precompiled header changes a
later build reuses objects built against the old one (#21621). Read the
version from the --version probe ESPHome already runs, once per binary.

esphome/build_gen/espidf.py

@@ -7,6 +7,7 @@ import re
 import textwrap
 
 from esphome.build_helpers import pch
+from esphome.build_helpers.ccache import CCACHE_PCH_MIN_VERSION
 from esphome.components.esp32 import (
     KEY_SRC_CMAKE,
     get_esp32_variant,
@@ -515,7 +516,7 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC
 # Where CMake puts the .gch of the src component; ccache reads the checksum
 # next to it in place of the .gch
 _PCH_SUM_PATH = "build/esp-idf/src/CMakeFiles/__idf_src.dir/cmake_pch.hxx.gch.sum"
-# Where the Windows gate records its choice
+# Where the CMake gate records its choice
 _PCH_CHOICE_VAR = "ESPHOME_PCH"
 
 
@@ -531,16 +532,32 @@ def _pch_cmake_block() -> str:
     block = f"""target_precompile_headers(${{COMPONENT_LIB}} PRIVATE
 {headers}
 )"""
-    if not pch.pch_needs_gcc_check():
+    if pch.pch_forced():
         return f"\n# ESPHome precompiled header\n{block}\n"
-    # Before the first configure only CMake knows the compiler version
-    return f"""
-# ESPHome precompiled header, unless GCC bug 14940 keeps it from loading
-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND ({pch.PCH_WINDOWS_CMAKE_OLD_GCC}))
+    # Only CMake knows the compiler version before the first configure, and
+    # the ccache IDF found is the one the build runs
+    gcc_gate = ""
+    if pch.pch_needs_gcc_check():
+        gcc_gate = f"""if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND ({pch.PCH_WINDOWS_CMAKE_OLD_GCC}))
   message(STATUS "ESPHome: GCC ${{CMAKE_CXX_COMPILER_VERSION}} cannot load a precompiled header on Windows; compiling without it")
-  set({_PCH_CHOICE_VAR} OFF CACHE BOOL "ESPHome precompiled header in use" FORCE)
-else()
-  set({_PCH_CHOICE_VAR} ON CACHE BOOL "ESPHome precompiled header in use" FORCE)
+  set(_esphome_pch OFF)
+endif()
+"""
+    min_ccache = ".".join(map(str, CCACHE_PCH_MIN_VERSION))
+    return f"""
+# ESPHome precompiled header, unless a check below rules it out
+set(_esphome_pch ON)
+{gcc_gate}if(CCACHE_ENABLE AND CCACHE_FOUND)
+  execute_process(COMMAND "${{CCACHE_FOUND}}" --version OUTPUT_VARIABLE _esphome_ccache ERROR_QUIET)
+  if(_esphome_ccache MATCHES "ccache version ([0-9.]+)")
+    if(CMAKE_MATCH_1 VERSION_LESS {min_ccache})
+      message(STATUS "ESPHome: ccache ${{CMAKE_MATCH_1}} can reuse stale objects with a precompiled header; compiling without it (upgrade ccache to {min_ccache} or newer, or set ESPHOME_PCH_ENABLE=1 to force)")
+      set(_esphome_pch OFF)
+    endif()
+  endif()
+endif()
+set({_PCH_CHOICE_VAR} ${{_esphome_pch}} CACHE BOOL "ESPHome precompiled header in use" FORCE)
+if(_esphome_pch)
 {textwrap.indent(block, "  ")}
 endif()
 """
@@ -557,10 +574,10 @@ def write_pch_checksum() -> None:
 
     if not pch.pch_enabled():
         return
-    # The gate's choice, cached by configure
-    if pch.pch_needs_gcc_check() and get_cmake_cache_value(_PCH_CHOICE_VAR) != "ON":
-        return
-    pch.log_pch_in_use()
+    # Written whatever the gate chose last time: a build tree without that
+    # choice, or a reconfigure that turns the pch on, must not see a stale sum
+    if pch.pch_forced() or get_cmake_cache_value(_PCH_CHOICE_VAR) == "ON":
+        pch.log_pch_in_use()
     checksum = pch.pch_checksum(
         CORE.relative_src_path(),
         pch.PCH_DEFAULT_HEADERS,

esphome/build_helpers/ccache.py

@@ -3,21 +3,68 @@ resolution, and default ``CCACHE_*`` values."""
 
 from __future__ import annotations
 
+import functools
 import logging
 import os
 from pathlib import Path
+import re
 
-from esphome.framework_helpers import strip_win_long_path_prefix, tool_version_runs
+from esphome.framework_helpers import (
+    strip_win_long_path_prefix,
+    tool_version_probe,
+    tool_version_runs,
+)
 from esphome.helpers import FALSY_ENV_STRINGS, TRUTHY_ENV_STRINGS
 
 _LOGGER = logging.getLogger(__name__)
 
 
+# Before 4.7 ccache stores a pch consumer whose header is newer than one
+# second without that header, so a later build reuses it after the header
+# changed (#21621)
+CCACHE_PCH_MIN_VERSION = (4, 7)
+
+_VERSION_RE = re.compile(r"ccache version (\d+(?:\.\d+)*)")
+
+
+@functools.cache
+def _ccache_probe(ccache: str) -> str | Exception:
+    """``ccache --version``, run once per binary per process."""
+    return tool_version_probe(ccache)
+
+
 def _ccache_runs(ccache: str) -> bool:
     """Return True when the ``ccache`` found on PATH actually runs."""
     return tool_version_runs(
         ccache,
         "Ignoring ccache at %s because it failed to run; compiling without ccache",
+        probe=_ccache_probe,
+    )
+
+
+@functools.cache
+def _ccache_too_old_for_pch(ccache: str) -> bool:
+    output = _ccache_probe(ccache)
+    if not isinstance(output, str) or (found := _VERSION_RE.search(output)) is None:
+        return False
+    if tuple(map(int, found[1].split("."))) >= CCACHE_PCH_MIN_VERSION:
+        return False
+    _LOGGER.info(
+        "ccache %s at %s can reuse stale objects with a precompiled header; "
+        "compiling without it (upgrade ccache to %s or newer, or set "
+        "ESPHOME_PCH_ENABLE=1 to force)",
+        found[1],
+        ccache,
+        ".".join(map(str, CCACHE_PCH_MIN_VERSION)),
+    )
+    return True
+
+
+def ccache_breaks_pch() -> bool:
+    """Whether the ccache this build uses is too old for a precompiled header.
+    ESP-IDF builds check the ccache IDF finds in CMake instead (build_gen.espidf)."""
+    return (ccache := resolve_ccache_path()) is not None and _ccache_too_old_for_pch(
+        ccache
     )
 
 

esphome/build_helpers/pch.py

@@ -12,7 +12,11 @@ import re
 import subprocess
 import sys
 
-from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env
+from esphome.build_helpers.ccache import (
+    ccache_breaks_pch,
+    effective_ccache_basedir,
+    parse_enable_env,
+)
 from esphome.const import PLATFORM_NRF52
 from esphome.helpers import write_file_if_changed
 
@@ -117,9 +121,16 @@ def pch_needs_gcc_check() -> bool:
     return sys.platform == "win32" and parse_enable_env("ESPHOME_PCH_ENABLE") is None
 
 
+def _ccache_rules_out_pch() -> bool:
+    """An old ccache, unless ``ESPHOME_PCH_ENABLE=1`` forces the pch on.
+    ESP-IDF runs the same check in CMake (build_gen.espidf)."""
+    return not pch_forced() and ccache_breaks_pch()
+
+
 def pch_usable(cxx: Sequence[Path | str]) -> bool:
-    """The knob plus the host rule; ``ESPHOME_PCH_ENABLE=1`` skips the rule."""
-    if not pch_enabled():
+    """The knob plus the ccache and host rules; ``ESPHOME_PCH_ENABLE=1``
+    skips the rules."""
+    if not pch_enabled() or _ccache_rules_out_pch():
         return False
     if not pch_needs_gcc_check():
         return True
@@ -162,7 +173,11 @@ def pch_script_enabled() -> bool:
     """Whether this PlatformIO build takes the pch script."""
     from esphome.core import CORE
 
-    return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS
+    return (
+        pch_enabled()
+        and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS
+        and not _ccache_rules_out_pch()
+    )
 
 
 def pch_header_text(include_headers: Iterable[str]) -> str:

esphome/framework_helpers.py

@@ -205,28 +205,42 @@ def run_command(
         return False, None, None
 
 
-def tool_version_runs(binary: str, warning: str) -> bool:
-    """Probe ``binary --version``; on failure warn with ``warning`` % binary.
+def tool_version_probe(binary: str) -> str | Exception:
+    """Run ``binary --version``: its stdout, or the error when it fails to run.
 
     ``shutil.which`` proves existence, not runnability (Windows .bat/.cmd
     shims, stale package-manager shims).
     """
     try:
-        subprocess.run(
+        result = subprocess.run(
             [binary, "--version"],
             check=True,
-            stdout=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
             stderr=subprocess.DEVNULL,
+            text=True,
+            errors="replace",
             timeout=15,
             # Repo-wide convention (posix_spawn fast path)
             close_fds=False,
         )
     except (OSError, subprocess.SubprocessError) as err:
-        # The cause (permission denied, missing DLL, timeout) is the one
-        # detail the user needs to fix it
-        _LOGGER.warning("%s (%s)", warning % binary, err)
-        return False
-    return True
+        return err
+    return result.stdout
+
+
+def tool_version_runs(
+    binary: str,
+    warning: str,
+    probe: Callable[[str], str | Exception] = tool_version_probe,
+) -> bool:
+    """Probe ``binary --version``; on failure warn with ``warning`` % binary.
+    ``probe`` lets a caller supply a cached probe."""
+    if not isinstance(result := probe(binary), Exception):
+        return True
+    # The cause (permission denied, missing DLL, timeout) is the one detail
+    # the user needs to fix it
+    _LOGGER.warning("%s (%s)", warning % binary, result)
+    return False
 
 
 def run_command_ok(*args, **kwargs) -> bool:

tests/unit_tests/build_gen/test_espidf.py

@@ -6,10 +6,11 @@ import json
 import logging
 import os
 from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 
+from esphome.build_helpers import pch
 from esphome.components.esp32 import (
     KEY_COMPONENTS,
     KEY_ESP32,
@@ -686,11 +687,39 @@ def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None
     assert "target_precompile_headers" not in get_component_cmakelists()
 
 
-def test_component_cmakelists_pch_gate_on_windows(
-    windows_gcc_rule: None, monkeypatch: pytest.MonkeyPatch
+def test_component_cmakelists_pch_ccache_gate(
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """The block carries the rule and records its choice; the knob drops
-    the gate."""
+    """With the knob unset, CMake checks the ccache IDF found and records its
+    choice; the knob drops the gate."""
+    from esphome.build_gen.espidf import get_component_cmakelists
+
+    monkeypatch.delenv("ESPHOME_PCH_ENABLE")
+    # Off Windows, so the GCC rule stays out of the block
+    monkeypatch.setattr(pch.sys, "platform", "linux")
+    # Code generation must not run ccache or need an installed toolchain
+    monkeypatch.setattr(pch, "ccache_breaks_pch", Mock(side_effect=AssertionError))
+    content = get_component_cmakelists()
+    assert (
+        "if(CCACHE_ENABLE AND CCACHE_FOUND)\n"
+        '  execute_process(COMMAND "${CCACHE_FOUND}" --version '
+        "OUTPUT_VARIABLE _esphome_ccache ERROR_QUIET)\n"
+        '  if(_esphome_ccache MATCHES "ccache version ([0-9.]+)")\n'
+        "    if(CMAKE_MATCH_1 VERSION_LESS 4.7)\n"
+    ) in content
+    assert (
+        'set(ESPHOME_PCH ${_esphome_pch} CACHE BOOL "ESPHome precompiled header in use" FORCE)\n'
+        "if(_esphome_pch)\n  target_precompile_headers(${COMPONENT_LIB} PRIVATE\n"
+    ) in content
+    assert "CMAKE_CXX_COMPILER_VERSION VERSION_LESS" not in content
+    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1")
+    content = get_component_cmakelists()
+    assert "CCACHE_FOUND" not in content
+    assert "\ntarget_precompile_headers(${COMPONENT_LIB} PRIVATE\n" in content
+
+
+def test_component_cmakelists_pch_gate_on_windows(windows_gcc_rule: None) -> None:
+    """On Windows the GCC rule joins the ccache check."""
     from esphome.build_gen.espidf import get_component_cmakelists
 
     content = get_component_cmakelists()
@@ -699,39 +728,42 @@ def test_component_cmakelists_pch_gate_on_windows(
         "(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.4 OR "
         "(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15 AND "
         "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.3)))\n"
-        "  message(STATUS " in content
-    )
+        "  message(STATUS "
+    ) in content
     assert (
-        '  set(ESPHOME_PCH OFF CACHE BOOL "ESPHome precompiled header in use" FORCE)\nelse()\n'
+        "  set(_esphome_pch OFF)\nendif()\nif(CCACHE_ENABLE AND CCACHE_FOUND)"
         in content
     )
-    assert (
-        '  set(ESPHOME_PCH ON CACHE BOOL "ESPHome precompiled header in use" FORCE)\n  target_precompile_headers(${COMPONENT_LIB} PRIVATE\n'
-        in content
-    )
-    assert "endif()" in content
-    monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1")
-    content = get_component_cmakelists()
-    assert "if(CMAKE_CXX_COMPILER_VERSION" not in content
-    assert "\ntarget_precompile_headers(${COMPONENT_LIB} PRIVATE\n" in content
 
 
-@pytest.mark.parametrize(("choice", "written"), [("OFF", False), ("ON", True)])
-def test_pch_checksum_follows_the_cmake_choice_on_windows(
-    windows_gcc_rule: None, tmp_path: Path, choice: str, written: bool
+@pytest.mark.parametrize(
+    ("choice", "logged"), [("OFF", False), ("ON", True), (None, False)]
+)
+def test_pch_checksum_ignores_the_cmake_choice(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+    choice: str | None,
+    logged: bool,
 ) -> None:
+    """The sum is always refreshed, so a reconfigure that turns the pch on,
+    or a build tree from before the gate, never pairs a new .gch with an old
+    sum; only the log line follows the choice."""
     from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum
 
+    monkeypatch.delenv("ESPHOME_PCH_ENABLE")
     _make_pch_project(tmp_path)
     with (
         patch(
             "esphome.espidf.toolchain.get_cmake_cache_value", return_value=choice
         ) as asked,
         patch.object(CORE, "name", "test"),
+        caplog.at_level("INFO"),
     ):
         write_pch_checksum()
     assert asked.call_args.args == ("ESPHOME_PCH",)
-    assert CORE.relative_build_path(_PCH_SUM_PATH).exists() is written
+    assert CORE.relative_build_path(_PCH_SUM_PATH).exists()
+    assert ("Compiling with a precompiled header" in caplog.text) is logged
 
 
 @pytest.mark.parametrize(

tests/unit_tests/build_helpers/test_ccache.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterator
+from contextlib import contextmanager
 import os
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -195,3 +197,57 @@ def test_ccache_env_includes_pch_settings(tmp_path: Path) -> None:
     env = _pch_ccache_env(tmp_path, {})
     assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
     assert env["CCACHE_PCH_EXTSUM"] == "true"
+
+
+@contextmanager
+def _fake_ccache(
+    banner: str | Exception, env: dict[str, str] | None = None
+) -> Iterator[MagicMock]:
+    """A ccache on PATH whose ``--version`` prints ``banner``."""
+    with (
+        patch.dict(os.environ, env or {}, clear=True),
+        patch("shutil.which", return_value="/usr/bin/ccache"),
+        patch.object(ccache, "tool_version_probe", return_value=banner) as probe,
+    ):
+        yield probe
+
+
+@pytest.mark.parametrize(
+    ("banner", "env", "expected"),
+    [
+        ("ccache version 4.2\n", {}, True),
+        ("ccache version 4.6.3\nFeatures: file-storage\n", {}, True),
+        # An explicit enable skips the runnability probe, not the version
+        ("ccache version 4.6.3\n", {"ESPHOME_CCACHE_ENABLE": "1"}, True),
+        ("ccache version 4.6.3\n", {"ESPHOME_CCACHE_ENABLE": "0"}, False),
+        ("ccache version 4.7\n", {}, False),
+        ("ccache version 4.13.6\n", {}, False),
+        ("something else\n", {}, False),
+    ],
+)
+def test_ccache_breaks_pch(banner: str, env: dict[str, str], expected: bool) -> None:
+    with _fake_ccache(banner, env):
+        assert ccache.ccache_breaks_pch() is expected
+
+
+def test_ccache_breaks_pch_probes_and_logs_once(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """The runnability probe and the version check share one spawn."""
+    with _fake_ccache("ccache version 4.5.1\n") as probe, caplog.at_level("INFO"):
+        assert ccache.resolve_ccache_path() == "/usr/bin/ccache"
+        assert ccache.ccache_breaks_pch()
+        assert ccache.ccache_breaks_pch()
+    probe.assert_called_once()
+    assert caplog.text.count("ccache 4.5.1 at /usr/bin/ccache") == 1
+    assert "upgrade ccache to 4.7 or newer" in caplog.text
+
+
+def test_ccache_breaks_pch_forced_broken_binary_is_quiet(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Forced on, a ccache that fails to run is still used, so the version
+    check must not claim the build compiles without it."""
+    with _fake_ccache(OSError("boom"), {"ESPHOME_CCACHE_ENABLE": "1"}):
+        assert not ccache.ccache_breaks_pch()
+    assert "compiling without ccache" not in caplog.text

tests/unit_tests/build_helpers/test_pch.py

@@ -37,6 +37,21 @@ def test_pch_enabled(value: str | None, expected: bool) -> None:
         assert pch.pch_enabled() is expected
 
 
+@pytest.mark.parametrize(("value", "expected"), [(None, False), ("1", True)])
+def test_old_ccache_rules_out_pch(value: str | None, expected: bool) -> None:
+    """An old ccache turns the pch off unless ESPHOME_PCH_ENABLE=1 forces it."""
+    env = {} if value is None else {"ESPHOME_PCH_ENABLE": value}
+    CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "rp2040"}
+    with (
+        patch.dict(os.environ, env, clear=True),
+        patch.object(pch, "ccache_breaks_pch", return_value=True),
+        patch.object(pch.sys, "platform", "linux"),
+    ):
+        assert pch.pch_enabled()
+        assert pch.pch_usable(("g++",)) is expected
+        assert pch.pch_script_enabled() is expected
+
+
 @pytest.mark.parametrize(
     ("version", "expected"),
     [

tests/unit_tests/conftest.py

@@ -213,6 +213,17 @@ def _default_pch_knobs(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1")
 
 
+@pytest.fixture(autouse=True)
+def _no_ambient_ccache(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The ccache on this machine does not decide the pch, and no cached
+    ccache probe crosses tests."""
+    from esphome.build_helpers import ccache, pch
+
+    monkeypatch.setattr(pch, "ccache_breaks_pch", lambda: False)
+    ccache._ccache_probe.cache_clear()
+    ccache._ccache_too_old_for_pch.cache_clear()
+
+
 @pytest.fixture
 def windows_gcc_rule(monkeypatch: pytest.MonkeyPatch) -> None:
     """Windows host, knob unset: the compiler version decides."""