Bypass the cache when the resolved config enables secure options the snapshot cannot see

.github/workflows/ci.yml

@@ -1230,7 +1230,7 @@ jobs:
               # The real idf.py must find nothing to configure or build in a
               # tree built above; catches drift on ESP-IDF bumps.
               echo "Checking the native ESP-IDF build matches idf.py"
-              python3 script/check_idf_py_equivalence.py
+              python3 script/check_idf_py_equivalence.py --require-cached-bootloader
             fi
           else
             echo "All components in this batch are validate-only -- skipping compile stage."

esphome/__main__.py

@@ -924,14 +924,11 @@ def _check_and_emit_build_info() -> None:
 
 def _get_configured_xtal_freq() -> int | None:
     """Read the configured crystal frequency from the sdkconfig file."""
+    from esphome.espidf import parse_sdkconfig
+
     sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}")
-    if not sdkconfig_path.is_file():
-        return None
-    with suppress(OSError, ValueError):
-        content = sdkconfig_path.read_text()
-        for line in content.splitlines():
-            if line.startswith("CONFIG_XTAL_FREQ="):
-                return int(line.split("=", 1)[1])
+    with suppress(OSError, ValueError, KeyError):
+        return int(parse_sdkconfig(sdkconfig_path)["CONFIG_XTAL_FREQ"])
     return None
 
 

esphome/build_gen/espidf.py

@@ -3,6 +3,7 @@
 import json
 import logging
 from pathlib import Path
+import re
 import textwrap
 
 from esphome.build_helpers import pch
@@ -80,6 +81,81 @@ def _cmake_quote(value: str) -> str:
     return f'"{escaped}"'
 
 
+# CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be
+# cleared at the CMake level (the same state IDF's RAM-app build type uses).
+# The macro is IDF's __build_process_project_includes plus two added lines;
+# esphome.espidf.bootloader falls back to in-tree if an IDF update changes it.
+IDF_BOOTLOADER_OVERRIDE = """\
+# ESPHome cached bootloader switch; see esphome/espidf/bootloader.py.
+if(ESPHOME_USE_CACHED_BOOTLOADER)
+    macro(__build_process_project_includes)
+        idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
+        include(${sdkconfig_cmake})
+        set(CONFIG_APP_BUILD_BOOTLOADER "")
+        # bt's CMakeLists reads the lowercase idf_target that the (now
+        # skipped) bootloader project_include leaks; keep it defined, or
+        # its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /.
+        idf_build_get_property(idf_target IDF_TARGET)
+        idf_build_get_property(build_properties __BUILD_PROPERTIES)
+        foreach(build_property ${build_properties})
+            idf_build_get_property(val ${build_property})
+            set(${build_property} "${val}")
+        endforeach()
+        idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
+        foreach(component_target ${build_component_targets})
+            __component_get_property(dir ${component_target} COMPONENT_DIR)
+            __component_get_property(_name ${component_target} COMPONENT_NAME)
+            set(COMPONENT_NAME ${_name})
+            set(COMPONENT_DIR ${dir})
+            set(COMPONENT_PATH ${dir})
+            if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
+                include(${COMPONENT_DIR}/project_include.cmake)
+            endif()
+        endforeach()
+    endmacro()
+endif()
+"""
+
+# The lines the override adds to IDF's macro; the staleness tripwire in
+# esphome.espidf.bootloader strips them before comparing with the live macro.
+BOOTLOADER_OVERRIDE_ADDED_LINES = (
+    'set(CONFIG_APP_BUILD_BOOTLOADER "")',
+    "idf_build_get_property(idf_target IDF_TARGET)",
+)
+
+_MACRO = re.compile(
+    r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL
+)
+
+
+def _normalized_macro(text: str) -> list[str] | None:
+    """The macro body as comment-free, whitespace-collapsed lines."""
+    if (match := _MACRO.search(text)) is None:
+        return None
+    return [
+        re.sub(r"\s+", " ", line)
+        for raw in match.group(1).splitlines()
+        if (line := raw.split("#", 1)[0].strip())
+    ]
+
+
+_EXPECTED_MACRO = [
+    line
+    for line in _normalized_macro(IDF_BOOTLOADER_OVERRIDE)
+    if line not in BOOTLOADER_OVERRIDE_ADDED_LINES
+]
+
+
+def idf_macro_matches() -> bool:
+    """Whether IDF's macro still matches the copy the override replays."""
+    from esphome.espidf import toolchain
+
+    # pylint: disable-next=protected-access
+    build_cmake = toolchain._get_idf_path() / "tools" / "cmake" / "build.cmake"
+    live = _normalized_macro(build_cmake.read_text(encoding="utf-8"))
+    return live == _EXPECTED_MACRO
+
+
 def get_project_cmakelists(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> str:
@@ -202,6 +278,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src)
 
 include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
+{IDF_BOOTLOADER_OVERRIDE}
 {cpp_standard_options}
 
 {cxx_compile_options}
@@ -214,6 +291,13 @@ include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
 project({CORE.name})
 
+# Record the bootloader customization channels the cache cannot replay;
+# esphome.espidf.bootloader bypasses the cache when any is non-empty.
+idf_build_get_property(esphome_bl_dirs BOOTLOADER_EXTRA_COMPONENT_DIRS)
+idf_build_get_property(esphome_bl_args EXTRA_CMAKE_ARGS)
+file(WRITE ${{CMAKE_BINARY_DIR}}/esphome_bootloader_hooks.txt
+     "${{esphome_bl_dirs}}${{BOOTLOADER_IGNORE_EXTRA_COMPONENT}}${{esphome_bl_args}}")
+
 # Emit per-memory-type JSON size data for ESPHome to read post-build.
 # json2 stays small; raw dumps every symbol (~2s on a large map) and
 # this command runs inside the link edge, blocking everything downstream.

esphome/espidf/__init__.py

@@ -5,7 +5,36 @@ package without the esp32 component package, so nothing here may pull
 in codegen or validation.
 """
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+# Written by the generated CMakeLists after project(); holds the bootloader
+# customization channels the cache cannot replay (see espidf.bootloader).
+BOOTLOADER_HOOKS_FILE = "esphome_bootloader_hooks.txt"
+
 
 def variant_to_idf_target(variant: str) -> str:
     """Map an esp32 variant name (e.g. "ESP32S3") to its ESP-IDF target name."""
     return variant.lower().replace("-", "")
+
+
+def parse_sdkconfig(path: Path) -> dict[str, str]:
+    """Map each CONFIG_X=value line of a text sdkconfig to its raw value.
+
+    Comment lines ("# CONFIG_X is not set") are skipped and quoted values
+    are unquoted.
+    """
+    result = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if not line.startswith("CONFIG_"):
+            continue
+        name, _, value = line.partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] == '"':
+            value = value[1:-1]
+        result[name] = value
+    return result

esphome/espidf/bootloader.py

@@ -0,0 +1,493 @@
+"""Machine-global bootloader cache for the native ESP-IDF toolchain.
+
+Only a bootloader built twice with identical bytes is cached; secure and
+signing builds keep the stock in-tree path.
+"""
+
+# pylint: disable=protected-access
+# This module is part of the espidf toolchain and shares its private helpers.
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import contextlib
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import shutil
+import tempfile
+import time
+
+from esphome.build_helpers.ccache import parse_enable_env
+from esphome.build_helpers.tool_runner import run_build_tool
+from esphome.core import CORE, EsphomeError
+from esphome.espidf import BOOTLOADER_HOOKS_FILE, framework, parse_sdkconfig, toolchain
+from esphome.framework_helpers import _rename_with_retry
+from esphome.helpers import copy_file_if_changed, file_compare, rmtree, write_file
+
+_LOGGER = logging.getLogger(__name__)
+
+BOOTLOADER_CACHE_ENV = "ESPHOME_BOOTLOADER_CACHE"
+# Bump when the build or storage recipe changes; old entries then miss.
+_CACHE_SCHEMA = "1"
+_SECURE_OPTION = re.compile(r"CONFIG_(SECURE_|FLASH_ENCRYPTION)")
+_DISABLED_VALUES = frozenset(("n", "0", ""))
+# Marks a config name the app's sdkconfig does not define.
+_ABSENT = "\x00absent"
+# CMake reads these from the environment on the initial configure.
+_FLAG_ENV_VARS = ("CFLAGS", "CXXFLAGS", "ASMFLAGS", "LDFLAGS")
+_OUTPUTS = ("bootloader.bin", "bootloader.elf", "bootloader.map")
+
+
+def bootloader_cache_enabled() -> bool:
+    """Whether this build may take its bootloader from the cache."""
+    cache = toolchain._cache()
+    if cache.bootloader_enabled is None:
+        cache.bootloader_enabled = _compute_enabled()
+    return cache.bootloader_enabled
+
+
+def _compute_enabled() -> bool:
+    if parse_enable_env(BOOTLOADER_CACHE_ENV) is False:
+        return False
+    if "IDF_PATH" in os.environ:
+        # An unmanaged IDF checkout can change under the cache.
+        return False
+    try:
+        if toolchain._get_framework_source_override():
+            # A custom IDF source can rewire the bootloader subproject in
+            # ways the key and the macro tripwire cannot see.
+            return False
+        # The snapshot holds what was configured; the live sdkconfig is
+        # rewritten by kconfgen with always-on *_SUPPORTED constants.
+        snapshot = CORE.relative_build_path(f"sdkconfig.{CORE.name}.esphomeinternal")
+        if not snapshot.is_file() or _snapshot_blocks_cache(snapshot):
+            return False
+        if CORE.relative_build_path("bootloader_components").exists():
+            # Project-local bootloader overrides are inputs the key can't see.
+            return False
+        if _recorded_bootloader_hooks():
+            # The last configure saw a component customizing the bootloader.
+            _LOGGER.info("Bootloader cache off: build customizes the bootloader")
+            return False
+        resolved = toolchain._load_sdkconfig_json(toolchain._build_dir())
+        if resolved is not None and _resolved_blocks_cache(resolved):
+            # Hand edits to the real sdkconfig survive snapshot preservation.
+            _LOGGER.info("Bootloader cache off: secure options in the tree")
+            return False
+        if not os.access(tools := framework.get_idf_tools_path(), os.W_OK):
+            # A read-only shared prefix would fail the cache on every build.
+            _LOGGER.info("Bootloader cache off: %s is not writable", tools)
+            return False
+        from esphome.build_gen.espidf import idf_macro_matches
+
+        if not idf_macro_matches():
+            _LOGGER.info("IDF changed its bootloader macro; building in-tree")
+            return False
+        return True
+    except (OSError, EsphomeError) as err:
+        _LOGGER.info("Bootloader cache disabled: %s", err)
+        return False
+    except (KeyError, ValueError) as err:
+        # More likely a regression than the environment; still fail safe.
+        _LOGGER.warning("Bootloader cache disabled: %s", err)
+        return False
+
+
+def _recorded_bootloader_hooks() -> str | None:
+    """The bootloader customization channels the last configure recorded.
+
+    Dumped by the generated CMakeLists after project(), so every source
+    counts; None means no configure has written the record yet.
+    """
+    path = toolchain._build_dir() / BOOTLOADER_HOOKS_FILE
+    try:
+        return path.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        return None
+
+
+def _snapshot_blocks_cache(sdkconfig_path: Path) -> bool:
+    """Whether the configured options rule the cache out up front.
+
+    Secure options block because key file contents are inputs the key cannot
+    see; reproducible build off blocks to skip the doomed probe every compile.
+    """
+    config = parse_sdkconfig(sdkconfig_path)
+    if any(
+        _SECURE_OPTION.match(name) and value not in _DISABLED_VALUES
+        for name, value in config.items()
+    ):
+        return True
+    return config.get("CONFIG_APP_REPRODUCIBLE_BUILD", "") in _DISABLED_VALUES
+
+
+def _resolved_blocks_cache(app_config: dict) -> bool:
+    """Secure options in the resolved config; catches hand-edited sdkconfigs.
+
+    Key files are inputs the cache key cannot hash; *_SUPPORTED capability
+    constants are always on and say nothing about what is enabled.
+    """
+    return any(
+        name.startswith(("SECURE_", "FLASH_ENCRYPTION"))
+        and not name.endswith("_SUPPORTED")
+        and bool(value)
+        for name, value in app_config.items()
+    )
+
+
+def tree_uses_cached_bootloader(build_dir: Path) -> bool:
+    """Whether a configured tree was set up for the cached bootloader."""
+    cmakecache = build_dir / "CMakeCache.txt"
+    if not cmakecache.is_file():
+        return False
+    cache = toolchain._parse_cmakecache(cmakecache)
+    return cache.get(toolchain.USE_CACHED_BOOTLOADER_DEFINE) == "1"
+
+
+def _cache_root() -> Path:
+    return (
+        framework.get_idf_tools_path()
+        / "bootloaders"
+        / toolchain._get_core_framework_version()
+        / toolchain._idf_target()
+    )
+
+
+def _load_config_names() -> list[str]:
+    """The known union of config option names the bootloader consumes.
+
+    Each harvest is an immutable file named by its digest; the union over
+    them is monotonic without any cross-process locking.
+    """
+    union: set[str] = set()
+    for path in _cache_root().glob("config_names/*.json"):
+        try:
+            names = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as err:
+            _LOGGER.info("Ignoring unreadable %s: %s", path, err)
+            continue
+        if isinstance(names, list) and all(isinstance(n, str) for n in names):
+            union.update(names)
+        else:
+            _LOGGER.info("Ignoring corrupt %s", path)
+    return sorted(union)
+
+
+def _merge_config_names(names: Iterable[str]) -> list[str]:
+    """Record this build's harvested names and return the resulting union.
+
+    A set that was already recorded is not rewritten; concurrent publishes
+    can only add files, never lose another's discovered names.
+    """
+    merged = sorted(set(names))
+    path = _cache_root() / "config_names" / f"{_compute_key(merged)}.json"
+    if not path.is_file():
+        write_file(path, json.dumps(merged, separators=(",", ":")))
+    # The union guards against our own file being unreadable right back.
+    return sorted(set(merged) | set(_load_config_names()))
+
+
+def _subproject_cmake_args(
+    cmake: str, python: str, sdkconfig: str, idf_path: str, project_dir: str
+) -> list[str]:
+    """The configure argv, mirroring IDF's bootloader ExternalProject args."""
+    from esphome.components.esp32 import idf_version
+    import esphome.config_validation as cv
+
+    # 5.x self-appends the bootloader dir to EXTRA_COMPONENT_DIRS; 6.x
+    # instead forwards IDF_BUILD_V2 (empty: the subproject never opts in).
+    idf6 = idf_version() >= cv.Version(6, 0, 0)
+    args = [
+        cmake,
+        "-G",
+        "Ninja",
+        f"-DSDKCONFIG={sdkconfig}",
+        f"-DIDF_PATH={idf_path}",
+        f"-DIDF_TARGET={toolchain._idf_target()}",
+        "-DPYTHON_DEPS_CHECKED=1",
+        f"-DPYTHON={python}",
+        f"-DEXTRA_COMPONENT_DIRS={'' if idf6 else f'{idf_path}/components/bootloader'}",
+    ]
+    if idf6:
+        args.append("-DIDF_BUILD_V2=")
+    args += [
+        f"-DPROJECT_SOURCE_DIR={project_dir}",
+        "-DIGNORE_EXTRA_COMPONENT=",
+        f"{idf_path}/components/bootloader/subproject",
+    ]
+    return args
+
+
+def _key_payload(names: list[str], app_config: dict, compiler: str, stamp: str) -> dict:
+    """Everything that can influence the built bootloader."""
+    return {
+        "schema": _CACHE_SCHEMA,
+        "idf": toolchain._get_core_framework_version(),
+        "stamp": stamp,
+        "target": toolchain._idf_target(),
+        "compiler": compiler,
+        # Placeholder paths: any change to the invocation shape misses.
+        "args": _subproject_cmake_args(
+            "cmake", "@PYTHON@", "@SDKCONFIG@", "@IDF@", "@PROJECT@"
+        ),
+        "config": {n: app_config.get(n, _ABSENT) for n in names},
+        "env": {n: os.environ.get(n, _ABSENT) for n in _FLAG_ENV_VARS},
+    }
+
+
+def _compute_key(payload: dict | list) -> str:
+    """The digest naming a cache entry or a recorded config name set."""
+    text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(text.encode()).hexdigest()[:16]
+
+
+def _build_standalone(build_dir: Path, verbose: bool) -> int:
+    """Configure and build the bootloader subproject into ``build_dir``."""
+    build_dir.mkdir(parents=True, exist_ok=True)
+    args = _subproject_cmake_args(
+        toolchain._get_idf_tool("cmake"),
+        toolchain._get_idf_tool("python"),
+        str(CORE.relative_build_path(f"sdkconfig.{CORE.name}")),
+        str(toolchain._get_idf_path()),
+        os.path.realpath(CORE.build_path),
+    )
+    filter_lines = None if verbose else toolchain.FILTER_IDF_LINES
+    for name, cmd in (("cmake", args), ("ninja", [toolchain._get_idf_tool("ninja")])):
+        log_path = build_dir / "log" / f"{name}_output.log"
+        rc = run_build_tool(
+            cmd,
+            cwd=build_dir,
+            env=toolchain._tool_env(),
+            filter_lines=filter_lines,
+            log_path=log_path,
+        )
+        if rc != 0:
+            _LOGGER.error("Bootloader %s failed with exit code %d", name, rc)
+            toolchain._print_hints(log_path)
+            return rc
+    return 0
+
+
+def _work_dir(prefix: str) -> Path:
+    """A private work dir in the cache; unique across shared tools prefixes."""
+    root = _cache_root()
+    root.mkdir(parents=True, exist_ok=True)
+    return Path(tempfile.mkdtemp(prefix=prefix, dir=root))
+
+
+def _bin_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _entry_intact(entry: Path) -> bool:
+    """Complete with the published bin bytes; anything else is a miss.
+
+    False only for provable damage; a transient read error raises so a
+    valid shared entry is never mistaken for a corrupt one and deleted.
+    """
+    if not all((entry / name).is_file() for name in _OUTPUTS):
+        return False
+    try:
+        meta = json.loads((entry / "meta.json").read_text(encoding="utf-8"))
+    except (FileNotFoundError, ValueError):
+        return False
+    expected = meta.get("bootloader_bin_sha256") if isinstance(meta, dict) else None
+    return expected == _bin_sha256(entry / "bootloader.bin")
+
+
+def _publish(build_dir: Path, key: str, payload: dict) -> Path:
+    """Move the distilled outputs into the cache; losing a race is fine."""
+    entry = _cache_root() / key
+    if entry.is_dir() and not _entry_intact(entry):
+        # A partial or corrupted entry (manual cleanup) must not win.
+        _LOGGER.debug("Discarding damaged entry %s", entry)
+        _remove_dir(entry)
+    stage = _work_dir(f".stage-{key}-")
+    try:
+        for name in _OUTPUTS:
+            shutil.copy2(build_dir / name, stage / name)
+        # The resolved bootloader config, kept for debugging entries.
+        shutil.copy2(build_dir / "config" / "sdkconfig.json", stage / "sdkconfig.json")
+        meta = {
+            **payload,
+            "bootloader_bin_sha256": _bin_sha256(stage / "bootloader.bin"),
+        }
+        write_file(stage / "meta.json", json.dumps(meta, indent=2, sort_keys=True))
+        stage.chmod(0o755)  # mkdtemp creates 0o700
+        _rename_with_retry(stage, entry)
+    except (OSError, EsphomeError) as err:
+        # Failed to stage, or another process published the same key first.
+        _remove_dir(stage)
+        with contextlib.suppress(OSError):
+            if _entry_intact(entry):
+                _LOGGER.debug("Reusing published %s after: %s", entry, err)
+                return entry
+        raise  # keep the original error even if the intact check breaks
+    return entry
+
+
+def _remove_dir(path: Path) -> None:
+    """Best-effort rmtree with the repo's read-only and retry hardening."""
+    try:
+        rmtree(path)
+    except OSError as err:
+        # Leftover work dirs are cleared by the daily prune; leave a trace.
+        _LOGGER.debug("Could not remove %s: %s", path, err)
+
+
+def _install_into_build(entry: Path) -> Path:
+    """Copy the cached outputs to build/bootloader, where every consumer reads."""
+    dest = toolchain._build_dir() / "bootloader"
+    if (dest / "CMakeCache.txt").is_file():
+        # A leftover in-tree sub-build; its ninja state must not linger.
+        rmtree(dest)
+    if (stamps := dest.parent / "bootloader-prefix").is_dir():
+        # Stale ExternalProject stamps would make a later flip back to
+        # in-tree skip the sub-configure and fail on the missing cache.
+        rmtree(stamps)
+    for name in _OUTPUTS:
+        copy_file_if_changed(entry / name, dest / name)
+    return dest
+
+
+def _bootloader_offset(app_config: dict | None) -> int | None:
+    """The flash offset the bootloader is written to, or None when unknown."""
+    value = (app_config or {}).get("BOOTLOADER_OFFSET_IN_FLASH")
+    return None if value is None else int(value)
+
+
+def _check_bootloader_size(bin_path: Path, offset: int, table_offset: int) -> None:
+    """The check_sizes.py bootloader check, without spawning a process."""
+    size = bin_path.stat().st_size
+    free = table_offset - offset - size
+    if free < 0:
+        raise EsphomeError(
+            f"Bootloader binary size {size:#x} bytes is too large for "
+            f"partition table offset {table_offset:#x} (overflows by "
+            f"{-free:#x} bytes). Increase CONFIG_PARTITION_TABLE_OFFSET."
+        )
+    _LOGGER.info(
+        "Bootloader binary size %#x bytes. %#x bytes (%d%%) free.",
+        size,
+        free,
+        round(100 * free / (table_offset - offset)),
+    )
+
+
+def _prune_stale_dirs() -> None:
+    """Drop day-old work dirs from crashed builds; live ones are younger."""
+    cutoff = time.time() - 86400
+    try:
+        paths = list(_cache_root().glob(".*"))
+    except OSError as err:
+        _LOGGER.debug("Prune skipped: %s", err)
+        return
+    for path in paths:
+        # A racing build may remove its own dir mid-scan; skip just that one.
+        with contextlib.suppress(OSError):
+            if path.is_dir() and path.stat().st_mtime < cutoff:
+                _remove_dir(path)
+
+
+def ensure_cached_bootloader(verbose: bool = False) -> bool:
+    """Put the cached bootloader into build/bootloader, building on a miss.
+
+    False tells the caller to fall back to the in-tree build; a too-large
+    bootloader raises instead, since in-tree would fail the same way.
+    """
+    try:
+        if _recorded_bootloader_hooks() != "":
+            # Non-empty: configure just recorded a bootloader customization.
+            # None: this configure did not write the record at all.
+            _LOGGER.info("Bootloader customized by the build; building in-tree")
+            return False
+        app_config = toolchain._load_sdkconfig_json(toolchain._build_dir())
+        if app_config is not None and _resolved_blocks_cache(app_config):
+            # Configure just resolved a secure option the snapshot cannot see.
+            _LOGGER.info("Secure options in the resolved config; building in-tree")
+            return False
+        offset = _bootloader_offset(app_config)
+        table_offset = (app_config or {}).get("PARTITION_TABLE_OFFSET")
+        compiler = toolchain._resolved_c_compiler()
+        if offset is None or table_offset is None or compiler is None:
+            _LOGGER.info(
+                "Bootloader cache unusable: offset=%s table_offset=%s compiler=%s",
+                offset,
+                table_offset,
+                compiler,
+            )
+            return False
+        dest = _install_cached(app_config, compiler, verbose)
+    except (OSError, ValueError, EsphomeError) as err:
+        _LOGGER.warning("Bootloader cache failed: %s", err)
+        return False
+    if dest is None:
+        return False
+    try:
+        # The size overflow raises out: the in-tree build would fail the same.
+        _check_bootloader_size(dest / "bootloader.bin", offset, int(table_offset))
+    except (OSError, ValueError) as err:
+        _LOGGER.warning("Bootloader cache failed: %s", err)
+        return False
+    return True
+
+
+def _install_cached(app_config: dict, compiler: str, verbose: bool) -> Path | None:
+    """The installed build/bootloader dir, building and publishing on a miss;
+    None means fall back to the in-tree build."""
+    if not (stamp := framework.read_idf_version_txt(toolchain._get_idf_path())):
+        _LOGGER.info("Framework version.txt missing; cannot key the bootloader")
+        return None
+    if names := _load_config_names():
+        entry = _cache_root() / _compute_key(
+            _key_payload(names, app_config, compiler, stamp)
+        )
+        if _entry_intact(entry):
+            return _install_into_build(entry)
+    with contextlib.ExitStack() as cleanup:
+        tmp = _work_dir(".build-")
+        cleanup.callback(_remove_dir, tmp)
+        if _build_standalone(tmp, verbose) != 0:
+            return None
+        if (built_config := toolchain._load_sdkconfig_json(tmp)) is None:
+            # After a successful build this most likely means an IDF change.
+            _LOGGER.warning(
+                "Bootloader build produced no %s", tmp / "config" / "sdkconfig.json"
+            )
+            return None
+        # Only a build byte identical to a second one in a different dir may
+        # be cached; any nondeterminism fails here instead of serving stale.
+        probe = _work_dir(".build-")
+        cleanup.callback(_remove_dir, probe)
+        if _build_standalone(probe, verbose) != 0:
+            return None
+        if not file_compare(tmp / "bootloader.bin", probe / "bootloader.bin"):
+            _LOGGER.info("Bootloader build is not byte reproducible; not caching")
+            return None
+        names = _merge_config_names(built_config)
+        payload = _key_payload(names, app_config, compiler, stamp)
+        key = _compute_key(payload)
+        entry = _publish(tmp, key, payload)
+        _LOGGER.info("Cached bootloader %s for later builds", key)
+    _prune_stale_dirs()
+    return _install_into_build(entry)
+
+
+def inject_bootloader_flash_file(flash_data: dict, build_dir: Path) -> bool:
+    """Add the cached bootloader to a flasher_args ``flash_files`` map.
+
+    Mode gated: IDF omits the entry on purpose for some secure-boot builds.
+    False means a cached-mode tree is missing pieces; the image would not boot.
+    """
+    if not tree_uses_cached_bootloader(build_dir):
+        return True
+    offset = _bootloader_offset(toolchain._load_sdkconfig_json(build_dir))
+    if offset is None or not (build_dir / "bootloader" / "bootloader.bin").is_file():
+        _LOGGER.error("Cached bootloader missing; factory image has no bootloader")
+        return False
+    flash_data.setdefault("flash_files", {})[hex(offset)] = "bootloader/bootloader.bin"
+    return True

esphome/espidf/framework.py

@@ -481,6 +481,17 @@ def _clone_idf_with_submodules(
         )
 
 
+def read_idf_version_txt(framework_path: Path) -> str:
+    """The installed framework's version.txt stamp, or empty."""
+    try:
+        return (framework_path / "version.txt").read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        return ""
+    except OSError as err:
+        _LOGGER.info("Cannot read %s: %s", framework_path / "version.txt", err)
+        return ""
+
+
 def _write_idf_version_txt(framework_path: Path, version: str) -> None:
     """Write <framework_path>/version.txt if missing.
 

esphome/espidf/toolchain.py

@@ -25,11 +25,14 @@ from esphome.core import CORE, EsphomeError
 from esphome.espidf import variant_to_idf_target
 from esphome.espidf.framework import check_esp_idf_install, get_framework_env
 from esphome.espidf.size_summary import print_summary
-from esphome.helpers import add_git_ceiling_directory, write_file
+from esphome.helpers import add_git_ceiling_directory, read_json_file, write_file
 
 _LOGGER = logging.getLogger(__name__)
 
 DOMAIN = "espidf_toolchain"
+# The -D that switches the generated CMakeLists between the cached and the
+# in-tree bootloader; also read back from CMakeCache.txt to identify a tree.
+USE_CACHED_BOOTLOADER_DEFINE = "ESPHOME_USE_CACHED_BOOTLOADER"
 
 
 @dataclass
@@ -37,6 +40,7 @@ class _CacheData:
     paths: dict[str, tuple] = field(default_factory=dict)
     env: dict[str, dict[str, str]] = field(default_factory=dict)
     cmake_output: dict[Path, str] = field(default_factory=dict)
+    bootloader_enabled: bool | None = None
     cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict)
 
 
@@ -247,6 +251,25 @@ def _build_dir() -> Path:
     return Path(os.path.realpath(CORE.build_path)) / "build"
 
 
+def _idf_target() -> str:
+    return variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
+
+
+def _load_sdkconfig_json(build_dir: Path) -> dict | None:
+    """A build's generated config/sdkconfig.json, or None."""
+    config = read_json_file(build_dir / "config" / "sdkconfig.json")
+    return config if isinstance(config, dict) else None
+
+
+def _resolved_c_compiler() -> str | None:
+    """The configured build's resolved C compiler, or None."""
+    description = read_json_file(_build_dir() / "project_description.json")
+    if not isinstance(description, dict):
+        return None
+    compiler = description.get("c_compiler")
+    return os.path.realpath(compiler) if compiler else None
+
+
 def _cache_entries() -> dict[str, str]:
     """The ``-D`` entries idf.py passes to cmake, in idf.py's order."""
     entries = {}
@@ -282,6 +305,13 @@ def _cache_entries_changed() -> bool:
     return any(cache.get(k) != v for k, v in _configure_defines().items())
 
 
+def _use_cached_bootloader() -> bool:
+    """Whether this run takes the bootloader from the machine-global cache."""
+    from esphome.espidf import bootloader
+
+    return bootloader.bootloader_cache_enabled()
+
+
 def _configure_defines() -> dict[str, str]:
     """Every ``-D`` idf.py passes to cmake, in its order."""
     return {
@@ -289,6 +319,9 @@ def _configure_defines() -> dict[str, str]:
         "PYTHON": _get_idf_tool("python"),
         "ESP_PLATFORM": "1",
         **_cache_entries(),
+        # ESPHome's own switch; idf.py never passes it and cmake keeps the
+        # cached value, so idf.py runs against the tree stay in the same mode.
+        USE_CACHED_BOOTLOADER_DEFINE: "1" if _use_cached_bootloader() else "0",
     }
 
 
@@ -433,7 +466,7 @@ def _builtin_component_cache_path() -> Path | None:
     """
     if "IDF_PATH" in os.environ:
         return None
-    target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
+    target = _idf_target()
     excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "")
     excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12]
     return (
@@ -675,6 +708,16 @@ def run_compile(config, verbose: bool) -> int:
     else:
         _LOGGER.info("Build configuration is up to date")
 
+    if _use_cached_bootloader():
+        from esphome.espidf import bootloader
+
+        if not bootloader.ensure_cached_bootloader(verbose):
+            # The stock in-tree build still works; flip this run over to it.
+            _LOGGER.warning("Cached bootloader unavailable; building it in-tree")
+            _cache().bootloader_enabled = False
+            if (rc := run_reconfigure(verbose)) != 0:
+                return rc
+
     # In testing mode, generate the linker script first, patch DRAM/IRAM sizes,
     # then build. memory.ld is regenerated by ninja during the build phase,
     # so we must patch after it's generated but before linking (same timing
@@ -809,6 +852,11 @@ def create_factory_bin() -> bool:
         _LOGGER.error("Failed to read flasher_args.json: %s", e)
         return False
 
+    from esphome.espidf import bootloader
+
+    if not bootloader.inject_bootloader_flash_file(flash_data, build_dir):
+        return False
+
     # Get flash size from config
     flash_size = CORE.data[KEY_ESP32][KEY_FLASH_SIZE]
 

esphome/helpers.py

@@ -10,7 +10,7 @@ import platform
 import re
 import stat
 import sys
-from typing import TYPE_CHECKING, TextIO
+from typing import TYPE_CHECKING, Any, TextIO
 
 from esphome.const import __version__ as ESPHOME_VERSION
 
@@ -513,6 +513,16 @@ def walk_files(path: Path):
             yield Path(root) / name
 
 
+def read_json_file(path: Path) -> Any:
+    """The parsed JSON file, or None when missing or unreadable."""
+    import json
+
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
 def read_file(path: Path) -> str:
     try:
         return path.read_text(encoding="utf-8")

script/check_idf_py_equivalence.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 import argparse
 import hashlib
 import json
+import os
 from pathlib import Path
 import re
 import subprocess
@@ -31,12 +32,35 @@ WATCHED = (
     "build/config/sdkconfig.h",
     "build/bootloader/bootloader.bin",
 )
-# Ninja logs whose outputs mean real work when their recorded mtime changes.
-# The top level re-logs the bootloader step's byproducts on every build, so
-# the bootloader is judged by its own sub-build log instead.
+# Ninja logs whose outputs mean real work when their recorded mtime changes;
+# the bootloader is judged by its own sub-build log when one exists.
 TOP_NINJA_LOG = "build/.ninja_log"
-NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log")
+
+
+def _ninja_logs(build_path: Path) -> list[str]:
+    """The mode comes from the configured tree, so a missing sub-build log
+    stays an error in the mode that requires one."""
+    from esphome.espidf import bootloader
+
+    logs = [TOP_NINJA_LOG]
+    if not bootloader.tree_uses_cached_bootloader(build_path / "build"):
+        logs.append("build/bootloader/.ninja_log")
+    return logs
+
+
 BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/")
+MACRO_CHANGED = (
+    "IDF changed __build_process_project_includes; update "
+    "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
+)
+BOOTLOADER_DIFFERS = (
+    "cached bootloader is not byte identical to the in-tree build; update "
+    "_subproject_cmake_args in esphome/espidf/bootloader.py"
+)
+BOOTLOADER_NOT_CACHED = (
+    "tree is not in cached bootloader mode, so parity was never compared; "
+    "the cache fell back or was disabled for this build"
+)
 WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld")
 DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*"
 
@@ -54,10 +78,10 @@ def _snapshot(build_path: Path, name: str) -> dict[str, str | None]:
     return {f: _digest(build_path / f) for f in watched(name)}
 
 
-def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
+def _ninja_mtimes(build_path: Path, logs: list[str]) -> dict[tuple[str, str], str]:
     """(log, output) -> recorded mtime; compaction-safe, unlike a line count."""
     mtimes = {}
-    for name in NINJA_LOGS:
+    for name in logs:
         log = build_path / name
         lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else []
         for fields in (line.split("\t") for line in lines if not line.startswith("#")):
@@ -68,10 +92,12 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
     return mtimes
 
 
-def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]:
+def _log_problems(
+    build_path: Path, mtimes: dict[tuple[str, str], str], logs: list[str]
+) -> list[str]:
     """A missing or unparsable ninja log would otherwise compare as unchanged."""
     problems = []
-    for log in NINJA_LOGS:
+    for log in logs:
         if not (build_path / log).is_file():
             problems.append(f"missing {log}")
         elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes):
@@ -97,15 +123,18 @@ def _setup_core(build_path: Path, description: dict) -> tuple[str, str]:
     return name, version
 
 
-def check(build_path: Path) -> list[str]:
+def check(build_path: Path, require_cached: bool = False) -> list[str]:
     """Return the problems found in one build tree."""
     # pylint: disable=protected-access
+    from esphome.build_gen.espidf import idf_macro_matches
     from esphome.espidf import toolchain
 
     description = json.loads(
         (build_path / "build" / "project_description.json").read_text(encoding="utf-8")
     )
     name, version = _setup_core(build_path, description)
+    if not idf_macro_matches():
+        return [MACRO_CHANGED]
     env = toolchain._get_idf_env(version)
     python = toolchain._get_idf_tool("python")
     idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py"
@@ -119,10 +148,11 @@ def check(build_path: Path) -> list[str]:
     if (rc := toolchain._run_ninja("all", verbose=False, jobs=None)) != 0:
         return [f"ESPHome's ninja build failed with exit code {rc}"]
     before = _snapshot(build_path, name)
-    mtimes_before = _ninja_mtimes(build_path)
+    logs = _ninja_logs(build_path)
+    mtimes_before = _ninja_mtimes(build_path, logs)
     # A moved or renamed output would otherwise compare as "unchanged".
     problems = [f"missing {f}" for f, digest in before.items() if digest is None]
-    if problems := problems + _log_problems(build_path, mtimes_before):
+    if problems := problems + _log_problems(build_path, mtimes_before, logs):
         return problems
     for action in ("reconfigure", "build"):
         result = subprocess.run(
@@ -136,9 +166,9 @@ def check(build_path: Path) -> list[str]:
         if result.returncode != 0:
             return [f"idf.py {action} failed:\n{result.stdout}{result.stderr}"]
     after = _snapshot(build_path, name)
-    mtimes_after = _ninja_mtimes(build_path)
+    mtimes_after = _ninja_mtimes(build_path, logs)
     problems = [f"idf.py changed {f}" for f in before if before[f] != after[f]]
-    problems += _log_problems(build_path, mtimes_after)
+    problems += _log_problems(build_path, mtimes_after, logs)
     for key in sorted(mtimes_before.keys() | mtimes_after.keys()):
         log, out = key
         if not out.endswith(WORK_SUFFIXES):
@@ -147,7 +177,46 @@ def check(build_path: Path) -> list[str]:
             problems.append(f"idf.py dropped {out} from {log}")
         elif mtimes_before.get(key) != mtimes_after[key]:
             problems.append(f"idf.py rebuilt {out}")
-    return problems
+    if problems:
+        return problems
+    return _bootloader_parity(build_path, require_cached)
+
+
+def _bootloader_parity(build_path: Path, require_cached: bool) -> list[str]:
+    """A cached bootloader must match the in-tree build of the same tree,
+    so an IDF-side change to the ExternalProject args fails CI."""
+    # pylint: disable=protected-access
+    from esphome.espidf import bootloader, toolchain
+
+    if not bootloader.tree_uses_cached_bootloader(build_path / "build"):
+        if require_cached:
+            return [BOOTLOADER_NOT_CACHED]
+        print("note: tree not in cached mode; bootloader parity not exercised")
+        return []
+    from esphome.helpers import rmtree
+
+    bin_path = build_path / "build" / "bootloader" / "bootloader.bin"
+    cached = bin_path.read_bytes()
+    # Rebuild from nothing: a warm sub-build never regenerates a deleted
+    # byproduct, and the comparison must not see leftovers of any kind.
+    rmtree(bin_path.parent)
+    if (stamps := build_path / "build" / "bootloader-prefix").is_dir():
+        rmtree(stamps)
+    os.environ[bootloader.BOOTLOADER_CACHE_ENV] = "0"
+    toolchain._cache().bootloader_enabled = None
+    try:
+        if toolchain.run_reconfigure() != 0:
+            return ["in-tree bootloader reconfigure failed"]
+        if toolchain._run_ninja("all", verbose=False, jobs=None) != 0:
+            return ["in-tree bootloader rebuild failed"]
+        if not bin_path.is_file():
+            return ["in-tree rebuild produced no bootloader"]
+        if bin_path.read_bytes() != cached:
+            return [BOOTLOADER_DIFFERS]
+        return []
+    finally:
+        os.environ.pop(bootloader.BOOTLOADER_CACHE_ENV, None)
+        toolchain._cache().bootloader_enabled = None
 
 
 def main() -> int:
@@ -158,6 +227,13 @@ def main() -> int:
         type=Path,
         help=f"ESPHome build dirs (default: the first native ESP-IDF tree in {DEFAULT_GLOB})",
     )
+    parser.add_argument(
+        "--require-cached-bootloader",
+        action="store_true",
+        help="fail a tree that is not in cached bootloader mode instead of "
+        "skipping the parity comparison (CI: a cache that always falls back "
+        "must not pass silently)",
+    )
     args = parser.parse_args()
     paths = args.build_paths or sorted(REPO_ROOT.glob(DEFAULT_GLOB))
     # Not resolved: SDKCONFIG must be spelled as the build spelled it.
@@ -180,7 +256,7 @@ def main() -> int:
 
     failed = False
     for tree in trees:
-        problems = check(tree)
+        problems = check(tree, args.require_cached_bootloader)
         print(f"{tree}: {'OK' if not problems else 'DIFFERS'}")
         for problem in problems:
             print(f"  {problem}")

tests/script/test_check_idf_py_equivalence.py

@@ -13,6 +13,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script"))
 
 import check_idf_py_equivalence as guard  # noqa: E402
 
+from esphome.build_gen import espidf as build_gen  # noqa: E402
 from esphome.core import CORE  # noqa: E402
 from esphome.espidf import toolchain  # noqa: E402
 
@@ -24,12 +25,23 @@ def _reset_core() -> Iterator[None]:
     CORE.reset()
 
 
-def _make_tree(tmp_path: Path) -> Path:
+BOOTLOADER_LOG = "build/bootloader/.ninja_log"
+ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG)
+
+
+def _make_tree(tmp_path: Path, cached_bootloader: bool = False) -> Path:
+    """A fake build tree; stock/bypass shape by default, or the cached
+    shape (define set to 1, bin present, no sub-build)."""
     tree = tmp_path / "config" / ".esphome" / "build" / "dev"
     build = tree / "build"
-    for name in (*guard.watched("dev"), *guard.NINJA_LOGS):
+    logs = (guard.TOP_NINJA_LOG,) if cached_bootloader else ALL_LOGS
+    for name in (*guard.watched("dev"), *logs):
         (tree / name).parent.mkdir(parents=True, exist_ok=True)
         (tree / name).write_bytes(b"x")
+    define = "1" if cached_bootloader else "0"
+    (build / "CMakeCache.txt").write_text(
+        f"{toolchain.USE_CACHED_BOOTLOADER_DEFINE}:UNINITIALIZED={define}\n"
+    )
     (build / "project_description.json").write_text(
         json.dumps(
             {
@@ -50,11 +62,26 @@ def _make_tree(tmp_path: Path) -> Path:
     return tree
 
 
+def _rebuilding_ninja(tree: Path, rc: int) -> Callable[..., int]:
+    """A ninja stand-in that recreates the bin the parity check deleted."""
+
+    def run(target: str, **kwargs: object) -> int:
+        bin_path = tree / "build" / "bootloader" / "bootloader.bin"
+        if not bin_path.exists():
+            bin_path.parent.mkdir(parents=True, exist_ok=True)
+            bin_path.write_bytes(b"x")
+        return rc
+
+    return run
+
+
 def _run_check(
     tree: Path,
     side_effect: Callable[[list[str]], None] = lambda cmd: None,
     rc: int = 0,
     esphome_rcs: tuple[int, int] = (0, 0),
+    macro_matches: bool = True,
+    require_cached: bool = False,
 ) -> tuple[list[str], list[list[str]]]:
     """Run check() with idf.py replaced by ``side_effect``; return problems, calls."""
     calls: list[list[str]] = []
@@ -69,16 +96,23 @@ def _run_check(
         patch.object(toolchain, "_get_idf_tool", return_value="/py"),
         patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
         patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]),
-        patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]),
+        patch.object(
+            toolchain, "_run_ninja", side_effect=_rebuilding_ninja(tree, esphome_rcs[1])
+        ),
+        patch.object(build_gen, "idf_macro_matches", return_value=macro_matches),
         patch.object(guard.subprocess, "run", side_effect=run),
     ):
-        return guard.check(tree), calls
+        return guard.check(tree, require_cached), calls
 
 
-def test_check_passes_when_idf_py_changes_nothing(tmp_path: Path) -> None:
+def test_check_passes_when_idf_py_changes_nothing(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
     tree = _make_tree(tmp_path)
     problems, calls = _run_check(tree)
     assert problems == []
+    # a stock tree says so instead of passing parity vacuously
+    assert "parity not exercised" in capsys.readouterr().out
     sdkconfig = f"SDKCONFIG={tree / 'sdkconfig.dev'}"
     assert calls == [
         ["/py", str(Path("/idf/tools/idf.py")), "-D", sdkconfig, "reconfigure"],
@@ -86,6 +120,12 @@ def test_check_passes_when_idf_py_changes_nothing(tmp_path: Path) -> None:
     ]
 
 
+def test_check_requires_cached_mode_when_asked(tmp_path: Path) -> None:
+    """CI must fail, not skip, when the cache never engaged (--require flag)."""
+    problems, _ = _run_check(_make_tree(tmp_path), require_cached=True)
+    assert problems == [guard.BOOTLOADER_NOT_CACHED]
+
+
 def test_check_reports_changed_files_and_rebuilt_outputs(tmp_path: Path) -> None:
     tree = _make_tree(tmp_path)
     build = tree / "build"
@@ -150,9 +190,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None:
     assert len(calls) == 1
 
 
-@pytest.mark.parametrize(
-    "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS]
-)
+@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS])
 def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None:
     """A moved or renamed output must not compare as unchanged."""
     tree = _make_tree(tmp_path)
@@ -179,7 +217,7 @@ def test_check_stops_when_the_esphome_baseline_fails(
     assert calls == []
 
 
-@pytest.mark.parametrize("log", guard.NINJA_LOGS)
+@pytest.mark.parametrize("log", ALL_LOGS)
 def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None:
     """A log format change must not leave the rebuild check with nothing to compare."""
     tree = _make_tree(tmp_path)
@@ -205,6 +243,104 @@ def test_main_rejects_a_path_that_is_not_a_tree(
     mock_check.assert_not_called()
 
 
+def test_check_accepts_a_cached_bootloader_tree(tmp_path: Path) -> None:
+    """No bootloader sub-build is the cached shape, not a missing input."""
+    tree = _make_tree(tmp_path, cached_bootloader=True)
+    problems, calls = _run_check(tree)
+    assert problems == []
+    assert len(calls) == 2
+
+
+def test_check_fails_when_the_cached_bootloader_differs_from_in_tree(
+    tmp_path: Path,
+) -> None:
+    """The parity rebuild catches IDF-side drift in the subproject args."""
+    tree = _make_tree(tmp_path, cached_bootloader=True)
+    bin_path = tree / "build" / "bootloader" / "bootloader.bin"
+    calls = {"n": 0}
+
+    def ninja(target: str, **kwargs: object) -> int:
+        calls["n"] += 1
+        if calls["n"] == 2:  # the parity rebuild
+            bin_path.parent.mkdir(parents=True, exist_ok=True)
+            bin_path.write_bytes(b"different")
+        return 0
+
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "_run_ninja", side_effect=ninja),
+        patch.object(build_gen, "idf_macro_matches", return_value=True),
+        patch.object(
+            guard.subprocess,
+            "run",
+            return_value=subprocess.CompletedProcess([], 0, "", ""),
+        ),
+    ):
+        problems = guard.check(tree)
+    assert problems == [guard.BOOTLOADER_DIFFERS]
+
+
+def test_check_reports_a_rebuild_that_produced_no_bootloader(
+    tmp_path: Path,
+) -> None:
+    """A rebuild that leaves no bin must fail, not compare the old bytes."""
+    tree = _make_tree(tmp_path, cached_bootloader=True)
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "_run_ninja", return_value=0),
+        patch.object(build_gen, "idf_macro_matches", return_value=True),
+        patch.object(
+            guard.subprocess,
+            "run",
+            return_value=subprocess.CompletedProcess([], 0, "", ""),
+        ),
+    ):
+        problems = guard.check(tree)
+    assert problems == ["in-tree rebuild produced no bootloader"]
+
+
+def test_check_reports_a_failed_parity_rebuild(tmp_path: Path) -> None:
+    tree = _make_tree(tmp_path, cached_bootloader=True)
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(toolchain, "run_reconfigure", side_effect=[0, 3]),
+        patch.object(toolchain, "_run_ninja", return_value=0),
+        patch.object(build_gen, "idf_macro_matches", return_value=True),
+        patch.object(
+            guard.subprocess,
+            "run",
+            return_value=subprocess.CompletedProcess([], 0, "", ""),
+        ),
+    ):
+        problems = guard.check(tree)
+    assert problems == ["in-tree bootloader reconfigure failed"]
+
+
+def test_check_requires_the_sub_log_on_a_stock_tree(tmp_path: Path) -> None:
+    """The mode comes from the define, so a vanished sub-build stays an error."""
+    tree = _make_tree(tmp_path)
+    (tree / BOOTLOADER_LOG).unlink()
+    problems, calls = _run_check(tree)
+    assert problems == [f"missing {BOOTLOADER_LOG}"]
+    assert calls == []
+
+
+def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None:
+    """An IDF bump that rewrites the overridden macro must fail CI."""
+    tree = _make_tree(tmp_path)
+    problems, calls = _run_check(tree, macro_matches=False)
+    assert problems == [guard.MACRO_CHANGED]
+    assert calls == []
+
+
 def test_main_without_build_trees(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:
@@ -245,4 +381,4 @@ def test_main_checks_only_the_first_found_tree(tmp_path: Path) -> None:
         patch.object(guard, "check", return_value=[]) as mock_check,
     ):
         assert guard.main() == 0
-    mock_check.assert_called_once_with(first)
+    mock_check.assert_called_once_with(first, False)

tests/unit_tests/build_gen/test_espidf.py

@@ -175,6 +175,20 @@ def test_get_project_cmakelists_size_command_uses_json2() -> None:
     assert "--format=json2" in content
 
 
+def test_get_project_cmakelists_records_bootloader_hooks() -> None:
+    """The post-project() dump feeds espidf.bootloader's bypass check."""
+    from esphome.espidf import BOOTLOADER_HOOKS_FILE
+
+    content = _render()
+    assert f"file(WRITE ${{CMAKE_BINARY_DIR}}/{BOOTLOADER_HOOKS_FILE}" in content
+    for source in (
+        "esphome_bl_dirs BOOTLOADER_EXTRA_COMPONENT_DIRS",
+        "${BOOTLOADER_IGNORE_EXTRA_COMPONENT}",
+        "esphome_bl_args EXTRA_CMAKE_ARGS",
+    ):
+        assert source in content
+
+
 def test_get_project_cmakelists_uses_supplied_builtin_components() -> None:
     """A cached list replaces project_description.json and is still filtered
     by EXCLUDE_COMPONENTS."""
@@ -623,3 +637,81 @@ def test_pch_checksum_disabled_writes_nothing(
     _make_pch_project(tmp_path)
     write_pch_checksum()
     assert not CORE.relative_build_path(_PCH_SUM_PATH).exists()
+
+
+# The macro body as shipped in tools/cmake/build.cmake; byte identical in
+# IDF 5.5.5 and 6.1.0, so one fixture covers both supported versions.
+IDF_BUILD_CMAKE = """\
+some_other_cmake()
+
+macro(__build_process_project_includes)
+    # Include the sdkconfig cmake file, since the following operations require
+    # knowledge of config values.
+    idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
+    include(${sdkconfig_cmake})
+
+    # Make each build property available as a read-only variable
+    idf_build_get_property(build_properties __BUILD_PROPERTIES)
+    foreach(build_property ${build_properties})
+        idf_build_get_property(val ${build_property})
+        set(${build_property} "${val}")
+    endforeach()
+
+    idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
+
+    # Include each component's project_include.cmake
+    foreach(component_target ${build_component_targets})
+        __component_get_property(dir ${component_target} COMPONENT_DIR)
+        __component_get_property(_name ${component_target} COMPONENT_NAME)
+        set(COMPONENT_NAME ${_name})
+        set(COMPONENT_DIR ${dir})
+        set(COMPONENT_PATH ${dir})  # this is deprecated, users are encouraged to use COMPONENT_DIR;
+                                    # retained for compatibility
+        if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
+            include(${COMPONENT_DIR}/project_include.cmake)
+        endif()
+    endforeach()
+endmacro()
+"""
+
+
+def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path:
+    idf = tmp_path / "idf"
+    (idf / "tools" / "cmake").mkdir(parents=True)
+    (idf / "tools" / "cmake" / "build.cmake").write_text(text)
+    return idf
+
+
+def test_normalized_macro_strips_comments_and_whitespace() -> None:
+    from esphome.build_gen.espidf import _normalized_macro
+
+    text = "macro(__build_process_project_includes)\n  a( b )  # tail\n\n  # only\n  c(d)\nendmacro()"
+    assert _normalized_macro(text) == ["a( b )", "c(d)"]
+
+
+def test_normalized_macro_none_without_macro() -> None:
+    from esphome.build_gen.espidf import _normalized_macro
+
+    assert _normalized_macro("nothing here") is None
+
+
+def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None:
+    """The override's embedded copy must equal what build.cmake ships."""
+    from esphome.build_gen.espidf import idf_macro_matches
+    from esphome.espidf import toolchain
+
+    idf = _write_idf_build_cmake(tmp_path)
+    with patch.object(toolchain, "_get_idf_path", return_value=idf):
+        assert idf_macro_matches() is True
+
+
+def test_idf_macro_mismatch_detected(tmp_path: Path) -> None:
+    from esphome.build_gen.espidf import idf_macro_matches
+    from esphome.espidf import toolchain
+
+    changed = IDF_BUILD_CMAKE.replace(
+        "include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)"
+    )
+    idf = _write_idf_build_cmake(tmp_path, changed)
+    with patch.object(toolchain, "_get_idf_path", return_value=idf):
+        assert idf_macro_matches() is False

tests/unit_tests/test_espidf_framework.py

@@ -2003,3 +2003,19 @@ def test_check_windows_path_length_long_path_warns(
     assert "long path support" in message
     # The install is global now; the remedy is the prefix env, not moving the project.
     assert "ESPHOME_ESP_IDF_PREFIX" in message
+
+
+def test_read_idf_version_txt(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
+    from esphome.espidf.framework import read_idf_version_txt
+
+    with caplog.at_level("INFO"):
+        assert read_idf_version_txt(tmp_path) == ""
+    assert "Cannot read" not in caplog.text  # missing is normal
+    (tmp_path / "version.txt").write_text("v5.5.5\n")
+    assert read_idf_version_txt(tmp_path) == "v5.5.5"
+    with (
+        patch.object(Path, "read_text", side_effect=PermissionError("denied")),
+        caplog.at_level("INFO"),
+    ):
+        assert read_idf_version_txt(tmp_path) == ""
+    assert "Cannot read" in caplog.text  # unreadable is worth a line

tests/unit_tests/test_espidf_toolchain.py

@@ -12,7 +12,12 @@ from unittest.mock import call, patch
 
 import pytest
 
-from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT
+from esphome.components.esp32.const import (
+    KEY_ESP32,
+    KEY_FLASH_SIZE,
+    KEY_IDF_VERSION,
+    KEY_VARIANT,
+)
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_COMPILE_PROCESS_LIMIT,
@@ -112,12 +117,17 @@ def _setup_build(setup_core: Path) -> tuple[Path, Path]:
 
 
 @contextmanager
-def _up_to_date_compile(ninja_side_effect=None) -> Iterator[tuple]:
+def _up_to_date_compile(
+    ninja_side_effect=None, cached_bootloader: bool = False
+) -> Iterator[tuple]:
     """Patch run_compile's staleness checks to "up to date"; yield the ninja
     and print_summary mocks."""
     with (
         patch.object(toolchain, "need_reconfigure", return_value=False),
         patch.object(toolchain, "_cache_entries_changed", return_value=False),
+        patch.object(
+            toolchain, "_use_cached_bootloader", return_value=cached_bootloader
+        ),
         patch.object(
             toolchain, "_run_ninja", return_value=0, side_effect=ninja_side_effect
         ) as mock_ninja,
@@ -710,6 +720,7 @@ def _fake_tools(env: dict[str, str] | None = None) -> Iterator:
             return_value={"PATH": "/bin", "IDF_CCACHE_ENABLE": "0", **(env or {})},
         ),
         patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
+        patch.object(toolchain, "_use_cached_bootloader", return_value=False),
         patch.object(toolchain, "run_build_tool", return_value=0) as mock_run,
     ):
         yield mock_run
@@ -741,6 +752,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None:
         "-DESP_PLATFORM=1",
         f"-DSDKCONFIG={sdkconfig}",
         "-DCCACHE_ENABLE=0",
+        "-DESPHOME_USE_CACHED_BOOTLOADER=0",
         project,
     ]
     kwargs = mock_run.call_args.kwargs
@@ -775,6 +787,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py(
         "-DPYTHON=/tools/python",
         "-DESP_PLATFORM=1",
         f"-DCCACHE_ENABLE={expected}",
+        "-DESPHOME_USE_CACHED_BOOTLOADER=0",
         str(build_dir.parent),
     ]
 
@@ -786,7 +799,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None:
         assert toolchain.run_reconfigure(verbose=True) == 0
     cmd = mock_run.call_args.args[0]
     assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd)
-    assert cmd[-2] == "-DCCACHE_ENABLE=0"
+    assert cmd[-3] == "-DCCACHE_ENABLE=0"
+    assert cmd[-2] == "-DESPHOME_USE_CACHED_BOOTLOADER=0"
     assert mock_run.call_args.kwargs["filter_lines"] is None
 
 
@@ -853,6 +867,7 @@ _CONFIGURED = (
     "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n"
     "PYTHON:UNINITIALIZED=/tools/python\n"
     "ESP_PLATFORM:UNINITIALIZED=1\n"
+    "ESPHOME_USE_CACHED_BOOTLOADER:UNINITIALIZED=0\n"
 )
 
 
@@ -1104,3 +1119,182 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N
     with patch.object(toolchain, "_get_cmake_output", return_value=output):
         assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF"
         assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None
+
+
+def test_use_cached_bootloader_delegates_to_predicate(setup_core: Path) -> None:
+    from esphome.espidf import bootloader
+
+    with patch.object(bootloader, "bootloader_cache_enabled", return_value=True):
+        assert toolchain._use_cached_bootloader() is True
+
+
+def test_configure_defines_follow_bootloader_mode(setup_core: Path) -> None:
+    """The define always rides along, so a mode flip reconfigures."""
+    with (
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_cache_entries", return_value={}),
+    ):
+        with patch.object(toolchain, "_use_cached_bootloader", return_value=True):
+            on = toolchain._configure_defines()
+        with patch.object(toolchain, "_use_cached_bootloader", return_value=False):
+            off = toolchain._configure_defines()
+    assert on["ESPHOME_USE_CACHED_BOOTLOADER"] == "1"
+    assert off["ESPHOME_USE_CACHED_BOOTLOADER"] == "0"
+
+
+def test_run_compile_installs_cached_bootloader(setup_core: Path) -> None:
+    from esphome.espidf import bootloader
+
+    _setup_build(setup_core)
+    with (
+        _up_to_date_compile(cached_bootloader=True),
+        patch.object(
+            bootloader, "ensure_cached_bootloader", return_value=True
+        ) as mock_ensure,
+        patch.object(toolchain, "run_reconfigure") as mock_reconfigure,
+    ):
+        assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
+    mock_ensure.assert_called_once_with(False)
+    mock_reconfigure.assert_not_called()
+
+
+@pytest.mark.parametrize("reconfigure_rc", [0, 5])
+def test_run_compile_falls_back_when_bootloader_cache_fails(
+    setup_core: Path, reconfigure_rc: int
+) -> None:
+    """A cache failure flips the run to the stock in-tree build."""
+    from esphome.espidf import bootloader
+
+    _setup_build(setup_core)
+    with (
+        _up_to_date_compile(cached_bootloader=True) as (mock_ninja, _),
+        patch.object(bootloader, "ensure_cached_bootloader", return_value=False),
+        patch.object(
+            toolchain, "run_reconfigure", return_value=reconfigure_rc
+        ) as mock_reconfigure,
+    ):
+        rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
+    assert rc == reconfigure_rc
+    # The fallback pins the memoized predicate for the rest of the run.
+    assert toolchain._cache().bootloader_enabled is False
+    mock_reconfigure.assert_called_once_with(False)
+    assert mock_ninja.called is (reconfigure_rc == 0)
+
+
+def test_create_factory_bin_merges_cached_bootloader(setup_core: Path) -> None:
+    """The cached bootloader joins the merge at its flash offset even
+    though IDF wrote no bootloader entry into flasher_args.json."""
+    _setup_build(setup_core)
+    build = CORE.relative_build_path("build")
+    (build / "bootloader").mkdir(parents=True)
+    (build / "bootloader" / "bootloader.bin").write_bytes(b"\xe9")
+    (build / "CMakeCache.txt").write_text(
+        f"{toolchain.USE_CACHED_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
+    )
+    (build / "config").mkdir()
+    (build / "config" / "sdkconfig.json").write_text(
+        json.dumps({"BOOTLOADER_OFFSET_IN_FLASH": 0x1000})
+    )
+    (build / "test.bin").write_bytes(b"a")
+    (build / "flasher_args.json").write_text(
+        json.dumps(
+            {
+                "flash_files": {"0x10000": "test.bin"},
+                "extra_esptool_args": {"chip": "esp32"},
+            }
+        )
+    )
+    CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB"
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(
+            toolchain.subprocess,
+            "run",
+            return_value=subprocess.CompletedProcess([], 0, "", ""),
+        ) as mock_run,
+    ):
+        assert toolchain.create_factory_bin() is True
+    cmd = mock_run.call_args.args[0]
+    offset_index = cmd.index("0x1000")
+    assert cmd[offset_index + 1] == str(build / "bootloader" / "bootloader.bin")
+    # Sections stay sorted by address; the bootloader comes first.
+    assert offset_index < cmd.index("0x10000")
+
+
+def test_create_factory_bin_fails_when_the_cached_bootloader_is_missing(
+    setup_core: Path,
+) -> None:
+    """A factory image without its bootloader would not boot; fail instead."""
+    _setup_build(setup_core)
+    build = CORE.relative_build_path("build")
+    build.mkdir(parents=True)
+    (build / "CMakeCache.txt").write_text(
+        f"{toolchain.USE_CACHED_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
+    )
+    (build / "flasher_args.json").write_text('{"flash_files": {}}')
+    with patch.object(toolchain.subprocess, "run") as mock_run:
+        assert toolchain.create_factory_bin() is False
+    mock_run.assert_not_called()
+
+
+def test_idf_target_from_variant(setup_core: Path) -> None:
+    CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32C6"}
+    assert toolchain._idf_target() == "esp32c6"
+
+
+def test_load_sdkconfig_json(setup_core: Path, tmp_path: Path) -> None:
+    assert toolchain._load_sdkconfig_json(tmp_path) is None
+    (tmp_path / "config").mkdir()
+    (tmp_path / "config" / "sdkconfig.json").write_text('{"A": 1}')
+    assert toolchain._load_sdkconfig_json(tmp_path) == {"A": 1}
+    (tmp_path / "config" / "sdkconfig.json").write_text("[]")
+    assert toolchain._load_sdkconfig_json(tmp_path) is None
+
+
+def test_resolved_c_compiler_from_project_description(setup_core: Path) -> None:
+    _setup_build(setup_core)
+    build = CORE.relative_build_path("build")
+    build.mkdir(parents=True)
+    (build / "project_description.json").write_text(
+        json.dumps({"c_compiler": "/tools/xtensa-esp32-elf-gcc"})
+    )
+    expected = os.path.realpath("/tools/xtensa-esp32-elf-gcc")
+    assert toolchain._resolved_c_compiler() == expected
+
+
+@pytest.mark.parametrize(
+    "description",
+    [None, "not json", "{}", '{"c_compiler": ""}', "[]"],
+    ids=["missing", "corrupt", "no-key", "empty", "not-a-dict"],
+)
+def test_resolved_c_compiler_none_without_a_usable_description(
+    setup_core: Path, description: str | None
+) -> None:
+    _setup_build(setup_core)
+    if description is not None:
+        build = CORE.relative_build_path("build")
+        build.mkdir(parents=True)
+        (build / "project_description.json").write_text(description)
+    assert toolchain._resolved_c_compiler() is None
+
+
+def test_parse_sdkconfig(tmp_path: Path) -> None:
+    from esphome.espidf import parse_sdkconfig
+
+    path = tmp_path / "sdkconfig"
+    path.write_text(
+        "# comment\n"
+        "# CONFIG_DISABLED is not set\n"
+        "CONFIG_INT=240\n"
+        'CONFIG_STR="key.pem"\n'
+        'CONFIG_EMPTY=""\n'
+        "CONFIG_BOOL=y\n"
+        "not_config=1\n"
+    )
+    assert parse_sdkconfig(path) == {
+        "CONFIG_INT": "240",
+        "CONFIG_STR": "key.pem",
+        "CONFIG_EMPTY": "",
+        "CONFIG_BOOL": "y",
+    }

tests/unit_tests/test_helpers.py

@@ -1250,3 +1250,17 @@ def test_get_usable_cpu_count_sources() -> None:
     mock_os_unknown = types.SimpleNamespace(cpu_count=lambda: None)
     with patch("esphome.helpers.os", mock_os_unknown):
         assert helpers.get_usable_cpu_count() == 1
+
+
+@pytest.mark.parametrize(
+    ("content", "expected"),
+    [('{"a": 1}', {"a": 1}), ("[1, 2]", [1, 2]), ("not json", None), (None, None)],
+    ids=["object", "list", "corrupt", "missing"],
+)
+def test_read_json_file(
+    tmp_path: Path, content: str | None, expected: dict | list | None
+) -> None:
+    path = tmp_path / "data.json"
+    if content is not None:
+        path.write_text(content)
+    assert helpers.read_json_file(path) == expected