[espidf] Cache the bootloader across builds

esphome/build_gen/espidf.py

@@ -80,6 +80,51 @@ def _cmake_quote(value: str) -> str:
     return f'"{escaped}"'
 
 
+# CONFIG_APP_BUILD_BOOTLOADER is a hidden Kconfig option force-selected by
+# APP_BUILD_TYPE_APP_2NDBOOT, so sdkconfig cannot turn it off. Clearing it at
+# the CMake level puts the build into the same first-class state IDF's RAM-app
+# build type uses: the bootloader guards return early, esptool_py drops the
+# bootloader flash entry, and nothing else changes (sdkconfig.h still says 1).
+# The macro below is IDF's __build_process_project_includes (tools/cmake/
+# build.cmake, identical in all supported IDF versions) with one added set();
+# esphome.espidf.bootloader compares the live macro against this copy and
+# falls back to the in-tree bootloader build if an IDF update changes it.
+IDF_BOOTLOADER_OVERRIDE = """\
+# ESPHome: with ESPHOME_USE_CACHED_BOOTLOADER the bootloader comes from a
+# machine-global cache instead of the in-tree ExternalProject. This is IDF's
+# own __build_process_project_includes macro with one added set(); see
+# esphome/espidf/bootloader.py.
+if(ESPHOME_USE_CACHED_BOOTLOADER)
+    macro(__build_process_project_includes)
+        idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
+        include(${sdkconfig_cmake})
+        # ESPHome: the bootloader comes from the machine-global cache
+        set(CONFIG_APP_BUILD_BOOTLOADER "")
+        idf_build_get_property(build_properties __BUILD_PROPERTIES)
+        foreach(build_property ${build_properties})
+            idf_build_get_property(val ${build_property})
+            set(${build_property} "${val}")
+        endforeach()
+        idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
+        foreach(component_target ${build_component_targets})
+            __component_get_property(dir ${component_target} COMPONENT_DIR)
+            __component_get_property(_name ${component_target} COMPONENT_NAME)
+            set(COMPONENT_NAME ${_name})
+            set(COMPONENT_DIR ${dir})
+            set(COMPONENT_PATH ${dir})
+            if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
+                include(${COMPONENT_DIR}/project_include.cmake)
+            endif()
+        endforeach()
+    endmacro()
+endif()
+"""
+
+# The one line the override adds to IDF's macro; the staleness tripwire in
+# esphome.espidf.bootloader strips it before comparing with the live macro.
+BOOTLOADER_OVERRIDE_ADDED_LINE = 'set(CONFIG_APP_BUILD_BOOTLOADER "")'
+
+
 def get_project_cmakelists(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> str:
@@ -202,6 +247,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src)
 
 include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
+{IDF_BOOTLOADER_OVERRIDE}
 {cpp_standard_options}
 
 {cxx_compile_options}

esphome/espidf/bootloader.py

@@ -0,0 +1,415 @@
+"""Machine-global bootloader cache for the native ESP-IDF toolchain.
+
+The bootloader is its own CMake project that depends only on the IDF version,
+target, compiler and the config subset its reduced Kconfig universe defines,
+so one build serves every device that shares those. With
+``CONFIG_APP_REPRODUCIBLE_BUILD`` (always on) the result is byte identical to
+the in-tree ExternalProject build. Builds that configure signing, secure boot
+or flash encryption never use the cache: their bootloader can depend on key
+file contents, and they keep the stock in-tree path.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import shutil
+import time
+
+from esphome.build_helpers.ccache import parse_enable_env
+from esphome.build_helpers.tool_runner import run_build_tool
+from esphome.const import KEY_ESP32, KEY_VARIANT
+from esphome.core import CORE, EsphomeError
+from esphome.espidf import toolchain, variant_to_idf_target
+from esphome.helpers import write_file
+
+_LOGGER = logging.getLogger(__name__)
+
+BOOTLOADER_CACHE_ENV = "ESPHOME_BOOTLOADER_CACHE"
+# Bump on any change to how cached bootloaders are built or stored; old
+# entries then miss instead of being served stale.
+_CACHE_SCHEMA = "1"
+_SECURE_OPTION = re.compile(r"CONFIG_(SECURE_|FLASH_ENCRYPTION)")
+_MACRO = re.compile(
+    r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL
+)
+# Marks a config name the app's sdkconfig does not define; distinct from
+# every real value.
+_ABSENT = "\x00absent"
+_OUTPUTS = ("bootloader.bin", "bootloader.elf", "bootloader.map")
+
+
+def bootloader_cache_enabled() -> bool:
+    """Whether this build may take its bootloader from the cache.
+
+    False keeps IDF's in-tree bootloader build, exactly as before.
+    """
+    cache = toolchain._cache()
+    if cache.bootloader_enabled is None:
+        cache.bootloader_enabled = _compute_enabled()
+    return cache.bootloader_enabled
+
+
+def _compute_enabled() -> bool:
+    if parse_enable_env(BOOTLOADER_CACHE_ENV) is False:
+        return False
+    if "IDF_PATH" in os.environ:
+        # An unmanaged IDF checkout can change under the cache.
+        return False
+    try:
+        # The .esphomeinternal snapshot holds exactly what ESPHome and the
+        # user configured; the live sdkconfig is rewritten by kconfgen with
+        # every resolved option, including always-on *_SUPPORTED constants.
+        snapshot = CORE.relative_build_path(f"sdkconfig.{CORE.name}.esphomeinternal")
+        if not snapshot.is_file() or _has_secure_options(snapshot):
+            return False
+        return idf_macro_matches()
+    except (OSError, KeyError, ValueError, EsphomeError) as err:
+        _LOGGER.debug("Bootloader cache disabled: %s", err)
+        return False
+
+
+def _has_secure_options(sdkconfig_path: Path) -> bool:
+    """Whether any signing, secure boot or encryption option is enabled.
+
+    Reads ESPHome's sdkconfig snapshot, so raw ``sdkconfig_options`` count.
+    A disabled value (``n``/``0``/empty) is the compiled-out state ESPHome
+    writes by default and is safe to cache; anything else bypasses.
+    """
+    for line in sdkconfig_path.read_text(encoding="utf-8").splitlines():
+        name, _, value = line.partition("=")
+        if _SECURE_OPTION.match(name.strip()) and value.strip().strip('"') not in (
+            "n",
+            "0",
+            "",
+        ):
+            return True
+    return False
+
+
+def _normalized_macro(text: str) -> list[str] | None:
+    """The macro body as comment-free, whitespace-collapsed lines."""
+    if (match := _MACRO.search(text)) is None:
+        return None
+    return [
+        re.sub(r"\s+", " ", line)
+        for raw in match.group(1).splitlines()
+        if (line := raw.split("#", 1)[0].strip())
+    ]
+
+
+def idf_macro_matches() -> bool:
+    """Whether IDF's macro still matches the copy the override replays.
+
+    A mismatch after an IDF update silently falls back to the in-tree
+    bootloader build; script/check_idf_py_equivalence.py reports it loudly.
+    """
+    from esphome.build_gen.espidf import (
+        BOOTLOADER_OVERRIDE_ADDED_LINE,
+        IDF_BOOTLOADER_OVERRIDE,
+    )
+
+    expected = _normalized_macro(IDF_BOOTLOADER_OVERRIDE)
+    expected.remove(BOOTLOADER_OVERRIDE_ADDED_LINE)
+    build_cmake = toolchain._get_idf_path() / "tools" / "cmake" / "build.cmake"
+    live = _normalized_macro(build_cmake.read_text(encoding="utf-8"))
+    return live == expected
+
+
+def _idf_target() -> str:
+    return variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
+
+
+def _cache_root() -> Path:
+    from esphome.espidf.framework import get_idf_tools_path
+
+    return (
+        get_idf_tools_path()
+        / "bootloaders"
+        / toolchain._get_core_framework_version()
+        / _idf_target()
+    )
+
+
+def _load_config_names() -> list[str] | None:
+    """The known union of config option names the bootloader consumes."""
+    try:
+        names = json.loads(
+            (_cache_root() / "config_names.json").read_text(encoding="utf-8")
+        )
+    except (OSError, ValueError):
+        return None
+    if isinstance(names, list) and names and all(isinstance(n, str) for n in names):
+        return names
+    return None
+
+
+def _merge_config_names(names) -> list[str]:
+    """Union-merge harvested names; the list only ever grows."""
+    merged = sorted(set(_load_config_names() or ()) | set(names))
+    write_file(
+        _cache_root() / "config_names.json",
+        json.dumps(merged, separators=(",", ":")),
+    )
+    return merged
+
+
+def _load_build_config(build_dir: Path) -> dict | None:
+    """A build's generated config/sdkconfig.json, or None."""
+    try:
+        return json.loads(
+            (build_dir / "config" / "sdkconfig.json").read_text(encoding="utf-8")
+        )
+    except (OSError, ValueError):
+        return None
+
+
+def _compiler_id() -> str | None:
+    """The app build's C compiler, resolved; the toolchain part of the key.
+
+    IDF's toolchain file sets the compiler, so CMakeCache.txt has no plain
+    CMAKE_C_COMPILER entry; the compile database names it instead. None
+    means the cache cannot be trusted for this run and the caller falls
+    back to the in-tree build.
+    """
+    try:
+        commands = json.loads(
+            (toolchain._build_dir() / "compile_commands.json").read_text(
+                encoding="utf-8"
+            )
+        )
+        compiler = commands[0]["command"].split()[0]
+    except (OSError, ValueError, LookupError):
+        cache_path = toolchain._build_dir() / "CMakeCache.txt"
+        if not cache_path.is_file():
+            return None
+        # Same install dir and version as the compiler itself.
+        compiler = toolchain._parse_cmakecache(cache_path).get("CMAKE_C_COMPILER_AR")
+    return os.path.realpath(compiler) if compiler else None
+
+
+def _subproject_cmake_args(
+    cmake: str, python: str, sdkconfig: str, idf_path: str, project_dir: str
+) -> list[str]:
+    """The configure argv, mirroring the ExternalProject in IDF's
+    components/bootloader/project_include.cmake for the pinned version."""
+    from esphome.components.esp32 import idf_version
+    import esphome.config_validation as cv
+
+    args = [
+        cmake,
+        "-G",
+        "Ninja",
+        f"-DSDKCONFIG={sdkconfig}",
+        f"-DIDF_PATH={idf_path}",
+        f"-DIDF_TARGET={_idf_target()}",
+        "-DPYTHON_DEPS_CHECKED=1",
+        f"-DPYTHON={python}",
+    ]
+    if idf_version() >= cv.Version(6, 0, 0):
+        # 6.x: no BOOTLOADER_EXTRA_COMPONENT_DIRS self-append; IDF_BUILD_V2
+        # forwarded (empty: the env opt-in is never set).
+        args += ["-DEXTRA_COMPONENT_DIRS=", "-DIDF_BUILD_V2="]
+    else:
+        args.append(f"-DEXTRA_COMPONENT_DIRS={idf_path}/components/bootloader")
+    args += [
+        f"-DPROJECT_SOURCE_DIR={project_dir}",
+        "-DIGNORE_EXTRA_COMPONENT=",
+        f"{idf_path}/components/bootloader/subproject",
+    ]
+    return args
+
+
+def _version_stamp() -> str:
+    """The installed framework's version.txt, part of the key."""
+    try:
+        return (
+            (toolchain._get_idf_path() / "version.txt")
+            .read_text(encoding="utf-8")
+            .strip()
+        )
+    except OSError:
+        return ""
+
+
+def _compute_key(names: list[str], app_config: dict, compiler: str) -> str:
+    """The cache entry name for this build's bootloader inputs."""
+    payload = json.dumps(
+        {
+            "schema": _CACHE_SCHEMA,
+            "idf": toolchain._get_core_framework_version(),
+            "source": toolchain._get_framework_source_override() or "",
+            "stamp": _version_stamp(),
+            "target": _idf_target(),
+            "compiler": compiler,
+            # Placeholder paths: any change to the invocation shape misses.
+            "args": _subproject_cmake_args(
+                "cmake", "@PYTHON@", "@SDKCONFIG@", "@IDF@", "@PROJECT@"
+            ),
+            "config": {n: app_config.get(n, _ABSENT) for n in sorted(names)},
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(payload.encode()).hexdigest()[:16]
+
+
+def _build_standalone(build_dir: Path, verbose: bool) -> int:
+    """Configure and build the bootloader subproject into ``build_dir``."""
+    build_dir.mkdir(parents=True, exist_ok=True)
+    args = _subproject_cmake_args(
+        toolchain._get_idf_tool("cmake"),
+        toolchain._get_idf_tool("python"),
+        str(CORE.relative_build_path(f"sdkconfig.{CORE.name}")),
+        str(toolchain._get_idf_path()),
+        os.path.realpath(CORE.build_path),
+    )
+    filter_lines = None if verbose else toolchain.FILTER_IDF_LINES
+    for name, cmd in (("cmake", args), ("ninja", [toolchain._get_idf_tool("ninja")])):
+        log_path = build_dir / "log" / f"{name}_output.log"
+        rc = run_build_tool(
+            cmd,
+            cwd=build_dir,
+            env=toolchain._tool_env(),
+            filter_lines=filter_lines,
+            log_path=log_path,
+        )
+        if rc != 0:
+            _LOGGER.error("Bootloader %s failed with exit code %d", name, rc)
+            toolchain._print_hints(log_path)
+            return rc
+    return 0
+
+
+def _publish(build_dir: Path, key: str, meta: dict) -> Path:
+    """Move the distilled outputs into the cache; losing a race is fine."""
+    entry = _cache_root() / key
+    if (entry / "bootloader.bin").is_file():
+        return entry
+    stage = _cache_root() / f".stage-{key}-{os.getpid()}"
+    stage.mkdir(parents=True, exist_ok=True)
+    for name in _OUTPUTS:
+        shutil.copy2(build_dir / name, stage / name)
+    shutil.copy2(build_dir / "config" / "sdkconfig.json", stage / "sdkconfig.json")
+    write_file(stage / "meta.json", json.dumps(meta, indent=2, sort_keys=True))
+    try:
+        stage.rename(entry)
+    except OSError:
+        # Another process published the same key first.
+        shutil.rmtree(stage, ignore_errors=True)
+        if not (entry / "bootloader.bin").is_file():
+            raise
+    return entry
+
+
+def _install_into_build(entry: Path) -> None:
+    """Copy the cached outputs to build/bootloader, the path every consumer
+    (factory image, OTA --bootloader, the CI guard) already reads."""
+    dest = toolchain._build_dir() / "bootloader"
+    if (dest / "CMakeCache.txt").is_file():
+        # A leftover in-tree sub-build; its ninja state must not linger.
+        shutil.rmtree(dest)
+    dest.mkdir(parents=True, exist_ok=True)
+    for name in _OUTPUTS:
+        if (src := entry / name).is_file():
+            shutil.copy2(src, dest / name)
+
+
+def _check_bootloader_size(bin_path: Path, app_config: dict) -> None:
+    """The check_sizes.py bootloader check, without spawning a process."""
+    offset = int(app_config.get("BOOTLOADER_OFFSET_IN_FLASH", 0))
+    table_offset = int(app_config.get("PARTITION_TABLE_OFFSET", 0x8000))
+    size = bin_path.stat().st_size
+    free = table_offset - offset - size
+    if free < 0:
+        raise EsphomeError(
+            f"Bootloader binary size {size:#x} bytes is too large for "
+            f"partition table offset {table_offset:#x} (overflows by "
+            f"{-free:#x} bytes). Increase CONFIG_PARTITION_TABLE_OFFSET."
+        )
+    _LOGGER.info(
+        "Bootloader binary size %#x bytes. %#x bytes (%d%%) free.",
+        size,
+        free,
+        round(100 * free / (table_offset - offset)),
+    )
+
+
+def _prune_stale_dirs() -> None:
+    """Drop day-old work dirs from crashed builds; live ones are younger."""
+    cutoff = time.time() - 86400
+    try:
+        for path in _cache_root().glob(".*"):
+            if path.is_dir() and path.stat().st_mtime < cutoff:
+                shutil.rmtree(path, ignore_errors=True)
+    except OSError:
+        pass
+
+
+def ensure_cached_bootloader(verbose: bool = False) -> int:
+    """Put the cached bootloader into build/bootloader, building on a miss.
+
+    Returns 0 on success. A nonzero tool exit code tells the caller to fall
+    back to the in-tree build; a too-large bootloader raises instead, since
+    the in-tree build would fail the same way.
+    """
+    app_config = _load_build_config(toolchain._build_dir())
+    compiler = _compiler_id()
+    if app_config is None or compiler is None:
+        _LOGGER.debug("Bootloader cache unusable: app configure outputs missing")
+        return 1
+    entry = None
+    if (names := _load_config_names()) is not None:
+        candidate = _cache_root() / _compute_key(names, app_config, compiler)
+        if (candidate / "bootloader.bin").is_file():
+            entry = candidate
+    if entry is None:
+        tmp = _cache_root() / f".build-{os.getpid()}"
+        try:
+            if (rc := _build_standalone(tmp, verbose)) != 0:
+                return rc
+            if (built_config := _load_build_config(tmp)) is None:
+                _LOGGER.debug("Bootloader build produced no sdkconfig.json")
+                return 1
+            names = _merge_config_names(built_config)
+            key = _compute_key(names, app_config, compiler)
+            entry = _publish(
+                tmp,
+                key,
+                {
+                    "schema": _CACHE_SCHEMA,
+                    "idf": toolchain._get_core_framework_version(),
+                    "target": _idf_target(),
+                    "compiler": compiler,
+                },
+            )
+            _LOGGER.info("Cached bootloader %s for later builds", key)
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+        _prune_stale_dirs()
+    _install_into_build(entry)
+    _check_bootloader_size(
+        toolchain._build_dir() / "bootloader" / "bootloader.bin", app_config
+    )
+    return 0
+
+
+def inject_bootloader_flash_file(flash_data: dict, build_dir: Path) -> None:
+    """Add the cached bootloader to a flasher_args ``flash_files`` map.
+
+    In cached mode IDF writes no bootloader entry; stock and bypass trees
+    already carry one, which makes this a no-op there.
+    """
+    flash_files = flash_data.setdefault("flash_files", {})
+    if any("bootloader/" in name for name in flash_files.values()):
+        return
+    if not (build_dir / "bootloader" / "bootloader.bin").is_file():
+        return
+    if (app_config := _load_build_config(build_dir)) is None:
+        return
+    offset = hex(int(app_config.get("BOOTLOADER_OFFSET_IN_FLASH", 0)))
+    flash_files[offset] = "bootloader/bootloader.bin"

esphome/espidf/toolchain.py

@@ -37,6 +37,8 @@ class _CacheData:
     paths: dict[str, tuple] = field(default_factory=dict)
     env: dict[str, dict[str, str]] = field(default_factory=dict)
     cmake_output: dict[Path, str] = field(default_factory=dict)
+    bootloader_enabled: bool | None = None
+    bootloader_fallback: bool = False
     cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict)
 
 
@@ -282,6 +284,15 @@ def _cache_entries_changed() -> bool:
     return any(cache.get(k) != v for k, v in _configure_defines().items())
 
 
+def _use_cached_bootloader() -> bool:
+    """Whether this run takes the bootloader from the machine-global cache."""
+    if _cache().bootloader_fallback:
+        return False
+    from esphome.espidf import bootloader
+
+    return bootloader.bootloader_cache_enabled()
+
+
 def _configure_defines() -> dict[str, str]:
     """Every ``-D`` idf.py passes to cmake, in its order."""
     return {
@@ -289,6 +300,10 @@ def _configure_defines() -> dict[str, str]:
         "PYTHON": _get_idf_tool("python"),
         "ESP_PLATFORM": "1",
         **_cache_entries(),
+        # ESPHome's own switch, appended after idf.py's entries. idf.py
+        # never passes it and cmake keeps the cached value, so an idf.py
+        # run against the tree stays in the same mode.
+        "ESPHOME_USE_CACHED_BOOTLOADER": "1" if _use_cached_bootloader() else "0",
     }
 
 
@@ -675,6 +690,16 @@ def run_compile(config, verbose: bool) -> int:
     else:
         _LOGGER.info("Build configuration is up to date")
 
+    if _use_cached_bootloader():
+        from esphome.espidf import bootloader
+
+        if bootloader.ensure_cached_bootloader(verbose) != 0:
+            # The stock in-tree build still works; flip this run over to it.
+            _LOGGER.warning("Cached bootloader unavailable; building it in-tree")
+            _cache().bootloader_fallback = True
+            if (rc := run_reconfigure(verbose)) != 0:
+                return rc
+
     # In testing mode, generate the linker script first, patch DRAM/IRAM sizes,
     # then build. memory.ld is regenerated by ninja during the build phase,
     # so we must patch after it's generated but before linking (same timing
@@ -809,6 +834,10 @@ def create_factory_bin() -> bool:
         _LOGGER.error("Failed to read flasher_args.json: %s", e)
         return False
 
+    from esphome.espidf import bootloader
+
+    bootloader.inject_bootloader_flash_file(flash_data, build_dir)
+
     # Get flash size from config
     flash_size = CORE.data[KEY_ESP32][KEY_FLASH_SIZE]
 

script/check_idf_py_equivalence.py

@@ -33,9 +33,18 @@ WATCHED = (
 )
 # Ninja logs whose outputs mean real work when their recorded mtime changes.
 # The top level re-logs the bootloader step's byproducts on every build, so
-# the bootloader is judged by its own sub-build log instead.
+# the bootloader is judged by its own sub-build log instead. With the cached
+# bootloader there is no sub-build at all; the cached bin is in WATCHED.
 TOP_NINJA_LOG = "build/.ninja_log"
-NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log")
+
+
+def _ninja_logs(build_path: Path) -> list[str]:
+    logs = [TOP_NINJA_LOG]
+    if (build_path / "build" / "bootloader" / "build.ninja").is_file():
+        logs.append("build/bootloader/.ninja_log")
+    return logs
+
+
 BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/")
 WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld")
 DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*"
@@ -57,7 +66,7 @@ def _snapshot(build_path: Path, name: str) -> dict[str, str | None]:
 def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
     """(log, output) -> recorded mtime; compaction-safe, unlike a line count."""
     mtimes = {}
-    for name in NINJA_LOGS:
+    for name in _ninja_logs(build_path):
         log = build_path / name
         lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else []
         for fields in (line.split("\t") for line in lines if not line.startswith("#")):
@@ -71,7 +80,7 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
 def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]:
     """A missing or unparsable ninja log would otherwise compare as unchanged."""
     problems = []
-    for log in NINJA_LOGS:
+    for log in _ninja_logs(build_path):
         if not (build_path / log).is_file():
             problems.append(f"missing {log}")
         elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes):
@@ -106,6 +115,13 @@ def check(build_path: Path) -> list[str]:
         (build_path / "build" / "project_description.json").read_text(encoding="utf-8")
     )
     name, version = _setup_core(build_path, description)
+    from esphome.espidf import bootloader
+
+    if not bootloader.idf_macro_matches():
+        return [
+            "IDF changed __build_process_project_includes; update "
+            "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
+        ]
     env = toolchain._get_idf_env(version)
     python = toolchain._get_idf_tool("python")
     idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py"

tests/script/test_check_idf_py_equivalence.py

@@ -14,7 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script"))
 import check_idf_py_equivalence as guard  # noqa: E402
 
 from esphome.core import CORE  # noqa: E402
-from esphome.espidf import toolchain  # noqa: E402
+from esphome.espidf import bootloader, toolchain  # noqa: E402
 
 
 @pytest.fixture(autouse=True)
@@ -24,12 +24,21 @@ def _reset_core() -> Iterator[None]:
     CORE.reset()
 
 
-def _make_tree(tmp_path: Path) -> Path:
+BOOTLOADER_LOG = "build/bootloader/.ninja_log"
+ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG)
+
+
+def _make_tree(tmp_path: Path, bootloader_subbuild: bool = True) -> Path:
+    """A fake build tree; the sub-build shape mirrors a bypass/stock build,
+    without it the cached-bootloader shape (bin present, no sub-build)."""
     tree = tmp_path / "config" / ".esphome" / "build" / "dev"
     build = tree / "build"
-    for name in (*guard.watched("dev"), *guard.NINJA_LOGS):
+    logs = ALL_LOGS if bootloader_subbuild else (guard.TOP_NINJA_LOG,)
+    for name in (*guard.watched("dev"), *logs):
         (tree / name).parent.mkdir(parents=True, exist_ok=True)
         (tree / name).write_bytes(b"x")
+    if bootloader_subbuild:
+        (build / "bootloader" / "build.ninja").write_bytes(b"x")
     (build / "project_description.json").write_text(
         json.dumps(
             {
@@ -70,6 +79,7 @@ def _run_check(
         patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
         patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]),
         patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]),
+        patch.object(bootloader, "idf_macro_matches", return_value=True),
         patch.object(guard.subprocess, "run", side_effect=run),
     ):
         return guard.check(tree), calls
@@ -150,9 +160,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None:
     assert len(calls) == 1
 
 
-@pytest.mark.parametrize(
-    "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS]
-)
+@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS])
 def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None:
     """A moved or renamed output must not compare as unchanged."""
     tree = _make_tree(tmp_path)
@@ -179,7 +187,7 @@ def test_check_stops_when_the_esphome_baseline_fails(
     assert calls == []
 
 
-@pytest.mark.parametrize("log", guard.NINJA_LOGS)
+@pytest.mark.parametrize("log", ALL_LOGS)
 def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None:
     """A log format change must not leave the rebuild check with nothing to compare."""
     tree = _make_tree(tmp_path)
@@ -205,6 +213,32 @@ def test_main_rejects_a_path_that_is_not_a_tree(
     mock_check.assert_not_called()
 
 
+def test_check_accepts_a_cached_bootloader_tree(tmp_path: Path) -> None:
+    """No bootloader sub-build is the cached shape, not a missing input."""
+    tree = _make_tree(tmp_path, bootloader_subbuild=False)
+    problems, calls = _run_check(tree)
+    assert problems == []
+    assert len(calls) == 2
+
+
+def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None:
+    """An IDF bump that rewrites the overridden macro must fail CI."""
+    tree = _make_tree(tmp_path)
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(bootloader, "idf_macro_matches", return_value=False),
+        patch.object(guard.subprocess, "run") as mock_run,
+    ):
+        problems = guard.check(tree)
+    assert problems == [
+        "IDF changed __build_process_project_includes; update "
+        "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
+    ]
+    mock_run.assert_not_called()
+
+
 def test_main_without_build_trees(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:

tests/unit_tests/test_espidf_bootloader.py

@@ -0,0 +1,650 @@
+"""Tests for esphome.espidf.bootloader."""
+
+# pylint: disable=protected-access
+
+import json
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT
+import esphome.config_validation as cv
+from esphome.core import CORE, EsphomeError
+from esphome.espidf import bootloader, toolchain
+
+# The macro body as shipped in build.cmake; byte identical in IDF 5.5.5 and
+# 6.1.0, so one fixture covers both supported versions.
+IDF_BUILD_CMAKE = """\
+some_other_cmake()
+
+macro(__build_process_project_includes)
+    # Include the sdkconfig cmake file, since the following operations require
+    # knowledge of config values.
+    idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
+    include(${sdkconfig_cmake})
+
+    # Make each build property available as a read-only variable
+    idf_build_get_property(build_properties __BUILD_PROPERTIES)
+    foreach(build_property ${build_properties})
+        idf_build_get_property(val ${build_property})
+        set(${build_property} "${val}")
+    endforeach()
+
+    idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
+
+    # Include each component's project_include.cmake
+    foreach(component_target ${build_component_targets})
+        __component_get_property(dir ${component_target} COMPONENT_DIR)
+        __component_get_property(_name ${component_target} COMPONENT_NAME)
+        set(COMPONENT_NAME ${_name})
+        set(COMPONENT_DIR ${dir})
+        set(COMPONENT_PATH ${dir})  # this is deprecated, users are encouraged to use COMPONENT_DIR;
+                                    # retained for compatibility
+        if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
+            include(${COMPONENT_DIR}/project_include.cmake)
+        endif()
+    endforeach()
+endmacro()
+"""
+
+
+@pytest.fixture(autouse=True)
+def _fresh_cache(setup_core: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Point CORE at a build tree and reset the per-run toolchain cache."""
+    monkeypatch.delenv(bootloader.BOOTLOADER_CACHE_ENV, raising=False)
+    monkeypatch.delenv("IDF_PATH", raising=False)
+    CORE.name = "dev"
+    CORE.build_path = setup_core / "build" / "dev"
+    CORE.data[KEY_ESP32] = {
+        KEY_VARIANT: "ESP32",
+        KEY_IDF_VERSION: cv.Version(5, 5, 5),
+    }
+    CORE.data.pop(toolchain.DOMAIN, None)
+
+
+def _write_snapshot(text: str = "") -> Path:
+    path = CORE.relative_build_path("sdkconfig.dev.esphomeinternal")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text)
+    return path
+
+
+def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path:
+    idf = tmp_path / "idf"
+    (idf / "tools" / "cmake").mkdir(parents=True)
+    (idf / "tools" / "cmake" / "build.cmake").write_text(text)
+    return idf
+
+
+# ---------------------------------------------------------------- predicate
+
+
+def test_enabled_env_zero_disables(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(bootloader.BOOTLOADER_CACHE_ENV, "0")
+    assert bootloader._compute_enabled() is False
+
+
+def test_enabled_user_idf_path_disables(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An unmanaged IDF checkout can change under the cache."""
+    monkeypatch.setenv("IDF_PATH", "/somewhere")
+    assert bootloader._compute_enabled() is False
+
+
+def test_enabled_missing_snapshot_disables() -> None:
+    """No snapshot means the config origin is unknown; fail safe."""
+    assert bootloader._compute_enabled() is False
+
+
+def test_enabled_secure_option_disables() -> None:
+    _write_snapshot("CONFIG_SECURE_BOOT=y\n")
+    assert bootloader._compute_enabled() is False
+
+
+def test_enabled_clean_snapshot_checks_macro(tmp_path: Path) -> None:
+    _write_snapshot("CONFIG_FOO=y\n")
+    idf = _write_idf_build_cmake(tmp_path)
+    with patch.object(toolchain, "_get_idf_path", return_value=idf):
+        assert bootloader._compute_enabled() is True
+
+
+def test_enabled_swallows_errors_as_disabled(tmp_path: Path) -> None:
+    """Any failure while deciding must read as disabled, never raise."""
+    _write_snapshot("CONFIG_FOO=y\n")
+    with patch.object(toolchain, "_get_idf_path", side_effect=OSError("boom")):
+        assert bootloader._compute_enabled() is False
+
+
+def test_bootloader_cache_enabled_is_cached_per_run() -> None:
+    with patch.object(
+        bootloader, "_compute_enabled", return_value=True
+    ) as mock_compute:
+        assert bootloader.bootloader_cache_enabled() is True
+        assert bootloader.bootloader_cache_enabled() is True
+    mock_compute.assert_called_once_with()
+
+
+@pytest.mark.parametrize(
+    ("text", "secure"),
+    [
+        ("CONFIG_SECURE_BOOT=y\n", True),
+        ("CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT=y\n", True),
+        ("CONFIG_FLASH_ENCRYPTION_ENABLED=y\n", True),
+        ('CONFIG_SECURE_BOOT_SIGNING_KEY="key.pem"\n', True),
+        ("CONFIG_SECURE_BOOT=n\n", False),
+        ("CONFIG_SECURE_BOOT=0\n", False),
+        ('CONFIG_SECURE_BOOT_SIGNING_KEY=""\n', False),
+        ("# CONFIG_SECURE_BOOT is not set\n", False),
+        ("CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y\n", False),
+        ("", False),
+    ],
+    ids=[
+        "secure-boot",
+        "signed-apps",
+        "flash-encryption",
+        "signing-key",
+        "value-n",
+        "value-0",
+        "empty-string",
+        "comment-line",
+        "unrelated",
+        "empty-file",
+    ],
+)
+def test_has_secure_options(tmp_path: Path, text: str, secure: bool) -> None:
+    """Only an enabled secure option bypasses; disabled values are the
+    compiled-out state ESPHome writes by default."""
+    path = tmp_path / "sdkconfig"
+    path.write_text(text)
+    assert bootloader._has_secure_options(path) is secure
+
+
+# ------------------------------------------------------------ macro tripwire
+
+
+def test_normalized_macro_strips_comments_and_whitespace() -> None:
+    text = "macro(__build_process_project_includes)\n  a( b )  # tail\n\n  # only\n  c(d)\nendmacro()"
+    assert bootloader._normalized_macro(text) == ["a( b )", "c(d)"]
+
+
+def test_normalized_macro_none_without_macro() -> None:
+    assert bootloader._normalized_macro("nothing here") is None
+
+
+def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None:
+    """The override's embedded copy must equal what build.cmake ships."""
+    idf = _write_idf_build_cmake(tmp_path)
+    with patch.object(toolchain, "_get_idf_path", return_value=idf):
+        assert bootloader.idf_macro_matches() is True
+
+
+def test_idf_macro_mismatch_detected(tmp_path: Path) -> None:
+    changed = IDF_BUILD_CMAKE.replace(
+        "include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)"
+    )
+    idf = _write_idf_build_cmake(tmp_path, changed)
+    with patch.object(toolchain, "_get_idf_path", return_value=idf):
+        assert bootloader.idf_macro_matches() is False
+
+
+# ------------------------------------------------------------------ cache key
+
+
+def _key(names: list[str], config: dict, compiler: str = "/tc/gcc") -> str:
+    with (
+        patch.object(toolchain, "_get_core_framework_version", return_value="5.5.5"),
+        patch.object(toolchain, "_get_framework_source_override", return_value=None),
+        patch.object(bootloader, "_version_stamp", return_value="v5.5.5"),
+    ):
+        return bootloader._compute_key(names, config, compiler)
+
+
+def test_key_is_stable_and_name_order_independent() -> None:
+    config = {"A": "1", "B": True}
+    key = _key(["A", "B"], config)
+    assert key == _key(["B", "A"], config)
+    assert len(key) == 16
+
+
+def test_key_distinguishes_absent_from_empty() -> None:
+    """An option missing from the app config is not the same as ''."""
+    assert _key(["A"], {"A": ""}) != _key(["A"], {})
+
+
+def test_key_changes_with_each_input() -> None:
+    base = _key(["A"], {"A": "1"})
+    assert _key(["A"], {"A": "2"}) != base
+    assert _key(["A"], {"A": "1"}, compiler="/other/gcc") != base
+    with (
+        patch.object(toolchain, "_get_core_framework_version", return_value="6.1.0"),
+        patch.object(toolchain, "_get_framework_source_override", return_value=None),
+        patch.object(bootloader, "_version_stamp", return_value="v5.5.5"),
+    ):
+        CORE.data[KEY_ESP32][KEY_IDF_VERSION] = cv.Version(6, 1, 0)
+        assert bootloader._compute_key(["A"], {"A": "1"}, "/tc/gcc") != base
+
+
+def test_key_ignores_unlisted_config_values() -> None:
+    """Only the names the bootloader consumes participate."""
+    assert _key(["A"], {"A": "1", "Z": "app-only"}) == _key(["A"], {"A": "1"})
+
+
+def test_version_stamp_reads_version_txt(tmp_path: Path) -> None:
+    (tmp_path / "version.txt").write_text("v5.5.5\n")
+    with patch.object(toolchain, "_get_idf_path", return_value=tmp_path):
+        assert bootloader._version_stamp() == "v5.5.5"
+
+
+def test_version_stamp_missing_file(tmp_path: Path) -> None:
+    with patch.object(toolchain, "_get_idf_path", return_value=tmp_path):
+        assert bootloader._version_stamp() == ""
+
+
+def test_cache_root_layout(tmp_path: Path) -> None:
+    """Entries live under the registered IDF tools cache, per version and
+    target, so clean-all removes them with the toolchains."""
+    from esphome.espidf import framework
+
+    with (
+        patch.object(framework, "get_idf_tools_path", return_value=tmp_path),
+        patch.object(toolchain, "_get_core_framework_version", return_value="5.5.5"),
+    ):
+        assert bootloader._cache_root() == tmp_path / "bootloaders" / "5.5.5" / "esp32"
+
+
+# ----------------------------------------------------------- config names
+
+
+def test_merge_config_names_unions_and_sorts(tmp_path: Path) -> None:
+    root = tmp_path / "cache"
+    with patch.object(bootloader, "_cache_root", return_value=root):
+        assert bootloader._merge_config_names(["B", "A"]) == ["A", "B"]
+        assert bootloader._merge_config_names(["C", "A"]) == ["A", "B", "C"]
+        assert bootloader._load_config_names() == ["A", "B", "C"]
+
+
+@pytest.mark.parametrize(
+    "content",
+    [None, "not json", "[]", '["a", 1]', '"str"'],
+    ids=["missing", "corrupt", "empty", "mixed-types", "not-a-list"],
+)
+def test_load_config_names_rejects_bad_files(
+    tmp_path: Path, content: str | None
+) -> None:
+    if content is not None:
+        (tmp_path / "config_names.json").write_text(content)
+    with patch.object(bootloader, "_cache_root", return_value=tmp_path):
+        assert bootloader._load_config_names() is None
+
+
+def test_load_build_config(tmp_path: Path) -> None:
+    assert bootloader._load_build_config(tmp_path) is None
+    (tmp_path / "config").mkdir()
+    (tmp_path / "config" / "sdkconfig.json").write_text('{"A": 1}')
+    assert bootloader._load_build_config(tmp_path) == {"A": 1}
+
+
+# ------------------------------------------------------------- compiler id
+
+
+def test_compiler_id_from_compile_commands(tmp_path: Path) -> None:
+    build = CORE.relative_build_path("build")
+    build.mkdir(parents=True)
+    (build / "compile_commands.json").write_text(
+        json.dumps([{"command": "/tools/xtensa-esp32-elf-gcc -c a.c"}])
+    )
+    assert bootloader._compiler_id() == "/tools/xtensa-esp32-elf-gcc"
+
+
+def test_compiler_id_falls_back_to_cmakecache(tmp_path: Path) -> None:
+    build = CORE.relative_build_path("build")
+    build.mkdir(parents=True)
+    (build / "CMakeCache.txt").write_text(
+        "CMAKE_C_COMPILER_AR:FILEPATH=/tools/xtensa-esp32-elf-gcc-ar\n"
+    )
+    assert bootloader._compiler_id() == "/tools/xtensa-esp32-elf-gcc-ar"
+
+
+def test_compiler_id_none_without_configure_outputs() -> None:
+    assert bootloader._compiler_id() is None
+
+
+# ------------------------------------------------------------- cmake args
+
+
+def test_subproject_args_for_idf_5() -> None:
+    args = bootloader._subproject_cmake_args("cmake", "/py", "/sdk", "/idf", "/proj")
+    assert args == [
+        "cmake",
+        "-G",
+        "Ninja",
+        "-DSDKCONFIG=/sdk",
+        "-DIDF_PATH=/idf",
+        "-DIDF_TARGET=esp32",
+        "-DPYTHON_DEPS_CHECKED=1",
+        "-DPYTHON=/py",
+        "-DEXTRA_COMPONENT_DIRS=/idf/components/bootloader",
+        "-DPROJECT_SOURCE_DIR=/proj",
+        "-DIGNORE_EXTRA_COMPONENT=",
+        "/idf/components/bootloader/subproject",
+    ]
+
+
+def test_subproject_args_for_idf_6() -> None:
+    """6.x dropped the bootloader dir self-append and forwards IDF_BUILD_V2."""
+    CORE.data[KEY_ESP32][KEY_IDF_VERSION] = cv.Version(6, 1, 0)
+    args = bootloader._subproject_cmake_args("cmake", "/py", "/sdk", "/idf", "/proj")
+    assert "-DEXTRA_COMPONENT_DIRS=" in args
+    assert "-DIDF_BUILD_V2=" in args
+    assert "-DEXTRA_COMPONENT_DIRS=/idf/components/bootloader" not in args
+    assert args[-1] == "/idf/components/bootloader/subproject"
+
+
+# -------------------------------------------------------- standalone build
+
+
+def test_build_standalone_runs_cmake_then_ninja(tmp_path: Path) -> None:
+    build_dir = tmp_path / "work"
+    with (
+        patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(toolchain, "_tool_env", return_value={"A": "1"}),
+        patch.object(bootloader, "run_build_tool", return_value=0) as mock_run,
+    ):
+        assert bootloader._build_standalone(build_dir, verbose=True) == 0
+    cmake_call, ninja_call = mock_run.call_args_list
+    assert cmake_call.args[0][0] == "/tools/cmake"
+    assert cmake_call.args[0][-1] == "/idf/components/bootloader/subproject"
+    assert cmake_call.kwargs["cwd"] == build_dir
+    assert cmake_call.kwargs["filter_lines"] is None  # verbose
+    assert cmake_call.kwargs["log_path"] == build_dir / "log" / "cmake_output.log"
+    assert ninja_call.args[0] == ["/tools/ninja"]
+
+
+def test_build_standalone_failure_prints_hints(tmp_path: Path) -> None:
+    build_dir = tmp_path / "work"
+    with (
+        patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
+        patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
+        patch.object(toolchain, "_tool_env", return_value={}),
+        patch.object(bootloader, "run_build_tool", return_value=2) as mock_run,
+        patch.object(toolchain, "_print_hints") as mock_hints,
+    ):
+        assert bootloader._build_standalone(build_dir, verbose=False) == 2
+    # Failed at cmake; ninja never ran, the hint scanner got the log.
+    assert len(mock_run.call_args_list) == 1
+    assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES
+    mock_hints.assert_called_once_with(build_dir / "log" / "cmake_output.log")
+
+
+# ----------------------------------------------------------------- publish
+
+
+def _make_built_tree(build_dir: Path) -> None:
+    build_dir.mkdir(parents=True, exist_ok=True)
+    for name in bootloader._OUTPUTS:
+        (build_dir / name).write_bytes(name.encode())
+    (build_dir / "config").mkdir(exist_ok=True)
+    (build_dir / "config" / "sdkconfig.json").write_text('{"A": "1"}')
+
+
+def test_publish_creates_entry(tmp_path: Path) -> None:
+    root = tmp_path / "cache"
+    build = tmp_path / "work"
+    _make_built_tree(build)
+    with patch.object(bootloader, "_cache_root", return_value=root):
+        entry = bootloader._publish(build, "k" * 16, {"idf": "5.5.5"})
+    assert entry == root / ("k" * 16)
+    assert (entry / "bootloader.bin").read_bytes() == b"bootloader.bin"
+    assert json.loads((entry / "meta.json").read_text()) == {"idf": "5.5.5"}
+    assert not list(root.glob(".stage-*"))
+
+
+def test_publish_existing_entry_short_circuits(tmp_path: Path) -> None:
+    root = tmp_path / "cache"
+    entry = root / ("k" * 16)
+    entry.mkdir(parents=True)
+    (entry / "bootloader.bin").write_bytes(b"winner")
+    with patch.object(bootloader, "_cache_root", return_value=root):
+        assert bootloader._publish(tmp_path / "work", "k" * 16, {}) == entry
+    assert (entry / "bootloader.bin").read_bytes() == b"winner"
+
+
+def test_publish_lost_race_reuses_winner(tmp_path: Path) -> None:
+    """A rename beaten by another process keeps that process's entry."""
+    root = tmp_path / "cache"
+    build = tmp_path / "work"
+    _make_built_tree(build)
+    entry = root / ("k" * 16)
+
+    def rename_raises(self: Path, target: Path) -> None:
+        entry.mkdir(parents=True)
+        (entry / "bootloader.bin").write_bytes(b"winner")
+        raise OSError("directory not empty")
+
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(Path, "rename", rename_raises),
+    ):
+        assert bootloader._publish(build, "k" * 16, {}) == entry
+    assert (entry / "bootloader.bin").read_bytes() == b"winner"
+    assert not list(root.glob(".stage-*"))
+
+
+def test_publish_rename_failure_without_winner_raises(tmp_path: Path) -> None:
+    root = tmp_path / "cache"
+    build = tmp_path / "work"
+    _make_built_tree(build)
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(Path, "rename", side_effect=OSError("denied")),
+        pytest.raises(OSError),
+    ):
+        bootloader._publish(build, "k" * 16, {})
+
+
+# ----------------------------------------------------------------- install
+
+
+def test_install_into_build_replaces_stale_subbuild(tmp_path: Path) -> None:
+    entry = tmp_path / "entry"
+    entry.mkdir()
+    # No .map: an optional output missing from the entry is skipped.
+    (entry / "bootloader.bin").write_bytes(b"bin")
+    (entry / "bootloader.elf").write_bytes(b"elf")
+    dest = CORE.relative_build_path("build", "bootloader")
+    dest.mkdir(parents=True)
+    (dest / "CMakeCache.txt").write_text("stale in-tree sub-build")
+    (dest / "old.obj").write_bytes(b"x")
+    bootloader._install_into_build(entry)
+    assert (dest / "bootloader.bin").read_bytes() == b"bin"
+    assert not (dest / "CMakeCache.txt").exists()
+    assert not (dest / "old.obj").exists()
+    assert not (dest / "bootloader.map").exists()
+
+
+# -------------------------------------------------------------- size check
+
+
+def test_size_check_passes_and_logs(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    bin_path = tmp_path / "bootloader.bin"
+    bin_path.write_bytes(b"\xe9" * 0x6620)
+    config = {"BOOTLOADER_OFFSET_IN_FLASH": 0x1000, "PARTITION_TABLE_OFFSET": 0x8000}
+    with caplog.at_level("INFO"):
+        bootloader._check_bootloader_size(bin_path, config)
+    assert "0x9e0 bytes (9%) free" in caplog.text
+
+
+def test_size_check_overflow_raises(tmp_path: Path) -> None:
+    bin_path = tmp_path / "bootloader.bin"
+    bin_path.write_bytes(b"\xe9" * 0x7100)
+    config = {"BOOTLOADER_OFFSET_IN_FLASH": 0x1000, "PARTITION_TABLE_OFFSET": 0x8000}
+    with pytest.raises(EsphomeError, match="overflows by 0x100 bytes"):
+        bootloader._check_bootloader_size(bin_path, config)
+
+
+# ------------------------------------------------------------ orchestration
+
+
+def test_ensure_fails_soft_without_configure_outputs() -> None:
+    """No app config or compiler id means fall back, not crash."""
+    assert bootloader.ensure_cached_bootloader() == 1
+
+
+def _orchestration_env(tmp_path: Path):
+    """Patches shared by the ensure_cached_bootloader tests."""
+    root = tmp_path / "cache-root"
+    build = CORE.relative_build_path("build")
+    build.mkdir(parents=True, exist_ok=True)
+    (build / "config").mkdir(exist_ok=True)
+    (build / "config" / "sdkconfig.json").write_text(
+        json.dumps(
+            {
+                "A": "1",
+                "BOOTLOADER_OFFSET_IN_FLASH": 0x1000,
+                "PARTITION_TABLE_OFFSET": 0x8000,
+            }
+        )
+    )
+    return root
+
+
+def test_ensure_cache_hit_installs_without_building(tmp_path: Path) -> None:
+    root = _orchestration_env(tmp_path)
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"),
+        patch.object(bootloader, "_compute_key", return_value="deadbeefdeadbeef"),
+        patch.object(bootloader, "_build_standalone") as mock_build,
+    ):
+        (root / "deadbeefdeadbeef").mkdir(parents=True)
+        for name in bootloader._OUTPUTS:
+            (root / "deadbeefdeadbeef" / name).write_bytes(b"\xe9" * 64)
+        (root / "config_names.json").write_text('["A"]')
+        assert bootloader.ensure_cached_bootloader() == 0
+    mock_build.assert_not_called()
+    installed = CORE.relative_build_path("build", "bootloader", "bootloader.bin")
+    assert installed.read_bytes() == b"\xe9" * 64
+
+
+def test_ensure_cache_miss_builds_and_publishes(tmp_path: Path) -> None:
+    """Known names but no matching entry: build, harvest, publish."""
+    root = _orchestration_env(tmp_path)
+    root.mkdir(parents=True)
+    (root / "config_names.json").write_text('["A"]')
+
+    def fake_build(build_dir: Path, verbose: bool) -> int:
+        _make_built_tree(build_dir)
+        return 0
+
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"),
+        patch.object(bootloader, "_compute_key", return_value="deadbeefdeadbeef"),
+        patch.object(bootloader, "_build_standalone", side_effect=fake_build),
+    ):
+        assert bootloader.ensure_cached_bootloader() == 0
+    # Entry published under the key, names harvested, work dir removed.
+    assert (root / "deadbeefdeadbeef" / "bootloader.bin").is_file()
+    with patch.object(bootloader, "_cache_root", return_value=root):
+        assert "A" in bootloader._load_config_names()
+    assert not list(root.glob(".build-*"))
+    installed = CORE.relative_build_path("build", "bootloader", "bootloader.bin")
+    assert installed.is_file()
+
+
+def test_ensure_returns_build_failure(tmp_path: Path) -> None:
+    root = _orchestration_env(tmp_path)
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"),
+        patch.object(bootloader, "_build_standalone", return_value=2),
+    ):
+        assert bootloader.ensure_cached_bootloader() == 2
+    assert not list(root.glob(".build-*"))
+
+
+def test_ensure_soft_fails_when_build_yields_no_config(tmp_path: Path) -> None:
+    root = _orchestration_env(tmp_path)
+
+    def build_without_config(build_dir: Path, verbose: bool) -> int:
+        build_dir.mkdir(parents=True, exist_ok=True)
+        return 0
+
+    with (
+        patch.object(bootloader, "_cache_root", return_value=root),
+        patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"),
+        patch.object(bootloader, "_build_standalone", side_effect=build_without_config),
+    ):
+        assert bootloader.ensure_cached_bootloader() == 1
+
+
+def test_prune_removes_only_old_work_dirs(tmp_path: Path) -> None:
+    """A concurrent build's fresh work dir must survive the prune."""
+    old = tmp_path / ".build-1"
+    fresh = tmp_path / ".build-2"
+    old.mkdir()
+    fresh.mkdir()
+    os.utime(old, (0, 0))
+    with patch.object(bootloader, "_cache_root", return_value=tmp_path):
+        bootloader._prune_stale_dirs()
+    assert not old.exists()
+    assert fresh.exists()
+
+
+def test_prune_swallows_cache_root_errors() -> None:
+    """Housekeeping never fails a build."""
+    with patch.object(bootloader, "_cache_root", side_effect=OSError("gone")):
+        bootloader._prune_stale_dirs()
+
+
+# --------------------------------------------------------- flash injection
+
+
+def _flash_build(tmp_path: Path, with_bin: bool = True, offset: int = 0x1000) -> Path:
+    build = tmp_path / "build"
+    (build / "config").mkdir(parents=True)
+    (build / "config" / "sdkconfig.json").write_text(
+        json.dumps({"BOOTLOADER_OFFSET_IN_FLASH": offset})
+    )
+    if with_bin:
+        (build / "bootloader").mkdir()
+        (build / "bootloader" / "bootloader.bin").write_bytes(b"\xe9")
+    return build
+
+
+def test_inject_adds_cached_bootloader(tmp_path: Path) -> None:
+    build = _flash_build(tmp_path)
+    flash_data = {"flash_files": {"0x10000": "dev.bin"}}
+    bootloader.inject_bootloader_flash_file(flash_data, build)
+    assert flash_data["flash_files"]["0x1000"] == "bootloader/bootloader.bin"
+
+
+def test_inject_uses_configured_offset(tmp_path: Path) -> None:
+    build = _flash_build(tmp_path, offset=0)
+    flash_data: dict = {}
+    bootloader.inject_bootloader_flash_file(flash_data, build)
+    assert flash_data["flash_files"] == {"0x0": "bootloader/bootloader.bin"}
+
+
+def test_inject_noop_when_idf_wrote_the_entry(tmp_path: Path) -> None:
+    """Stock and bypass trees already carry a bootloader entry."""
+    build = _flash_build(tmp_path)
+    flash_data = {"flash_files": {"0x1000": "bootloader/bootloader.bin"}}
+    bootloader.inject_bootloader_flash_file(flash_data, build)
+    assert flash_data["flash_files"] == {"0x1000": "bootloader/bootloader.bin"}
+
+
+def test_inject_noop_without_bin_or_config(tmp_path: Path) -> None:
+    flash_data: dict = {"flash_files": {}}
+    bootloader.inject_bootloader_flash_file(
+        flash_data, _flash_build(tmp_path / "a", with_bin=False)
+    )
+    no_config = tmp_path / "b" / "build"
+    (no_config / "bootloader").mkdir(parents=True)
+    (no_config / "bootloader" / "bootloader.bin").write_bytes(b"\xe9")
+    bootloader.inject_bootloader_flash_file(flash_data, no_config)
+    assert flash_data == {"flash_files": {}}

tests/unit_tests/test_espidf_toolchain.py

@@ -12,7 +12,12 @@ from unittest.mock import call, patch
 
 import pytest
 
-from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT
+from esphome.components.esp32.const import (
+    KEY_ESP32,
+    KEY_FLASH_SIZE,
+    KEY_IDF_VERSION,
+    KEY_VARIANT,
+)
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_COMPILE_PROCESS_LIMIT,
@@ -118,6 +123,7 @@ def _up_to_date_compile(ninja_side_effect=None) -> Iterator[tuple]:
     with (
         patch.object(toolchain, "need_reconfigure", return_value=False),
         patch.object(toolchain, "_cache_entries_changed", return_value=False),
+        patch.object(toolchain, "_use_cached_bootloader", return_value=False),
         patch.object(
             toolchain, "_run_ninja", return_value=0, side_effect=ninja_side_effect
         ) as mock_ninja,
@@ -710,6 +716,7 @@ def _fake_tools(env: dict[str, str] | None = None) -> Iterator:
             return_value={"PATH": "/bin", "IDF_CCACHE_ENABLE": "0", **(env or {})},
         ),
         patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
+        patch.object(toolchain, "_use_cached_bootloader", return_value=False),
         patch.object(toolchain, "run_build_tool", return_value=0) as mock_run,
     ):
         yield mock_run
@@ -741,6 +748,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None:
         "-DESP_PLATFORM=1",
         f"-DSDKCONFIG={sdkconfig}",
         "-DCCACHE_ENABLE=0",
+        "-DESPHOME_USE_CACHED_BOOTLOADER=0",
         project,
     ]
     kwargs = mock_run.call_args.kwargs
@@ -775,6 +783,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py(
         "-DPYTHON=/tools/python",
         "-DESP_PLATFORM=1",
         f"-DCCACHE_ENABLE={expected}",
+        "-DESPHOME_USE_CACHED_BOOTLOADER=0",
         str(build_dir.parent),
     ]
 
@@ -786,7 +795,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None:
         assert toolchain.run_reconfigure(verbose=True) == 0
     cmd = mock_run.call_args.args[0]
     assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd)
-    assert cmd[-2] == "-DCCACHE_ENABLE=0"
+    assert cmd[-3] == "-DCCACHE_ENABLE=0"
+    assert cmd[-2] == "-DESPHOME_USE_CACHED_BOOTLOADER=0"
     assert mock_run.call_args.kwargs["filter_lines"] is None
 
 
@@ -853,6 +863,7 @@ _CONFIGURED = (
     "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n"
     "PYTHON:UNINITIALIZED=/tools/python\n"
     "ESP_PLATFORM:UNINITIALIZED=1\n"
+    "ESPHOME_USE_CACHED_BOOTLOADER:UNINITIALIZED=0\n"
 )
 
 
@@ -1104,3 +1115,114 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N
     with patch.object(toolchain, "_get_cmake_output", return_value=output):
         assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF"
         assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None
+
+
+def test_use_cached_bootloader_fallback_wins(setup_core: Path) -> None:
+    """A mid-run fallback pins the rest of the run to the in-tree build."""
+    toolchain._cache().bootloader_fallback = True
+    assert toolchain._use_cached_bootloader() is False
+
+
+def test_use_cached_bootloader_delegates_to_predicate(setup_core: Path) -> None:
+    from esphome.espidf import bootloader
+
+    with patch.object(bootloader, "bootloader_cache_enabled", return_value=True):
+        assert toolchain._use_cached_bootloader() is True
+
+
+def test_configure_defines_follow_bootloader_mode(setup_core: Path) -> None:
+    """The define always rides along, so a mode flip reconfigures."""
+    with (
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(toolchain, "_cache_entries", return_value={}),
+    ):
+        with patch.object(toolchain, "_use_cached_bootloader", return_value=True):
+            on = toolchain._configure_defines()
+        with patch.object(toolchain, "_use_cached_bootloader", return_value=False):
+            off = toolchain._configure_defines()
+    assert on["ESPHOME_USE_CACHED_BOOTLOADER"] == "1"
+    assert off["ESPHOME_USE_CACHED_BOOTLOADER"] == "0"
+
+
+def test_run_compile_installs_cached_bootloader(setup_core: Path) -> None:
+    from esphome.espidf import bootloader
+
+    _setup_build(setup_core)
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=False),
+        patch.object(toolchain, "_cache_entries_changed", return_value=False),
+        patch.object(toolchain, "_use_cached_bootloader", return_value=True),
+        patch.object(
+            bootloader, "ensure_cached_bootloader", return_value=0
+        ) as mock_ensure,
+        patch.object(toolchain, "run_reconfigure") as mock_reconfigure,
+        patch.object(toolchain, "_run_ninja", return_value=0),
+        patch.object(toolchain, "print_summary"),
+    ):
+        assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
+    mock_ensure.assert_called_once_with(False)
+    mock_reconfigure.assert_not_called()
+
+
+@pytest.mark.parametrize("reconfigure_rc", [0, 5])
+def test_run_compile_falls_back_when_bootloader_cache_fails(
+    setup_core: Path, reconfigure_rc: int
+) -> None:
+    """A cache failure flips the run to the stock in-tree build."""
+    from esphome.espidf import bootloader
+
+    _setup_build(setup_core)
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=False),
+        patch.object(toolchain, "_cache_entries_changed", return_value=False),
+        patch.object(toolchain, "_use_cached_bootloader", return_value=True),
+        patch.object(bootloader, "ensure_cached_bootloader", return_value=1),
+        patch.object(
+            toolchain, "run_reconfigure", return_value=reconfigure_rc
+        ) as mock_reconfigure,
+        patch.object(toolchain, "_run_ninja", return_value=0) as mock_ninja,
+        patch.object(toolchain, "print_summary"),
+    ):
+        rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
+    assert rc == reconfigure_rc
+    assert toolchain._cache().bootloader_fallback is True
+    mock_reconfigure.assert_called_once_with(False)
+    assert mock_ninja.called is (reconfigure_rc == 0)
+
+
+def test_create_factory_bin_merges_cached_bootloader(setup_core: Path) -> None:
+    """The cached bootloader joins the merge at its flash offset even
+    though IDF wrote no bootloader entry into flasher_args.json."""
+    _setup_build(setup_core)
+    build = CORE.relative_build_path("build")
+    (build / "bootloader").mkdir(parents=True)
+    (build / "bootloader" / "bootloader.bin").write_bytes(b"\xe9")
+    (build / "config").mkdir()
+    (build / "config" / "sdkconfig.json").write_text(
+        json.dumps({"BOOTLOADER_OFFSET_IN_FLASH": 0x1000})
+    )
+    (build / "test.bin").write_bytes(b"a")
+    (build / "flasher_args.json").write_text(
+        json.dumps(
+            {
+                "flash_files": {"0x10000": "test.bin"},
+                "extra_esptool_args": {"chip": "esp32"},
+            }
+        )
+    )
+    CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB"
+    with (
+        patch.object(toolchain, "_get_idf_env", return_value={}),
+        patch.object(toolchain, "_get_idf_tool", return_value="/py"),
+        patch.object(
+            toolchain.subprocess,
+            "run",
+            return_value=subprocess.CompletedProcess([], 0, "", ""),
+        ) as mock_run,
+    ):
+        assert toolchain.create_factory_bin() is True
+    cmd = mock_run.call_args.args[0]
+    offset_index = cmd.index("0x1000")
+    assert cmd[offset_index + 1] == str(build / "bootloader" / "bootloader.bin")
+    # Sections stay sorted by address; the bootloader comes first.
+    assert offset_index < cmd.index("0x10000")