From 2a45f5de3afe36f17bc3949bb48b1375d5c4f2d1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:04:15 +0200 Subject: [PATCH] [espidf] Cache the bootloader across builds --- esphome/build_gen/espidf.py | 46 ++ esphome/espidf/bootloader.py | 415 +++++++++++ esphome/espidf/toolchain.py | 29 + script/check_idf_py_equivalence.py | 24 +- tests/script/test_check_idf_py_equivalence.py | 48 +- tests/unit_tests/test_espidf_bootloader.py | 650 ++++++++++++++++++ tests/unit_tests/test_espidf_toolchain.py | 126 +++- 7 files changed, 1325 insertions(+), 13 deletions(-) create mode 100644 esphome/espidf/bootloader.py create mode 100644 tests/unit_tests/test_espidf_bootloader.py diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 9f48013a4e8..ac9a22b18a2 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -80,6 +80,51 @@ def _cmake_quote(value: str) -> str: return f'"{escaped}"' +# CONFIG_APP_BUILD_BOOTLOADER is a hidden Kconfig option force-selected by +# APP_BUILD_TYPE_APP_2NDBOOT, so sdkconfig cannot turn it off. Clearing it at +# the CMake level puts the build into the same first-class state IDF's RAM-app +# build type uses: the bootloader guards return early, esptool_py drops the +# bootloader flash entry, and nothing else changes (sdkconfig.h still says 1). +# The macro below is IDF's __build_process_project_includes (tools/cmake/ +# build.cmake, identical in all supported IDF versions) with one added set(); +# esphome.espidf.bootloader compares the live macro against this copy and +# falls back to the in-tree bootloader build if an IDF update changes it. +IDF_BOOTLOADER_OVERRIDE = """\ +# ESPHome: with ESPHOME_USE_CACHED_BOOTLOADER the bootloader comes from a +# machine-global cache instead of the in-tree ExternalProject. This is IDF's +# own __build_process_project_includes macro with one added set(); see +# esphome/espidf/bootloader.py. +if(ESPHOME_USE_CACHED_BOOTLOADER) + macro(__build_process_project_includes) + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + # ESPHome: the bootloader comes from the machine-global cache + set(CONFIG_APP_BUILD_BOOTLOADER "") + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() + endmacro() +endif() +""" + +# The one line the override adds to IDF's macro; the staleness tripwire in +# esphome.espidf.bootloader strips it before comparing with the live macro. +BOOTLOADER_OVERRIDE_ADDED_LINE = 'set(CONFIG_APP_BUILD_BOOTLOADER "")' + + def get_project_cmakelists( minimal: bool = False, builtin_components: list[str] | None = None ) -> str: @@ -202,6 +247,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{IDF_BOOTLOADER_OVERRIDE} {cpp_standard_options} {cxx_compile_options} diff --git a/esphome/espidf/bootloader.py b/esphome/espidf/bootloader.py new file mode 100644 index 00000000000..b893d18b602 --- /dev/null +++ b/esphome/espidf/bootloader.py @@ -0,0 +1,415 @@ +"""Machine-global bootloader cache for the native ESP-IDF toolchain. + +The bootloader is its own CMake project that depends only on the IDF version, +target, compiler and the config subset its reduced Kconfig universe defines, +so one build serves every device that shares those. With +``CONFIG_APP_REPRODUCIBLE_BUILD`` (always on) the result is byte identical to +the in-tree ExternalProject build. Builds that configure signing, secure boot +or flash encryption never use the cache: their bootloader can depend on key +file contents, and they keep the stock in-tree path. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +from pathlib import Path +import re +import shutil +import time + +from esphome.build_helpers.ccache import parse_enable_env +from esphome.build_helpers.tool_runner import run_build_tool +from esphome.const import KEY_ESP32, KEY_VARIANT +from esphome.core import CORE, EsphomeError +from esphome.espidf import toolchain, variant_to_idf_target +from esphome.helpers import write_file + +_LOGGER = logging.getLogger(__name__) + +BOOTLOADER_CACHE_ENV = "ESPHOME_BOOTLOADER_CACHE" +# Bump on any change to how cached bootloaders are built or stored; old +# entries then miss instead of being served stale. +_CACHE_SCHEMA = "1" +_SECURE_OPTION = re.compile(r"CONFIG_(SECURE_|FLASH_ENCRYPTION)") +_MACRO = re.compile( + r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL +) +# Marks a config name the app's sdkconfig does not define; distinct from +# every real value. +_ABSENT = "\x00absent" +_OUTPUTS = ("bootloader.bin", "bootloader.elf", "bootloader.map") + + +def bootloader_cache_enabled() -> bool: + """Whether this build may take its bootloader from the cache. + + False keeps IDF's in-tree bootloader build, exactly as before. + """ + cache = toolchain._cache() + if cache.bootloader_enabled is None: + cache.bootloader_enabled = _compute_enabled() + return cache.bootloader_enabled + + +def _compute_enabled() -> bool: + if parse_enable_env(BOOTLOADER_CACHE_ENV) is False: + return False + if "IDF_PATH" in os.environ: + # An unmanaged IDF checkout can change under the cache. + return False + try: + # The .esphomeinternal snapshot holds exactly what ESPHome and the + # user configured; the live sdkconfig is rewritten by kconfgen with + # every resolved option, including always-on *_SUPPORTED constants. + snapshot = CORE.relative_build_path(f"sdkconfig.{CORE.name}.esphomeinternal") + if not snapshot.is_file() or _has_secure_options(snapshot): + return False + return idf_macro_matches() + except (OSError, KeyError, ValueError, EsphomeError) as err: + _LOGGER.debug("Bootloader cache disabled: %s", err) + return False + + +def _has_secure_options(sdkconfig_path: Path) -> bool: + """Whether any signing, secure boot or encryption option is enabled. + + Reads ESPHome's sdkconfig snapshot, so raw ``sdkconfig_options`` count. + A disabled value (``n``/``0``/empty) is the compiled-out state ESPHome + writes by default and is safe to cache; anything else bypasses. + """ + for line in sdkconfig_path.read_text(encoding="utf-8").splitlines(): + name, _, value = line.partition("=") + if _SECURE_OPTION.match(name.strip()) and value.strip().strip('"') not in ( + "n", + "0", + "", + ): + return True + return False + + +def _normalized_macro(text: str) -> list[str] | None: + """The macro body as comment-free, whitespace-collapsed lines.""" + if (match := _MACRO.search(text)) is None: + return None + return [ + re.sub(r"\s+", " ", line) + for raw in match.group(1).splitlines() + if (line := raw.split("#", 1)[0].strip()) + ] + + +def idf_macro_matches() -> bool: + """Whether IDF's macro still matches the copy the override replays. + + A mismatch after an IDF update silently falls back to the in-tree + bootloader build; script/check_idf_py_equivalence.py reports it loudly. + """ + from esphome.build_gen.espidf import ( + BOOTLOADER_OVERRIDE_ADDED_LINE, + IDF_BOOTLOADER_OVERRIDE, + ) + + expected = _normalized_macro(IDF_BOOTLOADER_OVERRIDE) + expected.remove(BOOTLOADER_OVERRIDE_ADDED_LINE) + build_cmake = toolchain._get_idf_path() / "tools" / "cmake" / "build.cmake" + live = _normalized_macro(build_cmake.read_text(encoding="utf-8")) + return live == expected + + +def _idf_target() -> str: + return variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) + + +def _cache_root() -> Path: + from esphome.espidf.framework import get_idf_tools_path + + return ( + get_idf_tools_path() + / "bootloaders" + / toolchain._get_core_framework_version() + / _idf_target() + ) + + +def _load_config_names() -> list[str] | None: + """The known union of config option names the bootloader consumes.""" + try: + names = json.loads( + (_cache_root() / "config_names.json").read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return None + if isinstance(names, list) and names and all(isinstance(n, str) for n in names): + return names + return None + + +def _merge_config_names(names) -> list[str]: + """Union-merge harvested names; the list only ever grows.""" + merged = sorted(set(_load_config_names() or ()) | set(names)) + write_file( + _cache_root() / "config_names.json", + json.dumps(merged, separators=(",", ":")), + ) + return merged + + +def _load_build_config(build_dir: Path) -> dict | None: + """A build's generated config/sdkconfig.json, or None.""" + try: + return json.loads( + (build_dir / "config" / "sdkconfig.json").read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return None + + +def _compiler_id() -> str | None: + """The app build's C compiler, resolved; the toolchain part of the key. + + IDF's toolchain file sets the compiler, so CMakeCache.txt has no plain + CMAKE_C_COMPILER entry; the compile database names it instead. None + means the cache cannot be trusted for this run and the caller falls + back to the in-tree build. + """ + try: + commands = json.loads( + (toolchain._build_dir() / "compile_commands.json").read_text( + encoding="utf-8" + ) + ) + compiler = commands[0]["command"].split()[0] + except (OSError, ValueError, LookupError): + cache_path = toolchain._build_dir() / "CMakeCache.txt" + if not cache_path.is_file(): + return None + # Same install dir and version as the compiler itself. + compiler = toolchain._parse_cmakecache(cache_path).get("CMAKE_C_COMPILER_AR") + return os.path.realpath(compiler) if compiler else None + + +def _subproject_cmake_args( + cmake: str, python: str, sdkconfig: str, idf_path: str, project_dir: str +) -> list[str]: + """The configure argv, mirroring the ExternalProject in IDF's + components/bootloader/project_include.cmake for the pinned version.""" + from esphome.components.esp32 import idf_version + import esphome.config_validation as cv + + args = [ + cmake, + "-G", + "Ninja", + f"-DSDKCONFIG={sdkconfig}", + f"-DIDF_PATH={idf_path}", + f"-DIDF_TARGET={_idf_target()}", + "-DPYTHON_DEPS_CHECKED=1", + f"-DPYTHON={python}", + ] + if idf_version() >= cv.Version(6, 0, 0): + # 6.x: no BOOTLOADER_EXTRA_COMPONENT_DIRS self-append; IDF_BUILD_V2 + # forwarded (empty: the env opt-in is never set). + args += ["-DEXTRA_COMPONENT_DIRS=", "-DIDF_BUILD_V2="] + else: + args.append(f"-DEXTRA_COMPONENT_DIRS={idf_path}/components/bootloader") + args += [ + f"-DPROJECT_SOURCE_DIR={project_dir}", + "-DIGNORE_EXTRA_COMPONENT=", + f"{idf_path}/components/bootloader/subproject", + ] + return args + + +def _version_stamp() -> str: + """The installed framework's version.txt, part of the key.""" + try: + return ( + (toolchain._get_idf_path() / "version.txt") + .read_text(encoding="utf-8") + .strip() + ) + except OSError: + return "" + + +def _compute_key(names: list[str], app_config: dict, compiler: str) -> str: + """The cache entry name for this build's bootloader inputs.""" + payload = json.dumps( + { + "schema": _CACHE_SCHEMA, + "idf": toolchain._get_core_framework_version(), + "source": toolchain._get_framework_source_override() or "", + "stamp": _version_stamp(), + "target": _idf_target(), + "compiler": compiler, + # Placeholder paths: any change to the invocation shape misses. + "args": _subproject_cmake_args( + "cmake", "@PYTHON@", "@SDKCONFIG@", "@IDF@", "@PROJECT@" + ), + "config": {n: app_config.get(n, _ABSENT) for n in sorted(names)}, + }, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode()).hexdigest()[:16] + + +def _build_standalone(build_dir: Path, verbose: bool) -> int: + """Configure and build the bootloader subproject into ``build_dir``.""" + build_dir.mkdir(parents=True, exist_ok=True) + args = _subproject_cmake_args( + toolchain._get_idf_tool("cmake"), + toolchain._get_idf_tool("python"), + str(CORE.relative_build_path(f"sdkconfig.{CORE.name}")), + str(toolchain._get_idf_path()), + os.path.realpath(CORE.build_path), + ) + filter_lines = None if verbose else toolchain.FILTER_IDF_LINES + for name, cmd in (("cmake", args), ("ninja", [toolchain._get_idf_tool("ninja")])): + log_path = build_dir / "log" / f"{name}_output.log" + rc = run_build_tool( + cmd, + cwd=build_dir, + env=toolchain._tool_env(), + filter_lines=filter_lines, + log_path=log_path, + ) + if rc != 0: + _LOGGER.error("Bootloader %s failed with exit code %d", name, rc) + toolchain._print_hints(log_path) + return rc + return 0 + + +def _publish(build_dir: Path, key: str, meta: dict) -> Path: + """Move the distilled outputs into the cache; losing a race is fine.""" + entry = _cache_root() / key + if (entry / "bootloader.bin").is_file(): + return entry + stage = _cache_root() / f".stage-{key}-{os.getpid()}" + stage.mkdir(parents=True, exist_ok=True) + for name in _OUTPUTS: + shutil.copy2(build_dir / name, stage / name) + shutil.copy2(build_dir / "config" / "sdkconfig.json", stage / "sdkconfig.json") + write_file(stage / "meta.json", json.dumps(meta, indent=2, sort_keys=True)) + try: + stage.rename(entry) + except OSError: + # Another process published the same key first. + shutil.rmtree(stage, ignore_errors=True) + if not (entry / "bootloader.bin").is_file(): + raise + return entry + + +def _install_into_build(entry: Path) -> None: + """Copy the cached outputs to build/bootloader, the path every consumer + (factory image, OTA --bootloader, the CI guard) already reads.""" + dest = toolchain._build_dir() / "bootloader" + if (dest / "CMakeCache.txt").is_file(): + # A leftover in-tree sub-build; its ninja state must not linger. + shutil.rmtree(dest) + dest.mkdir(parents=True, exist_ok=True) + for name in _OUTPUTS: + if (src := entry / name).is_file(): + shutil.copy2(src, dest / name) + + +def _check_bootloader_size(bin_path: Path, app_config: dict) -> None: + """The check_sizes.py bootloader check, without spawning a process.""" + offset = int(app_config.get("BOOTLOADER_OFFSET_IN_FLASH", 0)) + table_offset = int(app_config.get("PARTITION_TABLE_OFFSET", 0x8000)) + size = bin_path.stat().st_size + free = table_offset - offset - size + if free < 0: + raise EsphomeError( + f"Bootloader binary size {size:#x} bytes is too large for " + f"partition table offset {table_offset:#x} (overflows by " + f"{-free:#x} bytes). Increase CONFIG_PARTITION_TABLE_OFFSET." + ) + _LOGGER.info( + "Bootloader binary size %#x bytes. %#x bytes (%d%%) free.", + size, + free, + round(100 * free / (table_offset - offset)), + ) + + +def _prune_stale_dirs() -> None: + """Drop day-old work dirs from crashed builds; live ones are younger.""" + cutoff = time.time() - 86400 + try: + for path in _cache_root().glob(".*"): + if path.is_dir() and path.stat().st_mtime < cutoff: + shutil.rmtree(path, ignore_errors=True) + except OSError: + pass + + +def ensure_cached_bootloader(verbose: bool = False) -> int: + """Put the cached bootloader into build/bootloader, building on a miss. + + Returns 0 on success. A nonzero tool exit code tells the caller to fall + back to the in-tree build; a too-large bootloader raises instead, since + the in-tree build would fail the same way. + """ + app_config = _load_build_config(toolchain._build_dir()) + compiler = _compiler_id() + if app_config is None or compiler is None: + _LOGGER.debug("Bootloader cache unusable: app configure outputs missing") + return 1 + entry = None + if (names := _load_config_names()) is not None: + candidate = _cache_root() / _compute_key(names, app_config, compiler) + if (candidate / "bootloader.bin").is_file(): + entry = candidate + if entry is None: + tmp = _cache_root() / f".build-{os.getpid()}" + try: + if (rc := _build_standalone(tmp, verbose)) != 0: + return rc + if (built_config := _load_build_config(tmp)) is None: + _LOGGER.debug("Bootloader build produced no sdkconfig.json") + return 1 + names = _merge_config_names(built_config) + key = _compute_key(names, app_config, compiler) + entry = _publish( + tmp, + key, + { + "schema": _CACHE_SCHEMA, + "idf": toolchain._get_core_framework_version(), + "target": _idf_target(), + "compiler": compiler, + }, + ) + _LOGGER.info("Cached bootloader %s for later builds", key) + finally: + shutil.rmtree(tmp, ignore_errors=True) + _prune_stale_dirs() + _install_into_build(entry) + _check_bootloader_size( + toolchain._build_dir() / "bootloader" / "bootloader.bin", app_config + ) + return 0 + + +def inject_bootloader_flash_file(flash_data: dict, build_dir: Path) -> None: + """Add the cached bootloader to a flasher_args ``flash_files`` map. + + In cached mode IDF writes no bootloader entry; stock and bypass trees + already carry one, which makes this a no-op there. + """ + flash_files = flash_data.setdefault("flash_files", {}) + if any("bootloader/" in name for name in flash_files.values()): + return + if not (build_dir / "bootloader" / "bootloader.bin").is_file(): + return + if (app_config := _load_build_config(build_dir)) is None: + return + offset = hex(int(app_config.get("BOOTLOADER_OFFSET_IN_FLASH", 0))) + flash_files[offset] = "bootloader/bootloader.bin" diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index d1aeae96031..28d78abbaa7 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -37,6 +37,8 @@ class _CacheData: paths: dict[str, tuple] = field(default_factory=dict) env: dict[str, dict[str, str]] = field(default_factory=dict) cmake_output: dict[Path, str] = field(default_factory=dict) + bootloader_enabled: bool | None = None + bootloader_fallback: bool = False cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict) @@ -282,6 +284,15 @@ def _cache_entries_changed() -> bool: return any(cache.get(k) != v for k, v in _configure_defines().items()) +def _use_cached_bootloader() -> bool: + """Whether this run takes the bootloader from the machine-global cache.""" + if _cache().bootloader_fallback: + return False + from esphome.espidf import bootloader + + return bootloader.bootloader_cache_enabled() + + def _configure_defines() -> dict[str, str]: """Every ``-D`` idf.py passes to cmake, in its order.""" return { @@ -289,6 +300,10 @@ def _configure_defines() -> dict[str, str]: "PYTHON": _get_idf_tool("python"), "ESP_PLATFORM": "1", **_cache_entries(), + # ESPHome's own switch, appended after idf.py's entries. idf.py + # never passes it and cmake keeps the cached value, so an idf.py + # run against the tree stays in the same mode. + "ESPHOME_USE_CACHED_BOOTLOADER": "1" if _use_cached_bootloader() else "0", } @@ -675,6 +690,16 @@ def run_compile(config, verbose: bool) -> int: else: _LOGGER.info("Build configuration is up to date") + if _use_cached_bootloader(): + from esphome.espidf import bootloader + + if bootloader.ensure_cached_bootloader(verbose) != 0: + # The stock in-tree build still works; flip this run over to it. + _LOGGER.warning("Cached bootloader unavailable; building it in-tree") + _cache().bootloader_fallback = True + if (rc := run_reconfigure(verbose)) != 0: + return rc + # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, # so we must patch after it's generated but before linking (same timing @@ -809,6 +834,10 @@ def create_factory_bin() -> bool: _LOGGER.error("Failed to read flasher_args.json: %s", e) return False + from esphome.espidf import bootloader + + bootloader.inject_bootloader_flash_file(flash_data, build_dir) + # Get flash size from config flash_size = CORE.data[KEY_ESP32][KEY_FLASH_SIZE] diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index fa349e223a4..fcd97798890 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -33,9 +33,18 @@ WATCHED = ( ) # Ninja logs whose outputs mean real work when their recorded mtime changes. # The top level re-logs the bootloader step's byproducts on every build, so -# the bootloader is judged by its own sub-build log instead. +# the bootloader is judged by its own sub-build log instead. With the cached +# bootloader there is no sub-build at all; the cached bin is in WATCHED. TOP_NINJA_LOG = "build/.ninja_log" -NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log") + + +def _ninja_logs(build_path: Path) -> list[str]: + logs = [TOP_NINJA_LOG] + if (build_path / "build" / "bootloader" / "build.ninja").is_file(): + logs.append("build/bootloader/.ninja_log") + return logs + + BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/") WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -57,7 +66,7 @@ def _snapshot(build_path: Path, name: str) -> dict[str, str | None]: def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: """(log, output) -> recorded mtime; compaction-safe, unlike a line count.""" mtimes = {} - for name in NINJA_LOGS: + for name in _ninja_logs(build_path): log = build_path / name lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else [] for fields in (line.split("\t") for line in lines if not line.startswith("#")): @@ -71,7 +80,7 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]: """A missing or unparsable ninja log would otherwise compare as unchanged.""" problems = [] - for log in NINJA_LOGS: + for log in _ninja_logs(build_path): if not (build_path / log).is_file(): problems.append(f"missing {log}") elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes): @@ -106,6 +115,13 @@ def check(build_path: Path) -> list[str]: (build_path / "build" / "project_description.json").read_text(encoding="utf-8") ) name, version = _setup_core(build_path, description) + from esphome.espidf import bootloader + + if not bootloader.idf_macro_matches(): + return [ + "IDF changed __build_process_project_includes; update " + "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" + ] env = toolchain._get_idf_env(version) python = toolchain._get_idf_tool("python") idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py" diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index f8135772d0c..d0bc5ca73ca 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -14,7 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script")) import check_idf_py_equivalence as guard # noqa: E402 from esphome.core import CORE # noqa: E402 -from esphome.espidf import toolchain # noqa: E402 +from esphome.espidf import bootloader, toolchain # noqa: E402 @pytest.fixture(autouse=True) @@ -24,12 +24,21 @@ def _reset_core() -> Iterator[None]: CORE.reset() -def _make_tree(tmp_path: Path) -> Path: +BOOTLOADER_LOG = "build/bootloader/.ninja_log" +ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG) + + +def _make_tree(tmp_path: Path, bootloader_subbuild: bool = True) -> Path: + """A fake build tree; the sub-build shape mirrors a bypass/stock build, + without it the cached-bootloader shape (bin present, no sub-build).""" tree = tmp_path / "config" / ".esphome" / "build" / "dev" build = tree / "build" - for name in (*guard.watched("dev"), *guard.NINJA_LOGS): + logs = ALL_LOGS if bootloader_subbuild else (guard.TOP_NINJA_LOG,) + for name in (*guard.watched("dev"), *logs): (tree / name).parent.mkdir(parents=True, exist_ok=True) (tree / name).write_bytes(b"x") + if bootloader_subbuild: + (build / "bootloader" / "build.ninja").write_bytes(b"x") (build / "project_description.json").write_text( json.dumps( { @@ -70,6 +79,7 @@ def _run_check( patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), + patch.object(bootloader, "idf_macro_matches", return_value=True), patch.object(guard.subprocess, "run", side_effect=run), ): return guard.check(tree), calls @@ -150,9 +160,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None: assert len(calls) == 1 -@pytest.mark.parametrize( - "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS] -) +@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS]) def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None: """A moved or renamed output must not compare as unchanged.""" tree = _make_tree(tmp_path) @@ -179,7 +187,7 @@ def test_check_stops_when_the_esphome_baseline_fails( assert calls == [] -@pytest.mark.parametrize("log", guard.NINJA_LOGS) +@pytest.mark.parametrize("log", ALL_LOGS) def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None: """A log format change must not leave the rebuild check with nothing to compare.""" tree = _make_tree(tmp_path) @@ -205,6 +213,32 @@ def test_main_rejects_a_path_that_is_not_a_tree( mock_check.assert_not_called() +def test_check_accepts_a_cached_bootloader_tree(tmp_path: Path) -> None: + """No bootloader sub-build is the cached shape, not a missing input.""" + tree = _make_tree(tmp_path, bootloader_subbuild=False) + problems, calls = _run_check(tree) + assert problems == [] + assert len(calls) == 2 + + +def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None: + """An IDF bump that rewrites the overridden macro must fail CI.""" + tree = _make_tree(tmp_path) + with ( + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(bootloader, "idf_macro_matches", return_value=False), + patch.object(guard.subprocess, "run") as mock_run, + ): + problems = guard.check(tree) + assert problems == [ + "IDF changed __build_process_project_includes; update " + "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" + ] + mock_run.assert_not_called() + + def test_main_without_build_trees( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit_tests/test_espidf_bootloader.py b/tests/unit_tests/test_espidf_bootloader.py new file mode 100644 index 00000000000..cd172c19443 --- /dev/null +++ b/tests/unit_tests/test_espidf_bootloader.py @@ -0,0 +1,650 @@ +"""Tests for esphome.espidf.bootloader.""" + +# pylint: disable=protected-access + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError +from esphome.espidf import bootloader, toolchain + +# The macro body as shipped in build.cmake; byte identical in IDF 5.5.5 and +# 6.1.0, so one fixture covers both supported versions. +IDF_BUILD_CMAKE = """\ +some_other_cmake() + +macro(__build_process_project_includes) + # Include the sdkconfig cmake file, since the following operations require + # knowledge of config values. + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + + # Make each build property available as a read-only variable + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + + # Include each component's project_include.cmake + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) # this is deprecated, users are encouraged to use COMPONENT_DIR; + # retained for compatibility + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() +endmacro() +""" + + +@pytest.fixture(autouse=True) +def _fresh_cache(setup_core: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point CORE at a build tree and reset the per-run toolchain cache.""" + monkeypatch.delenv(bootloader.BOOTLOADER_CACHE_ENV, raising=False) + monkeypatch.delenv("IDF_PATH", raising=False) + CORE.name = "dev" + CORE.build_path = setup_core / "build" / "dev" + CORE.data[KEY_ESP32] = { + KEY_VARIANT: "ESP32", + KEY_IDF_VERSION: cv.Version(5, 5, 5), + } + CORE.data.pop(toolchain.DOMAIN, None) + + +def _write_snapshot(text: str = "") -> Path: + path = CORE.relative_build_path("sdkconfig.dev.esphomeinternal") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + +def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path: + idf = tmp_path / "idf" + (idf / "tools" / "cmake").mkdir(parents=True) + (idf / "tools" / "cmake" / "build.cmake").write_text(text) + return idf + + +# ---------------------------------------------------------------- predicate + + +def test_enabled_env_zero_disables(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(bootloader.BOOTLOADER_CACHE_ENV, "0") + assert bootloader._compute_enabled() is False + + +def test_enabled_user_idf_path_disables(monkeypatch: pytest.MonkeyPatch) -> None: + """An unmanaged IDF checkout can change under the cache.""" + monkeypatch.setenv("IDF_PATH", "/somewhere") + assert bootloader._compute_enabled() is False + + +def test_enabled_missing_snapshot_disables() -> None: + """No snapshot means the config origin is unknown; fail safe.""" + assert bootloader._compute_enabled() is False + + +def test_enabled_secure_option_disables() -> None: + _write_snapshot("CONFIG_SECURE_BOOT=y\n") + assert bootloader._compute_enabled() is False + + +def test_enabled_clean_snapshot_checks_macro(tmp_path: Path) -> None: + _write_snapshot("CONFIG_FOO=y\n") + idf = _write_idf_build_cmake(tmp_path) + with patch.object(toolchain, "_get_idf_path", return_value=idf): + assert bootloader._compute_enabled() is True + + +def test_enabled_swallows_errors_as_disabled(tmp_path: Path) -> None: + """Any failure while deciding must read as disabled, never raise.""" + _write_snapshot("CONFIG_FOO=y\n") + with patch.object(toolchain, "_get_idf_path", side_effect=OSError("boom")): + assert bootloader._compute_enabled() is False + + +def test_bootloader_cache_enabled_is_cached_per_run() -> None: + with patch.object( + bootloader, "_compute_enabled", return_value=True + ) as mock_compute: + assert bootloader.bootloader_cache_enabled() is True + assert bootloader.bootloader_cache_enabled() is True + mock_compute.assert_called_once_with() + + +@pytest.mark.parametrize( + ("text", "secure"), + [ + ("CONFIG_SECURE_BOOT=y\n", True), + ("CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT=y\n", True), + ("CONFIG_FLASH_ENCRYPTION_ENABLED=y\n", True), + ('CONFIG_SECURE_BOOT_SIGNING_KEY="key.pem"\n', True), + ("CONFIG_SECURE_BOOT=n\n", False), + ("CONFIG_SECURE_BOOT=0\n", False), + ('CONFIG_SECURE_BOOT_SIGNING_KEY=""\n', False), + ("# CONFIG_SECURE_BOOT is not set\n", False), + ("CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y\n", False), + ("", False), + ], + ids=[ + "secure-boot", + "signed-apps", + "flash-encryption", + "signing-key", + "value-n", + "value-0", + "empty-string", + "comment-line", + "unrelated", + "empty-file", + ], +) +def test_has_secure_options(tmp_path: Path, text: str, secure: bool) -> None: + """Only an enabled secure option bypasses; disabled values are the + compiled-out state ESPHome writes by default.""" + path = tmp_path / "sdkconfig" + path.write_text(text) + assert bootloader._has_secure_options(path) is secure + + +# ------------------------------------------------------------ macro tripwire + + +def test_normalized_macro_strips_comments_and_whitespace() -> None: + text = "macro(__build_process_project_includes)\n a( b ) # tail\n\n # only\n c(d)\nendmacro()" + assert bootloader._normalized_macro(text) == ["a( b )", "c(d)"] + + +def test_normalized_macro_none_without_macro() -> None: + assert bootloader._normalized_macro("nothing here") is None + + +def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None: + """The override's embedded copy must equal what build.cmake ships.""" + idf = _write_idf_build_cmake(tmp_path) + with patch.object(toolchain, "_get_idf_path", return_value=idf): + assert bootloader.idf_macro_matches() is True + + +def test_idf_macro_mismatch_detected(tmp_path: Path) -> None: + changed = IDF_BUILD_CMAKE.replace( + "include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)" + ) + idf = _write_idf_build_cmake(tmp_path, changed) + with patch.object(toolchain, "_get_idf_path", return_value=idf): + assert bootloader.idf_macro_matches() is False + + +# ------------------------------------------------------------------ cache key + + +def _key(names: list[str], config: dict, compiler: str = "/tc/gcc") -> str: + with ( + patch.object(toolchain, "_get_core_framework_version", return_value="5.5.5"), + patch.object(toolchain, "_get_framework_source_override", return_value=None), + patch.object(bootloader, "_version_stamp", return_value="v5.5.5"), + ): + return bootloader._compute_key(names, config, compiler) + + +def test_key_is_stable_and_name_order_independent() -> None: + config = {"A": "1", "B": True} + key = _key(["A", "B"], config) + assert key == _key(["B", "A"], config) + assert len(key) == 16 + + +def test_key_distinguishes_absent_from_empty() -> None: + """An option missing from the app config is not the same as ''.""" + assert _key(["A"], {"A": ""}) != _key(["A"], {}) + + +def test_key_changes_with_each_input() -> None: + base = _key(["A"], {"A": "1"}) + assert _key(["A"], {"A": "2"}) != base + assert _key(["A"], {"A": "1"}, compiler="/other/gcc") != base + with ( + patch.object(toolchain, "_get_core_framework_version", return_value="6.1.0"), + patch.object(toolchain, "_get_framework_source_override", return_value=None), + patch.object(bootloader, "_version_stamp", return_value="v5.5.5"), + ): + CORE.data[KEY_ESP32][KEY_IDF_VERSION] = cv.Version(6, 1, 0) + assert bootloader._compute_key(["A"], {"A": "1"}, "/tc/gcc") != base + + +def test_key_ignores_unlisted_config_values() -> None: + """Only the names the bootloader consumes participate.""" + assert _key(["A"], {"A": "1", "Z": "app-only"}) == _key(["A"], {"A": "1"}) + + +def test_version_stamp_reads_version_txt(tmp_path: Path) -> None: + (tmp_path / "version.txt").write_text("v5.5.5\n") + with patch.object(toolchain, "_get_idf_path", return_value=tmp_path): + assert bootloader._version_stamp() == "v5.5.5" + + +def test_version_stamp_missing_file(tmp_path: Path) -> None: + with patch.object(toolchain, "_get_idf_path", return_value=tmp_path): + assert bootloader._version_stamp() == "" + + +def test_cache_root_layout(tmp_path: Path) -> None: + """Entries live under the registered IDF tools cache, per version and + target, so clean-all removes them with the toolchains.""" + from esphome.espidf import framework + + with ( + patch.object(framework, "get_idf_tools_path", return_value=tmp_path), + patch.object(toolchain, "_get_core_framework_version", return_value="5.5.5"), + ): + assert bootloader._cache_root() == tmp_path / "bootloaders" / "5.5.5" / "esp32" + + +# ----------------------------------------------------------- config names + + +def test_merge_config_names_unions_and_sorts(tmp_path: Path) -> None: + root = tmp_path / "cache" + with patch.object(bootloader, "_cache_root", return_value=root): + assert bootloader._merge_config_names(["B", "A"]) == ["A", "B"] + assert bootloader._merge_config_names(["C", "A"]) == ["A", "B", "C"] + assert bootloader._load_config_names() == ["A", "B", "C"] + + +@pytest.mark.parametrize( + "content", + [None, "not json", "[]", '["a", 1]', '"str"'], + ids=["missing", "corrupt", "empty", "mixed-types", "not-a-list"], +) +def test_load_config_names_rejects_bad_files( + tmp_path: Path, content: str | None +) -> None: + if content is not None: + (tmp_path / "config_names.json").write_text(content) + with patch.object(bootloader, "_cache_root", return_value=tmp_path): + assert bootloader._load_config_names() is None + + +def test_load_build_config(tmp_path: Path) -> None: + assert bootloader._load_build_config(tmp_path) is None + (tmp_path / "config").mkdir() + (tmp_path / "config" / "sdkconfig.json").write_text('{"A": 1}') + assert bootloader._load_build_config(tmp_path) == {"A": 1} + + +# ------------------------------------------------------------- compiler id + + +def test_compiler_id_from_compile_commands(tmp_path: Path) -> None: + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "compile_commands.json").write_text( + json.dumps([{"command": "/tools/xtensa-esp32-elf-gcc -c a.c"}]) + ) + assert bootloader._compiler_id() == "/tools/xtensa-esp32-elf-gcc" + + +def test_compiler_id_falls_back_to_cmakecache(tmp_path: Path) -> None: + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + "CMAKE_C_COMPILER_AR:FILEPATH=/tools/xtensa-esp32-elf-gcc-ar\n" + ) + assert bootloader._compiler_id() == "/tools/xtensa-esp32-elf-gcc-ar" + + +def test_compiler_id_none_without_configure_outputs() -> None: + assert bootloader._compiler_id() is None + + +# ------------------------------------------------------------- cmake args + + +def test_subproject_args_for_idf_5() -> None: + args = bootloader._subproject_cmake_args("cmake", "/py", "/sdk", "/idf", "/proj") + assert args == [ + "cmake", + "-G", + "Ninja", + "-DSDKCONFIG=/sdk", + "-DIDF_PATH=/idf", + "-DIDF_TARGET=esp32", + "-DPYTHON_DEPS_CHECKED=1", + "-DPYTHON=/py", + "-DEXTRA_COMPONENT_DIRS=/idf/components/bootloader", + "-DPROJECT_SOURCE_DIR=/proj", + "-DIGNORE_EXTRA_COMPONENT=", + "/idf/components/bootloader/subproject", + ] + + +def test_subproject_args_for_idf_6() -> None: + """6.x dropped the bootloader dir self-append and forwards IDF_BUILD_V2.""" + CORE.data[KEY_ESP32][KEY_IDF_VERSION] = cv.Version(6, 1, 0) + args = bootloader._subproject_cmake_args("cmake", "/py", "/sdk", "/idf", "/proj") + assert "-DEXTRA_COMPONENT_DIRS=" in args + assert "-DIDF_BUILD_V2=" in args + assert "-DEXTRA_COMPONENT_DIRS=/idf/components/bootloader" not in args + assert args[-1] == "/idf/components/bootloader/subproject" + + +# -------------------------------------------------------- standalone build + + +def test_build_standalone_runs_cmake_then_ninja(tmp_path: Path) -> None: + build_dir = tmp_path / "work" + with ( + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(toolchain, "_tool_env", return_value={"A": "1"}), + patch.object(bootloader, "run_build_tool", return_value=0) as mock_run, + ): + assert bootloader._build_standalone(build_dir, verbose=True) == 0 + cmake_call, ninja_call = mock_run.call_args_list + assert cmake_call.args[0][0] == "/tools/cmake" + assert cmake_call.args[0][-1] == "/idf/components/bootloader/subproject" + assert cmake_call.kwargs["cwd"] == build_dir + assert cmake_call.kwargs["filter_lines"] is None # verbose + assert cmake_call.kwargs["log_path"] == build_dir / "log" / "cmake_output.log" + assert ninja_call.args[0] == ["/tools/ninja"] + + +def test_build_standalone_failure_prints_hints(tmp_path: Path) -> None: + build_dir = tmp_path / "work" + with ( + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(toolchain, "_tool_env", return_value={}), + patch.object(bootloader, "run_build_tool", return_value=2) as mock_run, + patch.object(toolchain, "_print_hints") as mock_hints, + ): + assert bootloader._build_standalone(build_dir, verbose=False) == 2 + # Failed at cmake; ninja never ran, the hint scanner got the log. + assert len(mock_run.call_args_list) == 1 + assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES + mock_hints.assert_called_once_with(build_dir / "log" / "cmake_output.log") + + +# ----------------------------------------------------------------- publish + + +def _make_built_tree(build_dir: Path) -> None: + build_dir.mkdir(parents=True, exist_ok=True) + for name in bootloader._OUTPUTS: + (build_dir / name).write_bytes(name.encode()) + (build_dir / "config").mkdir(exist_ok=True) + (build_dir / "config" / "sdkconfig.json").write_text('{"A": "1"}') + + +def test_publish_creates_entry(tmp_path: Path) -> None: + root = tmp_path / "cache" + build = tmp_path / "work" + _make_built_tree(build) + with patch.object(bootloader, "_cache_root", return_value=root): + entry = bootloader._publish(build, "k" * 16, {"idf": "5.5.5"}) + assert entry == root / ("k" * 16) + assert (entry / "bootloader.bin").read_bytes() == b"bootloader.bin" + assert json.loads((entry / "meta.json").read_text()) == {"idf": "5.5.5"} + assert not list(root.glob(".stage-*")) + + +def test_publish_existing_entry_short_circuits(tmp_path: Path) -> None: + root = tmp_path / "cache" + entry = root / ("k" * 16) + entry.mkdir(parents=True) + (entry / "bootloader.bin").write_bytes(b"winner") + with patch.object(bootloader, "_cache_root", return_value=root): + assert bootloader._publish(tmp_path / "work", "k" * 16, {}) == entry + assert (entry / "bootloader.bin").read_bytes() == b"winner" + + +def test_publish_lost_race_reuses_winner(tmp_path: Path) -> None: + """A rename beaten by another process keeps that process's entry.""" + root = tmp_path / "cache" + build = tmp_path / "work" + _make_built_tree(build) + entry = root / ("k" * 16) + + def rename_raises(self: Path, target: Path) -> None: + entry.mkdir(parents=True) + (entry / "bootloader.bin").write_bytes(b"winner") + raise OSError("directory not empty") + + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(Path, "rename", rename_raises), + ): + assert bootloader._publish(build, "k" * 16, {}) == entry + assert (entry / "bootloader.bin").read_bytes() == b"winner" + assert not list(root.glob(".stage-*")) + + +def test_publish_rename_failure_without_winner_raises(tmp_path: Path) -> None: + root = tmp_path / "cache" + build = tmp_path / "work" + _make_built_tree(build) + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(Path, "rename", side_effect=OSError("denied")), + pytest.raises(OSError), + ): + bootloader._publish(build, "k" * 16, {}) + + +# ----------------------------------------------------------------- install + + +def test_install_into_build_replaces_stale_subbuild(tmp_path: Path) -> None: + entry = tmp_path / "entry" + entry.mkdir() + # No .map: an optional output missing from the entry is skipped. + (entry / "bootloader.bin").write_bytes(b"bin") + (entry / "bootloader.elf").write_bytes(b"elf") + dest = CORE.relative_build_path("build", "bootloader") + dest.mkdir(parents=True) + (dest / "CMakeCache.txt").write_text("stale in-tree sub-build") + (dest / "old.obj").write_bytes(b"x") + bootloader._install_into_build(entry) + assert (dest / "bootloader.bin").read_bytes() == b"bin" + assert not (dest / "CMakeCache.txt").exists() + assert not (dest / "old.obj").exists() + assert not (dest / "bootloader.map").exists() + + +# -------------------------------------------------------------- size check + + +def test_size_check_passes_and_logs( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + bin_path = tmp_path / "bootloader.bin" + bin_path.write_bytes(b"\xe9" * 0x6620) + config = {"BOOTLOADER_OFFSET_IN_FLASH": 0x1000, "PARTITION_TABLE_OFFSET": 0x8000} + with caplog.at_level("INFO"): + bootloader._check_bootloader_size(bin_path, config) + assert "0x9e0 bytes (9%) free" in caplog.text + + +def test_size_check_overflow_raises(tmp_path: Path) -> None: + bin_path = tmp_path / "bootloader.bin" + bin_path.write_bytes(b"\xe9" * 0x7100) + config = {"BOOTLOADER_OFFSET_IN_FLASH": 0x1000, "PARTITION_TABLE_OFFSET": 0x8000} + with pytest.raises(EsphomeError, match="overflows by 0x100 bytes"): + bootloader._check_bootloader_size(bin_path, config) + + +# ------------------------------------------------------------ orchestration + + +def test_ensure_fails_soft_without_configure_outputs() -> None: + """No app config or compiler id means fall back, not crash.""" + assert bootloader.ensure_cached_bootloader() == 1 + + +def _orchestration_env(tmp_path: Path): + """Patches shared by the ensure_cached_bootloader tests.""" + root = tmp_path / "cache-root" + build = CORE.relative_build_path("build") + build.mkdir(parents=True, exist_ok=True) + (build / "config").mkdir(exist_ok=True) + (build / "config" / "sdkconfig.json").write_text( + json.dumps( + { + "A": "1", + "BOOTLOADER_OFFSET_IN_FLASH": 0x1000, + "PARTITION_TABLE_OFFSET": 0x8000, + } + ) + ) + return root + + +def test_ensure_cache_hit_installs_without_building(tmp_path: Path) -> None: + root = _orchestration_env(tmp_path) + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"), + patch.object(bootloader, "_compute_key", return_value="deadbeefdeadbeef"), + patch.object(bootloader, "_build_standalone") as mock_build, + ): + (root / "deadbeefdeadbeef").mkdir(parents=True) + for name in bootloader._OUTPUTS: + (root / "deadbeefdeadbeef" / name).write_bytes(b"\xe9" * 64) + (root / "config_names.json").write_text('["A"]') + assert bootloader.ensure_cached_bootloader() == 0 + mock_build.assert_not_called() + installed = CORE.relative_build_path("build", "bootloader", "bootloader.bin") + assert installed.read_bytes() == b"\xe9" * 64 + + +def test_ensure_cache_miss_builds_and_publishes(tmp_path: Path) -> None: + """Known names but no matching entry: build, harvest, publish.""" + root = _orchestration_env(tmp_path) + root.mkdir(parents=True) + (root / "config_names.json").write_text('["A"]') + + def fake_build(build_dir: Path, verbose: bool) -> int: + _make_built_tree(build_dir) + return 0 + + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"), + patch.object(bootloader, "_compute_key", return_value="deadbeefdeadbeef"), + patch.object(bootloader, "_build_standalone", side_effect=fake_build), + ): + assert bootloader.ensure_cached_bootloader() == 0 + # Entry published under the key, names harvested, work dir removed. + assert (root / "deadbeefdeadbeef" / "bootloader.bin").is_file() + with patch.object(bootloader, "_cache_root", return_value=root): + assert "A" in bootloader._load_config_names() + assert not list(root.glob(".build-*")) + installed = CORE.relative_build_path("build", "bootloader", "bootloader.bin") + assert installed.is_file() + + +def test_ensure_returns_build_failure(tmp_path: Path) -> None: + root = _orchestration_env(tmp_path) + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"), + patch.object(bootloader, "_build_standalone", return_value=2), + ): + assert bootloader.ensure_cached_bootloader() == 2 + assert not list(root.glob(".build-*")) + + +def test_ensure_soft_fails_when_build_yields_no_config(tmp_path: Path) -> None: + root = _orchestration_env(tmp_path) + + def build_without_config(build_dir: Path, verbose: bool) -> int: + build_dir.mkdir(parents=True, exist_ok=True) + return 0 + + with ( + patch.object(bootloader, "_cache_root", return_value=root), + patch.object(bootloader, "_compiler_id", return_value="/tc/gcc"), + patch.object(bootloader, "_build_standalone", side_effect=build_without_config), + ): + assert bootloader.ensure_cached_bootloader() == 1 + + +def test_prune_removes_only_old_work_dirs(tmp_path: Path) -> None: + """A concurrent build's fresh work dir must survive the prune.""" + old = tmp_path / ".build-1" + fresh = tmp_path / ".build-2" + old.mkdir() + fresh.mkdir() + os.utime(old, (0, 0)) + with patch.object(bootloader, "_cache_root", return_value=tmp_path): + bootloader._prune_stale_dirs() + assert not old.exists() + assert fresh.exists() + + +def test_prune_swallows_cache_root_errors() -> None: + """Housekeeping never fails a build.""" + with patch.object(bootloader, "_cache_root", side_effect=OSError("gone")): + bootloader._prune_stale_dirs() + + +# --------------------------------------------------------- flash injection + + +def _flash_build(tmp_path: Path, with_bin: bool = True, offset: int = 0x1000) -> Path: + build = tmp_path / "build" + (build / "config").mkdir(parents=True) + (build / "config" / "sdkconfig.json").write_text( + json.dumps({"BOOTLOADER_OFFSET_IN_FLASH": offset}) + ) + if with_bin: + (build / "bootloader").mkdir() + (build / "bootloader" / "bootloader.bin").write_bytes(b"\xe9") + return build + + +def test_inject_adds_cached_bootloader(tmp_path: Path) -> None: + build = _flash_build(tmp_path) + flash_data = {"flash_files": {"0x10000": "dev.bin"}} + bootloader.inject_bootloader_flash_file(flash_data, build) + assert flash_data["flash_files"]["0x1000"] == "bootloader/bootloader.bin" + + +def test_inject_uses_configured_offset(tmp_path: Path) -> None: + build = _flash_build(tmp_path, offset=0) + flash_data: dict = {} + bootloader.inject_bootloader_flash_file(flash_data, build) + assert flash_data["flash_files"] == {"0x0": "bootloader/bootloader.bin"} + + +def test_inject_noop_when_idf_wrote_the_entry(tmp_path: Path) -> None: + """Stock and bypass trees already carry a bootloader entry.""" + build = _flash_build(tmp_path) + flash_data = {"flash_files": {"0x1000": "bootloader/bootloader.bin"}} + bootloader.inject_bootloader_flash_file(flash_data, build) + assert flash_data["flash_files"] == {"0x1000": "bootloader/bootloader.bin"} + + +def test_inject_noop_without_bin_or_config(tmp_path: Path) -> None: + flash_data: dict = {"flash_files": {}} + bootloader.inject_bootloader_flash_file( + flash_data, _flash_build(tmp_path / "a", with_bin=False) + ) + no_config = tmp_path / "b" / "build" + (no_config / "bootloader").mkdir(parents=True) + (no_config / "bootloader" / "bootloader.bin").write_bytes(b"\xe9") + bootloader.inject_bootloader_flash_file(flash_data, no_config) + assert flash_data == {"flash_files": {}} diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index ee21cd77a9c..54ef2a05f76 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -12,7 +12,12 @@ from unittest.mock import call, patch import pytest -from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +from esphome.components.esp32.const import ( + KEY_ESP32, + KEY_FLASH_SIZE, + KEY_IDF_VERSION, + KEY_VARIANT, +) import esphome.config_validation as cv from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, @@ -118,6 +123,7 @@ def _up_to_date_compile(ninja_side_effect=None) -> Iterator[tuple]: with ( patch.object(toolchain, "need_reconfigure", return_value=False), patch.object(toolchain, "_cache_entries_changed", return_value=False), + patch.object(toolchain, "_use_cached_bootloader", return_value=False), patch.object( toolchain, "_run_ninja", return_value=0, side_effect=ninja_side_effect ) as mock_ninja, @@ -710,6 +716,7 @@ def _fake_tools(env: dict[str, str] | None = None) -> Iterator: return_value={"PATH": "/bin", "IDF_CCACHE_ENABLE": "0", **(env or {})}, ), patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_use_cached_bootloader", return_value=False), patch.object(toolchain, "run_build_tool", return_value=0) as mock_run, ): yield mock_run @@ -741,6 +748,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None: "-DESP_PLATFORM=1", f"-DSDKCONFIG={sdkconfig}", "-DCCACHE_ENABLE=0", + "-DESPHOME_USE_CACHED_BOOTLOADER=0", project, ] kwargs = mock_run.call_args.kwargs @@ -775,6 +783,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py( "-DPYTHON=/tools/python", "-DESP_PLATFORM=1", f"-DCCACHE_ENABLE={expected}", + "-DESPHOME_USE_CACHED_BOOTLOADER=0", str(build_dir.parent), ] @@ -786,7 +795,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None: assert toolchain.run_reconfigure(verbose=True) == 0 cmd = mock_run.call_args.args[0] assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd) - assert cmd[-2] == "-DCCACHE_ENABLE=0" + assert cmd[-3] == "-DCCACHE_ENABLE=0" + assert cmd[-2] == "-DESPHOME_USE_CACHED_BOOTLOADER=0" assert mock_run.call_args.kwargs["filter_lines"] is None @@ -853,6 +863,7 @@ _CONFIGURED = ( "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n" "PYTHON:UNINITIALIZED=/tools/python\n" "ESP_PLATFORM:UNINITIALIZED=1\n" + "ESPHOME_USE_CACHED_BOOTLOADER:UNINITIALIZED=0\n" ) @@ -1104,3 +1115,114 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N with patch.object(toolchain, "_get_cmake_output", return_value=output): assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF" assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None + + +def test_use_cached_bootloader_fallback_wins(setup_core: Path) -> None: + """A mid-run fallback pins the rest of the run to the in-tree build.""" + toolchain._cache().bootloader_fallback = True + assert toolchain._use_cached_bootloader() is False + + +def test_use_cached_bootloader_delegates_to_predicate(setup_core: Path) -> None: + from esphome.espidf import bootloader + + with patch.object(bootloader, "bootloader_cache_enabled", return_value=True): + assert toolchain._use_cached_bootloader() is True + + +def test_configure_defines_follow_bootloader_mode(setup_core: Path) -> None: + """The define always rides along, so a mode flip reconfigures.""" + with ( + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object(toolchain, "_cache_entries", return_value={}), + ): + with patch.object(toolchain, "_use_cached_bootloader", return_value=True): + on = toolchain._configure_defines() + with patch.object(toolchain, "_use_cached_bootloader", return_value=False): + off = toolchain._configure_defines() + assert on["ESPHOME_USE_CACHED_BOOTLOADER"] == "1" + assert off["ESPHOME_USE_CACHED_BOOTLOADER"] == "0" + + +def test_run_compile_installs_cached_bootloader(setup_core: Path) -> None: + from esphome.espidf import bootloader + + _setup_build(setup_core) + with ( + patch.object(toolchain, "need_reconfigure", return_value=False), + patch.object(toolchain, "_cache_entries_changed", return_value=False), + patch.object(toolchain, "_use_cached_bootloader", return_value=True), + patch.object( + bootloader, "ensure_cached_bootloader", return_value=0 + ) as mock_ensure, + patch.object(toolchain, "run_reconfigure") as mock_reconfigure, + patch.object(toolchain, "_run_ninja", return_value=0), + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + mock_ensure.assert_called_once_with(False) + mock_reconfigure.assert_not_called() + + +@pytest.mark.parametrize("reconfigure_rc", [0, 5]) +def test_run_compile_falls_back_when_bootloader_cache_fails( + setup_core: Path, reconfigure_rc: int +) -> None: + """A cache failure flips the run to the stock in-tree build.""" + from esphome.espidf import bootloader + + _setup_build(setup_core) + with ( + patch.object(toolchain, "need_reconfigure", return_value=False), + patch.object(toolchain, "_cache_entries_changed", return_value=False), + patch.object(toolchain, "_use_cached_bootloader", return_value=True), + patch.object(bootloader, "ensure_cached_bootloader", return_value=1), + patch.object( + toolchain, "run_reconfigure", return_value=reconfigure_rc + ) as mock_reconfigure, + patch.object(toolchain, "_run_ninja", return_value=0) as mock_ninja, + patch.object(toolchain, "print_summary"), + ): + rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) + assert rc == reconfigure_rc + assert toolchain._cache().bootloader_fallback is True + mock_reconfigure.assert_called_once_with(False) + assert mock_ninja.called is (reconfigure_rc == 0) + + +def test_create_factory_bin_merges_cached_bootloader(setup_core: Path) -> None: + """The cached bootloader joins the merge at its flash offset even + though IDF wrote no bootloader entry into flasher_args.json.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + (build / "bootloader").mkdir(parents=True) + (build / "bootloader" / "bootloader.bin").write_bytes(b"\xe9") + (build / "config").mkdir() + (build / "config" / "sdkconfig.json").write_text( + json.dumps({"BOOTLOADER_OFFSET_IN_FLASH": 0x1000}) + ) + (build / "test.bin").write_bytes(b"a") + (build / "flasher_args.json").write_text( + json.dumps( + { + "flash_files": {"0x10000": "test.bin"}, + "extra_esptool_args": {"chip": "esp32"}, + } + ) + ) + CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB" + with ( + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object( + toolchain.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0, "", ""), + ) as mock_run, + ): + assert toolchain.create_factory_bin() is True + cmd = mock_run.call_args.args[0] + offset_index = cmd.index("0x1000") + assert cmd[offset_index + 1] == str(build / "bootloader" / "bootloader.bin") + # Sections stay sorted by address; the bootloader comes first. + assert offset_index < cmd.index("0x10000")