[espidf] Add --skip-bootloader for OTA only builds (#19925)

This commit is contained in:
J. Nick Koston
2026-10-01 13:55:30 -05:00
committed by GitHub
parent 3ed1bc7e20
commit 0ed997da92
11 changed files with 765 additions and 48 deletions
+45 -2
View File
@@ -148,6 +148,7 @@ class ArgsProtocol(Protocol):
file: str | None
no_logs: bool
only_generate: bool
skip_bootloader: bool
show_secrets: bool
dashboard: bool
configuration: str
@@ -827,6 +828,14 @@ def write_cpp_file() -> int:
def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
if CORE.skip_bootloader and not (CORE.is_esp32 and CORE.using_toolchain_esp_idf):
# Info, not a warning: an orchestrator cannot see YAML toolchain
# overrides, this is its expected no-op, and a full build is safe.
_LOGGER.info(
"--skip-bootloader ignored: only supported on ESP32 with the "
"esp-idf toolchain"
)
CORE.skip_bootloader = False
# Keep this gate here, NOT in config validation: device-builder needs
# `esphome config` to keep succeeding with placeholders so onboarding can run.
if CONF_WIFI in config:
@@ -857,7 +866,10 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
return rc
# Create factory.bin, ota.bin, and firmware.elf copy
toolchain.create_factory_bin()
if not toolchain.create_factory_bin():
# A build whose factory image could not be produced must not
# exit 0; downloads would serve an image from an older build.
return 1
toolchain.create_ota_bin()
toolchain.create_elf_copy()
from esphome.build_helpers.idedata import warn_if_idedata_missing
@@ -975,8 +987,9 @@ def upload_using_esptool(
# and partitions included where the target needs them)
image = native.get_factory_firmware_path()
if not image.is_file():
hint = getattr(native, "missing_image_hint", lambda: None)()
raise EsphomeError(
f"{image} does not exist; compile the configuration first"
hint or f"{image} does not exist; compile the configuration first"
)
flash_images = [FlashImage(path=image, offset="0x0")]
else:
@@ -1379,6 +1392,12 @@ def _upload_via_native_api(
ota_type = espota2.OTA_TYPE_UPDATE_PARTITION_TABLE
elif getattr(args, "bootloader", False):
check_partition_access("--bootloader")
if (
getattr(args, "file", None) is None
and (native := native_backend())
and (hint := getattr(native, "missing_image_hint", lambda: None)())
):
raise EsphomeError(hint)
binary = CORE.bootloader_bin
ota_type = espota2.OTA_TYPE_UPDATE_BOOTLOADER
if getattr(args, "file", None) is not None:
@@ -1754,6 +1773,18 @@ def command_logs(args: ArgsProtocol, config: ConfigType) -> int | None:
def command_run(args: ArgsProtocol, config: ConfigType) -> int | None:
if (
CORE.skip_bootloader
and CORE.is_esp32
and CORE.using_toolchain_esp_idf
and any(
get_port_type(device) == PortType.SERIAL for device in (args.device or [])
)
):
# Fail before the compile: the result could never flash over serial.
# Elsewhere the flag is ignored, so serial stays fine there.
_LOGGER.error("--skip-bootloader builds cannot be flashed over serial")
return 1
exit_code = write_cpp(config)
if exit_code != 0:
return exit_code
@@ -2139,6 +2170,15 @@ SIMPLE_CONFIG_ACTIONS = [
]
def _add_skip_bootloader_arg(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--skip-bootloader",
help="Do not build the bootloader or the factory image; "
"the result can only be flashed over OTA.",
action="store_true",
)
def _add_states_args(parser: argparse.ArgumentParser) -> None:
"""Add mutually exclusive ``--states``/``--no-states`` flags to a parser.
@@ -2280,6 +2320,7 @@ def parse_args(argv):
help="Only generate source code, do not compile.",
action="store_true",
)
_add_skip_bootloader_arg(parser_compile)
parser_upload = subparsers.add_parser(
"upload",
@@ -2376,6 +2417,7 @@ def parse_args(argv):
parser_run.add_argument(
"--no-logs", help="Disable starting logs.", action="store_true"
)
_add_skip_bootloader_arg(parser_run)
_add_states_args(parser_run)
@@ -2635,6 +2677,7 @@ def run_esphome(argv):
CORE.config_path = conf_path
CORE.dashboard = args.dashboard
CORE.skip_bootloader = getattr(args, "skip_bootloader", False)
if args.toolchain is not None:
# CLI toolchain wins over esp32.toolchain in YAML.
CORE.toolchain = args.toolchain
+74
View File
@@ -3,6 +3,7 @@
import json
import logging
from pathlib import Path
import re
import textwrap
from esphome.build_helpers import pch
@@ -120,6 +121,78 @@ def _cmake_quote(value: str) -> str:
return f'"{escaped}"'
# CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be
# cleared at the CMake level (the same state IDF's RAM-app build type uses).
# The macro is IDF's __build_process_project_includes plus two added lines;
# the flag is ignored and the bootloader builds as usual if IDF changes it.
IDF_BOOTLOADER_OVERRIDE = """\
# ESPHome bootloader skip switch; see esphome/espidf/toolchain.py.
if(ESPHOME_SKIP_BOOTLOADER)
macro(__build_process_project_includes)
idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
include(${sdkconfig_cmake})
set(CONFIG_APP_BUILD_BOOTLOADER "")
# bt's CMakeLists reads the lowercase idf_target that the (now
# skipped) bootloader project_include leaks; keep it defined, or
# its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /.
idf_build_get_property(idf_target IDF_TARGET)
idf_build_get_property(build_properties __BUILD_PROPERTIES)
foreach(build_property ${build_properties})
idf_build_get_property(val ${build_property})
set(${build_property} "${val}")
endforeach()
idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
foreach(component_target ${build_component_targets})
__component_get_property(dir ${component_target} COMPONENT_DIR)
__component_get_property(_name ${component_target} COMPONENT_NAME)
set(COMPONENT_NAME ${_name})
set(COMPONENT_DIR ${dir})
set(COMPONENT_PATH ${dir})
if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
include(${COMPONENT_DIR}/project_include.cmake)
endif()
endforeach()
endmacro()
endif()
"""
# The lines the override adds to IDF's macro; idf_macro_matches() below
# strips them before comparing with the live macro.
BOOTLOADER_OVERRIDE_ADDED_LINES = (
'set(CONFIG_APP_BUILD_BOOTLOADER "")',
"idf_build_get_property(idf_target IDF_TARGET)",
)
_MACRO = re.compile(
r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL
)
def _normalized_macro(text: str) -> list[str] | None:
"""The macro body as comment-free, whitespace-collapsed lines."""
if (match := _MACRO.search(text)) is None:
return None
return [
re.sub(r"\s+", " ", line)
for raw in match.group(1).splitlines()
if (line := raw.split("#", 1)[0].strip())
]
_EXPECTED_MACRO = [
line
for line in _normalized_macro(IDF_BOOTLOADER_OVERRIDE)
if line not in BOOTLOADER_OVERRIDE_ADDED_LINES
]
def idf_macro_matches(idf_path: Path) -> bool:
"""Whether IDF's macro still matches the copy the override replays."""
build_cmake = idf_path / "tools" / "cmake" / "build.cmake"
live = _normalized_macro(build_cmake.read_text(encoding="utf-8"))
return live == _EXPECTED_MACRO
def get_project_cmakelists(
minimal: bool = False, builtin_components: list[str] | None = None
) -> str:
@@ -258,6 +331,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src)
include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
{IDF_BOOTLOADER_OVERRIDE}
{ldgen_override}
{cpp_standard_options}
+18 -9
View File
@@ -670,20 +670,29 @@ def get_download_types(storage_json):
# No recorded firmware path means nothing was built; no downloads.
if storage_json.firmware_bin_path is None:
return []
return [
{
"title": "Factory format (Previously Modern)",
"description": "For use with ESPHome Web and other tools.",
"file": "firmware.factory.bin",
"download": f"{storage_json.name}.factory.bin",
},
from esphome.espidf.toolchain import tree_skips_bootloader
types = []
# A --skip-bootloader tree deliberately has no factory image; an
# unreadable tree (PlatformIO, capability probes) reads as full.
if not tree_skips_bootloader(Path(storage_json.firmware_bin_path).parent):
types.append(
{
"title": "Factory format (Previously Modern)",
"description": "For use with ESPHome Web and other tools.",
"file": "firmware.factory.bin",
"download": f"{storage_json.name}.factory.bin",
}
)
types.append(
{
"title": "OTA format (Previously Legacy)",
"description": "For OTA updating a device.",
"file": "firmware.ota.bin",
"download": f"{storage_json.name}.ota.bin",
},
]
}
)
return types
def only_on_variant(*, supported=None, unsupported=None, msg_prefix="This feature"):
+3
View File
@@ -589,6 +589,8 @@ class EsphomeCore:
self.vscode = False
# True if running in testing mode (disables validation checks for grouped testing)
self.testing_mode = False
# True if this build skips the bootloader and factory image (OTA only)
self.skip_bootloader = False
# The name of the node
self.name: str | None = None
# The friendly name of the node
@@ -692,6 +694,7 @@ class EsphomeCore:
from esphome.pins import PIN_SCHEMA_REGISTRY
self.dashboard = False
self.skip_bootloader = False
self.name = None
self.friendly_name = None
self.area = None
+77 -5
View File
@@ -26,11 +26,14 @@ from esphome.core import CORE, EsphomeError
from esphome.espidf import variant_to_idf_target
from esphome.espidf.framework import check_esp_idf_install, get_framework_env
from esphome.espidf.size_summary import print_summary
from esphome.helpers import add_git_ceiling_directory, get_bool_env, write_file
from esphome.helpers import add_git_ceiling_directory, get_bool_env, rmtree, write_file
_LOGGER = logging.getLogger(__name__)
DOMAIN = "espidf_toolchain"
# The -D that tells the generated CMakeLists to skip the in-tree bootloader
# build; also read back from CMakeCache.txt to identify a tree.
SKIP_BOOTLOADER_DEFINE = "ESPHOME_SKIP_BOOTLOADER"
@dataclass
@@ -38,6 +41,7 @@ class _CacheData:
paths: dict[str, tuple] = field(default_factory=dict)
env: dict[str, dict[str, str]] = field(default_factory=dict)
cmake_output: dict[Path, str] = field(default_factory=dict)
skip_bootloader: bool | None = None
cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict)
@@ -285,6 +289,48 @@ def _cache_entries_changed() -> bool:
return any(cache.get(k) != v for k, v in _configure_defines().items())
def tree_skips_bootloader(build_dir: Path) -> bool:
"""Whether a configured tree was set up to skip the bootloader build.
Total: an unreadable tree reads as the stock full build.
"""
try:
cache = _parse_cmakecache(build_dir / "CMakeCache.txt")
except (OSError, ValueError) as err:
_LOGGER.debug("Cannot read %s, assuming a full build: %s", build_dir, err)
return False
return cache.get(SKIP_BOOTLOADER_DEFINE) == "1"
def _skip_bootloader() -> bool:
"""Whether this tree should not build a bootloader at all; per-run memo."""
cache = _cache()
if cache.skip_bootloader is None:
cache.skip_bootloader = _compute_skip_bootloader()
return cache.skip_bootloader
def _compute_skip_bootloader() -> bool:
if not CORE.skip_bootloader:
return False
from esphome.build_gen.espidf import idf_macro_matches
if not idf_macro_matches(_get_idf_path()):
_LOGGER.warning(
"--skip-bootloader ignored: IDF changed its bootloader macro; "
"building the bootloader"
)
return False
return True
def missing_image_hint() -> str | None:
"""Why an expected flash image is absent, for upload error messages."""
if tree_skips_bootloader(_build_dir()):
return "this build was compiled with --skip-bootloader; recompile without it"
return None
def _configure_defines() -> dict[str, str]:
"""Every ``-D`` idf.py passes to cmake, in its order."""
return {
@@ -292,6 +338,9 @@ def _configure_defines() -> dict[str, str]:
"PYTHON": _get_idf_tool("python"),
"ESP_PLATFORM": "1",
**_cache_entries(),
# ESPHome's own switch; idf.py never passes it and cmake keeps the
# cached value, so idf.py runs against the tree stay in the same mode.
SKIP_BOOTLOADER_DEFINE: "1" if _skip_bootloader() else "0",
}
@@ -312,6 +361,14 @@ def run_reconfigure(verbose: bool = False) -> int:
"""Run the CMake configure, with the arguments idf.py uses."""
build_dir = _build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
if _skip_bootloader() and not tree_skips_bootloader(build_dir):
# Flipping into skip mode: full-mode leftovers are stale for
# OTA --bootloader and downloads, and a partial cleanup would
# poison the flip back (deleted byproducts never regenerate).
for stale in ("bootloader", "bootloader-prefix"):
if (path := build_dir / stale).is_dir():
rmtree(path)
get_factory_firmware_path().unlink(missing_ok=True)
cmd = [_get_idf_tool("cmake"), "-G", "Ninja"]
if _idf_py().binary_dir_arg:
cmd += ["-B", str(build_dir)]
@@ -861,6 +918,20 @@ def get_idedata() -> dict | None:
def create_factory_bin() -> bool:
"""Create factory.bin by merging bootloader, partition table, and app."""
build_dir = CORE.relative_build_path("build")
if tree_skips_bootloader(build_dir):
# Nothing to merge, and nothing stale: the flip into skip mode
# already removed the factory image and the sub-build.
_LOGGER.info("Bootloader skipped; no factory image")
return True
if _merge_factory_bin(build_dir):
return True
# Never leave an image that does not match this build.
get_factory_firmware_path().unlink(missing_ok=True)
return False
def _merge_factory_bin(build_dir: Path) -> bool:
"""Run the esptool merge for a full-build tree."""
flasher_args_path = build_dir / "flasher_args.json"
if not flasher_args_path.is_file():
@@ -883,10 +954,11 @@ def create_factory_bin() -> bool:
flash_data.get("flash_files", {}).items(), key=lambda kv: int(kv[0], 16)
):
file_path = build_dir / fname
if file_path.is_file():
sections.extend([addr, str(file_path)])
else:
_LOGGER.warning("Flash file not found: %s", file_path)
if not file_path.is_file():
# A partial factory image would not boot; never write one.
_LOGGER.error("Flash file not found: %s", file_path)
return False
sections.extend([addr, str(file_path)])
if not sections:
_LOGGER.warning("No flash sections found")
+57 -20
View File
@@ -30,14 +30,31 @@ WATCHED = (
"build/compile_commands.json",
"build/project_description.json",
"build/config/sdkconfig.h",
"build/bootloader/bootloader.bin",
)
# Ninja logs whose outputs mean real work when their recorded mtime changes.
# The top level re-logs the bootloader step's byproducts on every build, so
# the bootloader is judged by its own sub-build log instead.
# Only a tree that builds the bootloader has one to watch.
BOOTLOADER_BIN = "build/bootloader/bootloader.bin"
OVERRIDE_INEFFECTIVE = (
"skip-mode tree built a bootloader; the IDF_BOOTLOADER_OVERRIDE macro in "
"esphome/build_gen/espidf.py is not taking effect"
)
# Ninja logs whose outputs mean real work when their recorded mtime changes;
# the bootloader is judged by its own sub-build log when one exists.
TOP_NINJA_LOG = "build/.ninja_log"
NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log")
def _ninja_logs(skip_bootloader: bool) -> list[str]:
"""A missing sub-build log stays an error in the mode that requires one."""
logs = [TOP_NINJA_LOG]
if not skip_bootloader:
logs.append("build/bootloader/.ninja_log")
return logs
BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/")
MACRO_CHANGED = (
"IDF changed __build_process_project_includes; update "
"IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
)
WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld")
DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*"
@@ -46,19 +63,24 @@ def _digest(path: Path) -> str | None:
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
def watched(name: str) -> list[str]:
"""Files that change if idf.py configures or builds differently."""
return [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"]
def watched(name: str, skip_bootloader: bool) -> list[str]:
"""The files idf.py must leave untouched for this tree's mode."""
files = [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"]
if not skip_bootloader:
files.append(BOOTLOADER_BIN)
return files
def _snapshot(build_path: Path, name: str) -> dict[str, str | None]:
return {f: _digest(build_path / f) for f in watched(name)}
def _snapshot(
build_path: Path, name: str, skip_bootloader: bool
) -> dict[str, str | None]:
return {f: _digest(build_path / f) for f in watched(name, skip_bootloader)}
def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
def _ninja_mtimes(build_path: Path, logs: list[str]) -> dict[tuple[str, str], str]:
"""(log, output) -> recorded mtime; compaction-safe, unlike a line count."""
mtimes = {}
for name in NINJA_LOGS:
for name in logs:
log = build_path / name
lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else []
for fields in (line.split("\t") for line in lines if not line.startswith("#")):
@@ -69,10 +91,12 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]:
return mtimes
def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]:
def _log_problems(
build_path: Path, mtimes: dict[tuple[str, str], str], logs: list[str]
) -> list[str]:
"""A missing or unparsable ninja log would otherwise compare as unchanged."""
problems = []
for log in NINJA_LOGS:
for log in logs:
if not (build_path / log).is_file():
problems.append(f"missing {log}")
elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes):
@@ -101,12 +125,22 @@ def _setup_core(build_path: Path, description: dict) -> tuple[str, str]:
def check(build_path: Path) -> list[str]:
"""Return the problems found in one build tree."""
# pylint: disable=protected-access
from esphome.build_gen.espidf import idf_macro_matches
from esphome.core import CORE
from esphome.espidf import toolchain
description = json.loads(
(build_path / "build" / "project_description.json").read_text(encoding="utf-8")
)
name, version = _setup_core(build_path, description)
# Reconfiguring must not flip the tree's bootloader mode: the check
# validates the shape the build produced, not this process's flags.
skip_bootloader = toolchain.tree_skips_bootloader(build_path / "build")
CORE.skip_bootloader = skip_bootloader
# A prior tree's memoized decision must not leak into this one.
toolchain._cache().skip_bootloader = None
if not idf_macro_matches(toolchain._get_idf_path(version)):
return [MACRO_CHANGED]
# ESP-IDF's openthread stamps the configure time into its compile flags;
# pin it before the env is cached so both configures get the same value.
os.environ["SOURCE_DATE_EPOCH"] = "0"
@@ -122,11 +156,14 @@ def check(build_path: Path) -> list[str]:
return [f"ESPHome's CMake configure failed with exit code {rc}"]
if (rc := toolchain._run_ninja("all", verbose=False, jobs=None)) != 0:
return [f"ESPHome's ninja build failed with exit code {rc}"]
before = _snapshot(build_path, name)
mtimes_before = _ninja_mtimes(build_path)
before = _snapshot(build_path, name, skip_bootloader)
logs = _ninja_logs(skip_bootloader)
mtimes_before = _ninja_mtimes(build_path, logs)
# A moved or renamed output would otherwise compare as "unchanged".
problems = [f"missing {f}" for f, digest in before.items() if digest is None]
if problems := problems + _log_problems(build_path, mtimes_before):
if skip_bootloader and (build_path / BOOTLOADER_BIN).is_file():
problems.append(OVERRIDE_INEFFECTIVE)
if problems := problems + _log_problems(build_path, mtimes_before, logs):
return problems
for action in ("reconfigure", "build"):
result = subprocess.run(
@@ -139,10 +176,10 @@ def check(build_path: Path) -> list[str]:
)
if result.returncode != 0:
return [f"idf.py {action} failed:\n{result.stdout}{result.stderr}"]
after = _snapshot(build_path, name)
mtimes_after = _ninja_mtimes(build_path)
after = _snapshot(build_path, name, skip_bootloader)
mtimes_after = _ninja_mtimes(build_path, logs)
problems = [f"idf.py changed {f}" for f in before if before[f] != after[f]]
problems += _log_problems(build_path, mtimes_after)
problems += _log_problems(build_path, mtimes_after, logs)
for key in sorted(mtimes_before.keys() | mtimes_after.keys()):
log, out = key
if not out.endswith(WORK_SUFFIXES):
+71 -9
View File
@@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script"))
import check_idf_py_equivalence as guard # noqa: E402
from esphome.build_gen import espidf as build_gen # noqa: E402
from esphome.core import CORE # noqa: E402
from esphome.espidf import toolchain # noqa: E402
@@ -25,12 +26,26 @@ def _reset_core() -> Iterator[None]:
CORE.reset()
def _make_tree(tmp_path: Path) -> Path:
BOOTLOADER_LOG = "build/bootloader/.ninja_log"
ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG)
def _make_tree(tmp_path: Path, skip_bootloader: bool = False) -> Path:
"""A fake build tree; stock shape by default, or the skip shape
(define set to 1, no bootloader bin, no sub-build)."""
tree = tmp_path / "config" / ".esphome" / "build" / "dev"
build = tree / "build"
for name in (*guard.watched("dev"), *guard.NINJA_LOGS):
files = [
*guard.watched("dev", skip_bootloader),
*guard._ninja_logs(skip_bootloader),
]
for name in files:
(tree / name).parent.mkdir(parents=True, exist_ok=True)
(tree / name).write_bytes(b"x")
define = "1" if skip_bootloader else "0"
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED={define}\n"
)
(build / "project_description.json").write_text(
json.dumps(
{
@@ -44,9 +59,10 @@ def _make_tree(tmp_path: Path) -> Path:
"# ninja log v7\n1\t2\t10\tesp-idf/a.obj\t0\n"
"1\t2\t10\tbootloader/bootloader.bin\t0\n"
)
(build / "bootloader" / ".ninja_log").write_text(
"# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n"
)
if not skip_bootloader:
(build / "bootloader" / ".ninja_log").write_text(
"# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n"
)
(tree / "sdkconfig.dev").write_text("")
return tree
@@ -56,6 +72,7 @@ def _run_check(
side_effect: Callable[[list[str]], None] = lambda cmd: None,
rc: int = 0,
esphome_rcs: tuple[int, int] = (0, 0),
macro_matches: bool = True,
envs: list[dict[str, str]] | None = None,
) -> tuple[list[str], list[list[str]]]:
"""Run check() with idf.py replaced by ``side_effect``; return problems, calls.
@@ -80,6 +97,7 @@ def _run_check(
patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]),
patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]),
patch.object(build_gen, "idf_macro_matches", return_value=macro_matches),
patch.object(guard.subprocess, "run", side_effect=run),
patch.dict(os.environ),
):
@@ -168,9 +186,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None:
assert len(calls) == 1
@pytest.mark.parametrize(
"remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS]
)
@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS])
def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None:
"""A moved or renamed output must not compare as unchanged."""
tree = _make_tree(tmp_path)
@@ -197,7 +213,7 @@ def test_check_stops_when_the_esphome_baseline_fails(
assert calls == []
@pytest.mark.parametrize("log", guard.NINJA_LOGS)
@pytest.mark.parametrize("log", ALL_LOGS)
def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None:
"""A log format change must not leave the rebuild check with nothing to compare."""
tree = _make_tree(tmp_path)
@@ -223,6 +239,52 @@ def test_main_rejects_a_path_that_is_not_a_tree(
mock_check.assert_not_called()
def test_check_resets_the_skip_memo_per_tree(tmp_path: Path) -> None:
"""A second tree must not inherit the first tree's memoized mode."""
tree = _make_tree(tmp_path)
CORE.skip_bootloader = True
toolchain._cache().skip_bootloader = True # leftover from a prior tree
problems, _ = _run_check(tree)
assert problems == []
assert toolchain._skip_bootloader() is False
def test_check_accepts_a_skip_bootloader_tree(tmp_path: Path) -> None:
"""No bootloader bin or sub-build is the skip shape, not missing input."""
tree = _make_tree(tmp_path, skip_bootloader=True)
problems, calls = _run_check(tree)
assert problems == []
assert len(calls) == 2
# The baseline reconfigure must not flip the tree's mode.
assert CORE.skip_bootloader is True
def test_check_flags_an_ineffective_override(tmp_path: Path) -> None:
"""A skip-mode tree that still built a bootloader must fail CI."""
tree = _make_tree(tmp_path, skip_bootloader=True)
(tree / guard.BOOTLOADER_BIN).parent.mkdir(parents=True)
(tree / guard.BOOTLOADER_BIN).write_bytes(b"x")
problems, _ = _run_check(tree)
assert problems == [guard.OVERRIDE_INEFFECTIVE]
def test_check_requires_the_sub_log_on_a_stock_tree(tmp_path: Path) -> None:
"""The mode comes from the define, so a vanished sub-build stays an error."""
tree = _make_tree(tmp_path)
(tree / BOOTLOADER_LOG).unlink()
problems, calls = _run_check(tree)
assert problems == [f"missing {BOOTLOADER_LOG}"]
assert calls == []
def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None:
"""An IDF bump that rewrites the overridden macro must fail CI."""
tree = _make_tree(tmp_path)
problems, calls = _run_check(tree, macro_matches=False)
assert problems == [guard.MACRO_CHANGED]
assert calls == []
def test_main_without_build_trees(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
+72
View File
@@ -670,3 +670,75 @@ def test_pch_checksum_disabled_writes_nothing(
_make_pch_project(tmp_path)
write_pch_checksum()
assert not CORE.relative_build_path(_PCH_SUM_PATH).exists()
# The macro body as shipped in tools/cmake/build.cmake; byte identical in
# IDF 5.5.5 and 6.1.0, so one fixture covers both supported versions.
IDF_BUILD_CMAKE = """\
some_other_cmake()
macro(__build_process_project_includes)
# Include the sdkconfig cmake file, since the following operations require
# knowledge of config values.
idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
include(${sdkconfig_cmake})
# Make each build property available as a read-only variable
idf_build_get_property(build_properties __BUILD_PROPERTIES)
foreach(build_property ${build_properties})
idf_build_get_property(val ${build_property})
set(${build_property} "${val}")
endforeach()
idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
# Include each component's project_include.cmake
foreach(component_target ${build_component_targets})
__component_get_property(dir ${component_target} COMPONENT_DIR)
__component_get_property(_name ${component_target} COMPONENT_NAME)
set(COMPONENT_NAME ${_name})
set(COMPONENT_DIR ${dir})
set(COMPONENT_PATH ${dir}) # this is deprecated, users are encouraged to use COMPONENT_DIR;
# retained for compatibility
if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
include(${COMPONENT_DIR}/project_include.cmake)
endif()
endforeach()
endmacro()
"""
def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path:
idf = tmp_path / "idf"
(idf / "tools" / "cmake").mkdir(parents=True)
(idf / "tools" / "cmake" / "build.cmake").write_text(text)
return idf
def test_normalized_macro_strips_comments_and_whitespace() -> None:
from esphome.build_gen.espidf import _normalized_macro
text = "macro(__build_process_project_includes)\n a( b ) # tail\n\n # only\n c(d)\nendmacro()"
assert _normalized_macro(text) == ["a( b )", "c(d)"]
def test_normalized_macro_none_without_macro() -> None:
from esphome.build_gen.espidf import _normalized_macro
assert _normalized_macro("nothing here") is None
def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None:
"""The override's embedded copy must equal what build.cmake ships."""
from esphome.build_gen.espidf import idf_macro_matches
assert idf_macro_matches(_write_idf_build_cmake(tmp_path)) is True
def test_idf_macro_mismatch_detected(tmp_path: Path) -> None:
from esphome.build_gen.espidf import idf_macro_matches
changed = IDF_BUILD_CMAKE.replace(
"include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)"
)
assert idf_macro_matches(_write_idf_build_cmake(tmp_path, changed)) is False
+29
View File
@@ -52,6 +52,35 @@ def test_recorded_firmware_path_yields_downloads(platform: str, tmp_path: Path)
)
def test_esp32_skip_bootloader_tree_offers_no_factory_entry(tmp_path: Path) -> None:
"""A --skip-bootloader tree has no factory image; do not offer one.
An unreadable tree (PlatformIO, capability probes passing a bare
path) reads as a full build and keeps the entry.
"""
from esphome.espidf.toolchain import SKIP_BOOTLOADER_DEFINE
storage = _wizard_storage()
storage.firmware_bin_path = tmp_path / "firmware.bin"
files = [entry["file"] for entry in _download_types("esp32", storage)]
assert files == ["firmware.factory.bin", "firmware.ota.bin"]
cache = tmp_path / "CMakeCache.txt"
cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n")
files = [entry["file"] for entry in _download_types("esp32", storage)]
assert files == ["firmware.ota.bin"]
# A capability probe passes the path as a plain string.
storage.firmware_bin_path = str(tmp_path / "firmware.bin")
files = [entry["file"] for entry in _download_types("esp32", storage)]
assert files == ["firmware.ota.bin"]
cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n")
files = [entry["file"] for entry in _download_types("esp32", storage)]
assert files == ["firmware.factory.bin", "firmware.ota.bin"]
def _nrf52_files(tmp_path: Path, *built: str) -> list[str]:
"""The files nrf52 offers for a build directory holding *built*."""
(tmp_path / "zephyr").mkdir()
+215 -3
View File
@@ -8,11 +8,16 @@ import json
import os
from pathlib import Path
import subprocess
from unittest.mock import call, patch
from unittest.mock import MagicMock, call, patch
import pytest
from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT
from esphome.components.esp32.const import (
KEY_ESP32,
KEY_FLASH_SIZE,
KEY_IDF_VERSION,
KEY_VARIANT,
)
import esphome.config_validation as cv
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
@@ -893,6 +898,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None:
"-DESP_PLATFORM=1",
f"-DSDKCONFIG={sdkconfig}",
"-DCCACHE_ENABLE=0",
"-DESPHOME_SKIP_BOOTLOADER=0",
project,
]
kwargs = mock_run.call_args.kwargs
@@ -927,6 +933,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py(
"-DPYTHON=/tools/python",
"-DESP_PLATFORM=1",
f"-DCCACHE_ENABLE={expected}",
"-DESPHOME_SKIP_BOOTLOADER=0",
str(build_dir.parent),
]
@@ -938,7 +945,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None:
assert toolchain.run_reconfigure(verbose=True) == 0
cmd = mock_run.call_args.args[0]
assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd)
assert cmd[-2] == "-DCCACHE_ENABLE=0"
assert cmd[-3] == "-DCCACHE_ENABLE=0"
assert cmd[-2] == "-DESPHOME_SKIP_BOOTLOADER=0"
assert mock_run.call_args.kwargs["filter_lines"] is None
@@ -1005,6 +1013,7 @@ _CONFIGURED = (
"PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n"
"PYTHON:UNINITIALIZED=/tools/python\n"
"ESP_PLATFORM:UNINITIALIZED=1\n"
"ESPHOME_SKIP_BOOTLOADER:UNINITIALIZED=0\n"
)
@@ -1256,3 +1265,206 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N
with patch.object(toolchain, "_get_cmake_output", return_value=output):
assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF"
assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None
def test_skip_bootloader_requires_flag_and_matching_macro(
setup_core: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""The switch is explicit, and an IDF macro change wins over it."""
from esphome.build_gen import espidf as build_gen
assert toolchain._skip_bootloader() is False
CORE.skip_bootloader = True
toolchain._cache().skip_bootloader = None # decision is memoized per run
with (
patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
patch.object(build_gen, "idf_macro_matches", return_value=True),
):
assert toolchain._skip_bootloader() is True
toolchain._cache().skip_bootloader = None
with (
patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")),
patch.object(build_gen, "idf_macro_matches", return_value=False) as mock_match,
caplog.at_level("WARNING"),
):
assert toolchain._skip_bootloader() is False
assert toolchain._skip_bootloader() is False
mock_match.assert_called_once() # the memo also dedupes the warning
assert "--skip-bootloader ignored" in caplog.text
def test_configure_defines_follow_skip_bootloader(setup_core: Path) -> None:
with (
patch.object(toolchain, "_get_idf_tool", return_value="/tools/python"),
patch.object(toolchain, "_cache_entries", return_value={}),
):
with patch.object(toolchain, "_skip_bootloader", return_value=True):
assert (
toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "1"
)
with patch.object(toolchain, "_skip_bootloader", return_value=False):
assert (
toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "0"
)
def test_tree_skips_bootloader_reads_the_define(tmp_path: Path) -> None:
build = tmp_path / "build"
build.mkdir()
assert toolchain.tree_skips_bootloader(build) is False # not configured
cache = build / "CMakeCache.txt"
cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n")
assert toolchain.tree_skips_bootloader(build) is True
cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n")
assert toolchain.tree_skips_bootloader(build) is False
def test_tree_skips_bootloader_unreadable_cache_reads_full(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""Unreadable trees fall back to the safe full build, with a trace."""
build = tmp_path / "build"
(build / "CMakeCache.txt").mkdir(parents=True) # read raises OSError
with caplog.at_level("DEBUG"):
assert toolchain.tree_skips_bootloader(build) is False
assert "assuming a full build" in caplog.text
def test_create_factory_bin_merges_the_listed_flash_files(
setup_core: Path,
) -> None:
"""Every listed file lands in the esptool merge argv, address sorted."""
_setup_build(setup_core)
CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB"
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n"
)
(build / "app.bin").write_bytes(b"app")
(build / "boot.bin").write_bytes(b"boot")
(build / "flasher_args.json").write_text(
'{"flash_files": {"0x10000": "app.bin", "0x0": "boot.bin"},'
' "extra_esptool_args": {"chip": "esp32"}}'
)
with (
patch.object(toolchain, "_get_idf_env", return_value={}),
patch.object(toolchain, "_get_idf_tool", return_value="/py"),
patch.object(
toolchain.subprocess, "run", return_value=MagicMock(returncode=0)
) as mock_run,
):
assert toolchain.create_factory_bin() is True
argv = mock_run.call_args.args[0]
boot = argv.index("0x0")
assert argv[boot + 1].endswith("boot.bin")
assert argv[boot + 2] == "0x10000"
def test_create_factory_bin_fails_on_a_missing_listed_flash_file(
setup_core: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A partial factory image would not boot; never write one."""
_setup_build(setup_core)
CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB"
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n"
)
(build / "flasher_args.json").write_text('{"flash_files": {"0x0": "missing.bin"}}')
stale = toolchain.get_factory_firmware_path()
stale.write_bytes(b"old")
with patch.object(toolchain.subprocess, "run") as mock_run:
assert toolchain.create_factory_bin() is False
mock_run.assert_not_called()
assert "Flash file not found" in caplog.text
# The image from an earlier build must not be served as this one.
assert not stale.exists()
def test_create_factory_bin_skip_mode_creates_nothing(
setup_core: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""The flip into skip mode already cleaned up; the merge just no-ops."""
_setup_build(setup_core)
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
)
with (
patch.object(toolchain.subprocess, "run") as mock_run,
caplog.at_level("INFO"),
):
assert toolchain.create_factory_bin() is True
mock_run.assert_not_called()
assert "no factory image" in caplog.text
def test_run_reconfigure_flip_into_skip_mode_cleans_up(setup_core: Path) -> None:
"""Full-mode leftovers are stale for OTA --bootloader and downloads, and
a partial cleanup would poison the flip back to full mode."""
_setup_build(setup_core)
build = CORE.relative_build_path("build")
(build / "bootloader").mkdir(parents=True)
(build / "bootloader" / "bootloader.bin").write_bytes(b"old")
# bootloader-prefix deliberately absent: cleanup skips what is not there.
stale = toolchain.get_factory_firmware_path()
stale.parent.mkdir(parents=True, exist_ok=True)
stale.write_bytes(b"old")
with (
patch.object(toolchain, "_skip_bootloader", return_value=True),
patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
patch.object(toolchain, "_tool_env", return_value={}),
patch.object(toolchain, "run_build_tool", return_value=0),
patch.object(toolchain, "_idf_py") as mock_idf_py,
):
mock_idf_py.return_value.binary_dir_arg = False
assert toolchain.run_reconfigure() == 0
assert not (build / "bootloader").exists()
assert not stale.exists()
def test_run_reconfigure_skip_steady_state_cleans_nothing(setup_core: Path) -> None:
"""Cleanup belongs to the flip; a reconfigure of a skip tree touches nothing."""
_setup_build(setup_core)
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
)
marker = build / "bootloader"
marker.mkdir()
with (
patch.object(toolchain, "_skip_bootloader", return_value=True),
patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"),
patch.object(toolchain, "_tool_env", return_value={}),
patch.object(toolchain, "run_build_tool", return_value=0),
patch.object(toolchain, "_idf_py") as mock_idf_py,
):
mock_idf_py.return_value.binary_dir_arg = False
assert toolchain.run_reconfigure() == 0
assert marker.exists()
def test_create_factory_bin_full_mode_needs_flasher_args(setup_core: Path) -> None:
"""Past the skip gate, a full tree without flasher_args fails as before."""
_setup_build(setup_core)
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n"
)
assert toolchain.create_factory_bin() is False
def test_missing_image_hint_names_the_flag(setup_core: Path) -> None:
_setup_build(setup_core)
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
assert toolchain.missing_image_hint() is None # stock tree
(build / "CMakeCache.txt").write_text(
f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
)
assert "--skip-bootloader" in toolchain.missing_image_hint()
+104
View File
@@ -1548,6 +1548,7 @@ class MockArgs:
partition_table: bool = False
bootloader: bool = False
states: bool | None = None
device: list[str] | None = None
def test_upload_program_serial_esp32(
@@ -5357,6 +5358,28 @@ def _setup_build_info_test(
return build_info_path, firmware_path
def test_compile_program_warns_and_ignores_skip_bootloader_elsewhere(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A full build is always safe; orchestrators cannot see YAML overrides."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device")
CORE.skip_bootloader = True
config: dict[str, Any] = {CONF_ESPHOME: {CONF_NAME: "test_device"}}
with (
patch(
"esphome.components.esp8266.check_rosetta",
side_effect=EsphomeError("stop here"),
),
pytest.raises(EsphomeError, match="stop here"),
caplog.at_level("INFO"),
):
compile_program(MockArgs(), config)
assert "--skip-bootloader ignored" in caplog.text
assert CORE.skip_bootloader is False
def test_compile_program_esp8266_runs_rosetta_check(tmp_path: Path) -> None:
"""Test that compile_program runs the Rosetta preflight for ESP8266 targets."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device")
@@ -6846,6 +6869,70 @@ def test_upload_using_esptool_arduino_toolchain(
assert cmd_list[firmware_offset_idx + 1] == str(factory)
def test_upload_using_esptool_skip_bootloader_tree_names_the_flag(
tmp_path: Path,
) -> None:
"""A serial flash needs the factory image the skip flag did not build."""
setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test")
CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"}
CORE.toolchain = Toolchain.ESP_IDF
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
from esphome.espidf import toolchain as espidf_toolchain
(build / "CMakeCache.txt").write_text(
f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
)
config = {CONF_ESPHOME: {"platformio_options": {}}}
with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"):
upload_using_esptool(config, "/dev/ttyUSB0", None, None)
def test_command_run_rejects_serial_device_with_skip_bootloader(
tmp_path: Path,
) -> None:
"""The compile could never be flashed over serial; fail before it runs."""
setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test")
CORE.skip_bootloader = True
CORE.toolchain = Toolchain.ESP_IDF
args = MockArgs(device=["/dev/ttyUSB0"])
with patch("esphome.__main__.write_cpp") as mock_write:
assert command_run(args, {}) == 1
mock_write.assert_not_called()
def test_upload_program_ota_bootloader_skip_tree_names_the_flag(
mock_run_ota: Mock,
mock_get_port_type: Mock,
tmp_path: Path,
) -> None:
"""OTA --bootloader on a skip tree errors before picking the binary."""
setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test")
CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"}
CORE.toolchain = Toolchain.ESP_IDF
build = CORE.relative_build_path("build")
build.mkdir(parents=True)
from esphome.espidf import toolchain as espidf_toolchain
(build / "CMakeCache.txt").write_text(
f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n"
)
mock_get_port_type.return_value = "NETWORK"
config = {
CONF_OTA: [
{
CONF_PLATFORM: CONF_ESPHOME,
CONF_PORT: 3232,
"allow_partition_access": True,
}
]
}
args = MockArgs(bootloader=True)
with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"):
upload_program(config, args, ["192.168.1.100"])
mock_run_ota.assert_not_called()
@pytest.mark.parametrize(
("toolchain", "pio_project_written"),
[
@@ -7073,6 +7160,23 @@ def test_compile_program_espidf_idedata_success_is_silent(
assert "idedata" not in caplog.text
def test_compile_program_espidf_failed_factory_bin_fails_the_build() -> None:
"""A compile whose factory image could not be produced must not exit 0."""
CORE.toolchain = Toolchain.ESP_IDF
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp32",
KEY_TARGET_FRAMEWORK: "esp-idf",
}
with (
patch("esphome.espidf.toolchain.run_compile", return_value=0),
patch("esphome.espidf.toolchain.create_factory_bin", return_value=False),
patch("esphome.espidf.toolchain.create_ota_bin") as mock_ota,
patch("esphome.__main__._check_and_emit_build_info"),
):
assert compile_program(MagicMock(), {}) == 1
mock_ota.assert_not_called()
def test_compile_program_espidf_idedata_none_warns(
caplog: pytest.LogCaptureFixture,
) -> None: