diff --git a/esphome/__main__.py b/esphome/__main__.py index d68319e63a..707ad03609 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Protocol # cause them to be loaded before external components are processed, resulting # in the built-in version being used instead of the external component one. from esphome import const, platform_hooks +from esphome.build_helpers.native import native_backend from esphome.const import ( ALLOWED_NAME_CHARS, ARGUMENT_HELP_DEVICE, @@ -817,7 +818,9 @@ def write_cpp_file() -> int: from esphome.build_gen import espidf espidf.write_project() - else: + elif not CORE.using_native_toolchain: + # Other native builds generate their project at compile time; + # never write a platformio.ini for them from esphome.build_gen import platformio platformio.write_project() @@ -859,20 +862,14 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: toolchain.create_factory_bin() toolchain.create_ota_bin() toolchain.create_elf_copy() - from esphome.build_helpers.idedata import IDEDATA_BEST_EFFORT_ERRORS + from esphome.build_helpers.idedata import warn_if_idedata_missing - try: - if toolchain.get_idedata() is None: - _LOGGER.warning("No idedata was generated for this build") - except IDEDATA_BEST_EFFORT_ERRORS as err: - # The firmware already built; an idedata failure must not fail - # a successful build. - _LOGGER.warning( - "Could not generate idedata: %s (IDE, clang-tidy, and " - "memory-analysis data will be unavailable for this build)", - err, - ) - _LOGGER.debug("Idedata failure detail", exc_info=True) + warn_if_idedata_missing(toolchain.get_idedata) + elif CORE.using_native_toolchain: + raise EsphomeError( + f"Toolchain '{CORE.toolchain.value}' resolved but no platform " + "backend claimed the build" + ) else: from esphome.platformio import toolchain @@ -975,12 +972,15 @@ def upload_using_esptool( if file is not None: flash_images = [FlashImage(path=file, offset="0x0")] - elif CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain - - flash_images = [ - FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0") - ] + elif (native := native_backend()) is not None: + # Every native backend supplies its own 0x0 flash image (bootloader + # and partitions included where the target needs them) + image = native.get_factory_firmware_path() + if not image.is_file(): + raise EsphomeError( + f"{image} does not exist; compile the configuration first" + ) + flash_images = [FlashImage(path=image, offset="0x0")] else: from esphome.platformio import toolchain @@ -1970,12 +1970,12 @@ def command_update_all(args: ArgsProtocol) -> int | None: def command_idedata(args: ArgsProtocol, config: ConfigType) -> int: import json - if CORE.using_toolchain_esp_idf: - # Native ESP-IDF derives idedata from the build's compile_commands.json, - # so the configuration must already be compiled. - from esphome.espidf import toolchain as espidf_toolchain + native_toolchain = native_backend() - idedata = espidf_toolchain.get_idedata() + if native_toolchain is not None: + # Native toolchains derive idedata from the build's + # compile_commands.json, so the configuration must already be compiled. + idedata = native_toolchain.get_idedata() if idedata is None: _LOGGER.error( "No idedata available; compile the configuration first", @@ -2014,6 +2014,17 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int: from esphome.analyze_memory.cli import MemoryAnalyzerCLI from esphome.analyze_memory.ram_strings import RamStringsAnalyzer + # Refuse an unsupported toolchain before paying for a full compile + native_toolchain = native_backend() + if native_toolchain is None and not CORE.using_toolchain_platformio: + _LOGGER.error( + "analyze-memory is not supported with the '%s' toolchain on %s; " + "re-run with --toolchain platformio", + CORE.toolchain.value if CORE.toolchain else "unresolved", + CORE.target_platform, + ) + return 1 + # Always compile to ensure fresh data (fast if no changes - just relinks) exit_code = write_cpp(config) if exit_code != 0: @@ -2025,13 +2036,30 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int: # Get idedata for analysis idedata = None - if CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain + if native_toolchain is not None: + objdump = native_toolchain.get_objdump_path() + readelf = native_toolchain.get_readelf_path() + for tool in (objdump, readelf): + if not tool.is_file(): + # The analyzer would silently fall back to host + # binutils, which cannot read the target ELF + _LOGGER.error( + "%s is missing; the toolchain install may be incomplete " + "(recompile, or run 'esphome clean-all' if it persists)", + tool, + ) + return 1 + objdump_path = str(objdump) + readelf_path = str(readelf) - objdump_path = str(toolchain.get_objdump_path()) - readelf_path = str(toolchain.get_readelf_path()) - - firmware_elf = toolchain.get_elf_path() + firmware_elf = native_toolchain.get_elf_path() + if not firmware_elf.is_file(): + # The analyzer swallows tool failures, so a missing ELF would + # produce an exit-0 zeroed report + _LOGGER.error( + "%s is missing; compile the configuration first", firmware_elf + ) + return 1 else: from esphome.platformio import toolchain diff --git a/esphome/analyze_memory/toolchain.py b/esphome/analyze_memory/toolchain.py index 19041ac807..b64af5ea5b 100644 --- a/esphome/analyze_memory/toolchain.py +++ b/esphome/analyze_memory/toolchain.py @@ -68,12 +68,16 @@ def idedata_candidates(build_path: Path) -> list[Path]: The candidate idedata JSON paths, most specific first """ name = build_path.name + data_dir = build_path.parent.parent / "idedata" + # Native backends suffix the cache by toolchain (.arduino.json) + suffixed = sorted(data_dir.glob(f"{name}.*.json")) if data_dir.is_dir() else [] return [ # In .pioenvs for test builds build_path / ".pioenvs" / name / "idedata.json", # Both toolchains cache it in the data dir, which holds this build dir: # /idedata/.json next to /build/ build_path.parent.parent / "idedata" / f"{name}.json", + *suffixed, # Regular builds, invoked from the config dir or from anywhere Path.cwd() / ".esphome" / "idedata" / f"{name}.json", Path.home() / ".esphome" / "idedata" / f"{name}.json", diff --git a/esphome/arduino8266/toolchain.py b/esphome/arduino8266/toolchain.py new file mode 100644 index 0000000000..339fb44337 --- /dev/null +++ b/esphome/arduino8266/toolchain.py @@ -0,0 +1,266 @@ +"""Native Arduino ESP8266 build driver (the PlatformIO ``run`` equivalent).""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +import subprocess +from typing import TYPE_CHECKING + +from esphome.build_helpers.ccache import resolve_ccache_path +from esphome.const import ( + CONF_COMPILE_PROCESS_LIMIT, + CONF_ESPHOME, + KEY_CORE, + KEY_FRAMEWORK_VERSION, +) +from esphome.core import CORE, EsphomeError +from esphome.helpers import write_file +from esphome.types import ConfigType + +if TYPE_CHECKING: + from esphome.arduino8266.framework import InstalledPaths + +_LOGGER = logging.getLogger(__name__) + +# ESP8266 user RAM (matches upload.maximum_ram_size in every board manifest) +_MAX_RAM_SIZE = 81920 + + +def _warn_ignored_platformio_options() -> None: + """Warn for component-added platformio options the native build drops.""" + from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS + + consumed = NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS + for key in sorted(CORE.platformio_options or {}): + if key not in consumed: + _LOGGER.warning( + "platformio_options->%s is ignored when building with the " + "native 'arduino' toolchain", + key, + ) + + +_RAM_SECTIONS = (".data", ".rodata", ".bss") +_FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata") + + +def get_build_dir() -> Path: + return CORE.relative_pioenvs_path(CORE.name) + + +def get_elf_path() -> Path: + return get_build_dir() / "firmware.elf" + + +def _toolchain_tool(name: str) -> Path: + # Imported here, not at module scope: the serial upload/logs fast path + # resolves this module for its artifact paths alone, and framework + # pulls in the whole package-download stack + from esphome.arduino8266 import framework + + return framework.toolchain_tool(framework.get_toolchain_path(), name) + + +def get_factory_firmware_path() -> Path: + """The image to serial-flash at 0x0 (same bytes as firmware.bin: the + 8266 factory copy exists for artifact-contract parity, not content).""" + return get_build_dir() / "firmware.factory.bin" + + +def get_addr2line_path() -> Path: + return _toolchain_tool("addr2line") + + +def get_objdump_path() -> Path: + return _toolchain_tool("objdump") + + +def get_readelf_path() -> Path: + return _toolchain_tool("readelf") + + +def run_compile(config: ConfigType, verbose: bool) -> int: + from esphome.arduino8266 import framework + from esphome.build_gen import arduino8266 as build_gen + + _warn_ignored_platformio_options() + paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]) + # Resolved once: the probe is not free and three consumers need it + ccache = resolve_ccache_path() + build_gen.write_project(paths, ccache) + + build_dir = get_build_dir() + env = framework.get_build_env(paths.toolchain, ccache) + + # The DB is a pure function of build.ninja; regenerate when it is older + compdb = build_dir / "compile_commands.json" + ninja_file = build_dir / "build.ninja" + if not compdb.is_file() or compdb.stat().st_mtime < ninja_file.stat().st_mtime: + _write_compile_commands(paths.ninja, build_dir, env) + + cmd = [str(paths.ninja)] + if verbose: + cmd.append("-v") + if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT): + cmd += ["-j", str(jobs)] + # Explicit targets: a generator defect that drops them fails loudly + # instead of a green no-op run leaving stale artifacts in place + targets = ["firmware.factory.bin", "firmware.ota.bin"] + cmd += targets + + # cwd, not -C: drops ninja's "Entering directory" banner + _LOGGER.debug("Running: %s", " ".join(cmd)) + rc = subprocess.run( + cmd, cwd=build_dir, env=env, check=False, close_fds=False + ).returncode + if rc != 0: + return rc + + # ninja already refused missing targets; existence covers a rule that + # ran but wrote elsewhere + build_dir_artifacts = ( + get_elf_path(), + build_dir / "firmware.bin", + get_factory_firmware_path(), + build_dir / "firmware.ota.bin", + ) + for artifact in build_dir_artifacts: + if not artifact.is_file(): + _LOGGER.error("Build produced no %s", artifact) + return 1 + + if not _print_size_summary(build_dir, paths): + # Cause already warned; name the consequence for CI harnesses + _LOGGER.warning("Firmware size summary unavailable for this build") + from esphome.build_helpers.idedata import warn_if_idedata_missing + + warn_if_idedata_missing(lambda: get_idedata(ccache)) + return 0 + + +def _write_compile_commands( + ninja_path: Path, build_dir: Path, env: dict[str, str] +) -> None: + compdb = build_dir / "compile_commands.json" + result = subprocess.run( + [str(ninja_path), "-C", str(build_dir), "-t", "compdb", "c", "cxx", "asm"], + env=env, + capture_output=True, + text=True, + check=False, + close_fds=False, + ) + if result.returncode != 0: + # Drop any stale database so consumers (IDE integration, clang-tidy, + # the memory analyzer) can't silently read outdated data. + compdb.unlink(missing_ok=True) + raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}") + try: + entries = json.loads(result.stdout) + except ValueError as err: + compdb.unlink(missing_ok=True) + raise EsphomeError( + f"ninja produced an unparsable compile database: {err} " + f"(output starts {result.stdout[:120]!r})" + ) from err + if not entries: + # compdb exits 0 with [] for unknown rule names; a renamed compile + # rule must fail the build, not silently strand every consumer + compdb.unlink(missing_ok=True) + raise EsphomeError( + "ninja produced an empty compile database; the generator's rule " + "names no longer match" + ) + write_file(compdb, result.stdout) + + +def _parse_app_size(build_dir: Path, paths: InstalledPaths) -> int | None: + """Read the app flash budget (irom0_0_seg length) from the linker script.""" + from esphome.build_gen.arduino8266 import get_flash_ld_path + from esphome.components.esp8266.build_surgery import segment_length + + # Warnings, not debug: without the app size the Flash summary line is + # dropped and CI's memory-impact extraction loses its flash metric. + ld_path = get_flash_ld_path(build_dir, paths) + try: + ld_text = ld_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as err: + # A corrupt script degrades the same way, never aborts the build + _LOGGER.warning("Cannot read linker script for the Flash summary: %s", err) + return None + if not (app_size := segment_length(ld_text, "irom0_0_seg")): + _LOGGER.warning("No usable irom0_0_seg in %s; skipping Flash summary", ld_path) + return None + return app_size + + +def _print_size_summary(build_dir: Path, paths: InstalledPaths) -> bool: + """Print the RAM/Flash lines ``ci_memory_impact_extract.py`` parses; + False when skipped.""" + from esphome.arduino8266.framework import toolchain_tool + from esphome.build_helpers.size_summary import print_size_line + + try: + result = subprocess.run( + [ + str(toolchain_tool(paths.toolchain, "size")), + "-A", + "-d", + str(build_dir / "firmware.elf"), + ], + capture_output=True, + text=True, + check=True, + close_fds=False, + ) + except (OSError, subprocess.CalledProcessError) as err: + # The summary is a bonus artifact like idedata; a truncated + # toolchain extraction must not discard an already-linked build + _LOGGER.warning("Could not summarize firmware size: %s", err) + return False + # -d prints decimal sizes; anything else trips the missing-sections guard + sections = { + parts[0]: int(parts[1]) + for line in result.stdout.splitlines() + if (parts := line.split())[:1] and parts[0].startswith(".") and len(parts) >= 2 + if parts[1].isdigit() + } + if missing := set(_RAM_SECTIONS + _FLASH_SECTIONS) - set(sections): + # A defaulted 0 would print a confidently wrong total for CI's metric + _LOGGER.warning( + "Size output is missing section(s) %s; skipping the size summary", + ", ".join(sorted(missing)), + ) + return False + # Resolve the flash budget before printing: a RAM line without its + # Flash line would skew CI's memory-impact extraction + app_size = _parse_app_size(build_dir, paths) + if not app_size: + return False + ram = sum(sections[s] for s in _RAM_SECTIONS) + flash = sum(sections[s] for s in _FLASH_SECTIONS) + print_size_line("RAM", ram, _MAX_RAM_SIZE) + print_size_line("Flash", flash, app_size) + return True + + +def get_idedata(ccache: str | None = None) -> dict | None: + """Derive idedata from the build's compile_commands.json (same + contract as ``espidf.toolchain.get_idedata``).""" + from esphome.build_helpers.idedata import load_or_build_idedata + + # A disabled ccache resolves to None without spawning anything, so + # re-resolving here costs nothing when the caller has no answer + launcher = ccache or resolve_ccache_path() + return load_or_build_idedata( + get_build_dir() / "compile_commands.json", + get_elf_path(), + # Suffixed so a platformio->arduino->platformio round trip on one + # config never serves the other toolchain's cache shape + CORE.relative_internal_path("idedata", f"{CORE.name}.arduino.json"), + # The compile DB's commands carry the same ccache prefix the ninja + # rules were generated with + launcher=str(launcher) if launcher else None, + ) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index b6ac88f123..285379ceea 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -1132,7 +1132,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: "rule elf2bin", # --flash_size deliberately stays board-derived, as under # PlatformIO (which reads upload.maximum_size, not the ldscript). - f" command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out", + # -W: the framework's own elf2bin.py trips SyntaxWarning on 3.12+. + f" command = $python -W ignore::SyntaxWarning {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out", " description = BIN $out", "rule copy", " command = $python $buildtool copy $in $out", diff --git a/esphome/build_helpers/idedata.py b/esphome/build_helpers/idedata.py index 038fe64970..98ee804403 100644 --- a/esphome/build_helpers/idedata.py +++ b/esphome/build_helpers/idedata.py @@ -11,6 +11,7 @@ consumers (IDE integration, clang-tidy) expect: from __future__ import annotations +from collections.abc import Callable import json import logging import os @@ -21,6 +22,8 @@ import subprocess from esphome.core import EsphomeError from esphome.helpers import write_file +_LOGGER = logging.getLogger(__name__) + # Everything idedata generation may raise after a successful link; idedata # is a bonus artifact, so consumers warn instead of failing the build IDEDATA_BEST_EFFORT_ERRORS = ( @@ -31,7 +34,21 @@ IDEDATA_BEST_EFFORT_ERRORS = ( ValueError, ) -_LOGGER = logging.getLogger(__name__) + +def warn_if_idedata_missing(get_idedata: Callable[[], dict | None]) -> None: + """Run an idedata generator, downgrading any failure to a warning: + the firmware already built.""" + try: + if get_idedata() is None: + _LOGGER.warning("No idedata was generated for this build") + except IDEDATA_BEST_EFFORT_ERRORS as err: + _LOGGER.warning( + "Could not generate idedata: %s (IDE, clang-tidy, and " + "memory-analysis data will be unavailable for this build)", + err, + ) + _LOGGER.debug("Idedata failure detail", exc_info=True) + # C++ translation-unit suffixes used to identify ESPHome source files. _CXX_SUFFIXES = (".cpp", ".cc") diff --git a/esphome/build_helpers/native.py b/esphome/build_helpers/native.py new file mode 100644 index 0000000000..cde526b803 --- /dev/null +++ b/esphome/build_helpers/native.py @@ -0,0 +1,34 @@ +"""Resolution of the native (non-PlatformIO) build backend for a config. + +Kept deliberately light: the serial upload and logs fast path resolves a +backend for its artifact paths alone, so importing this must not pull in a +platform component package or the backend itself. +""" + +from __future__ import annotations + +import importlib +from types import ModuleType + +from esphome.const import Toolchain +from esphome.core import CORE, EsphomeError + +# Native build backend per (target platform, toolchain) +NATIVE_TOOLCHAIN_MODULES = { + ("esp32", Toolchain.ESP_IDF): "esphome.espidf.toolchain", + ("esp8266", Toolchain.ARDUINO): "esphome.arduino8266.toolchain", +} + + +def native_backend() -> ModuleType | None: + """The native build backend module for the resolved toolchain.""" + if not CORE.using_native_toolchain: + return None + key = (CORE.target_platform, CORE.toolchain) + if (module_path := NATIVE_TOOLCHAIN_MODULES.get(key)) is None: + # Degrading to the PlatformIO path would build with the wrong backend + raise EsphomeError( + f"Toolchain '{CORE.toolchain.value}' has no native build backend " + f"module for platform {CORE.target_platform}" + ) + return importlib.import_module(module_path) diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 56509b9fab..808d7ca7a9 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -5,6 +5,7 @@ import re import subprocess from typing import Any +from esphome.build_helpers.native import native_backend import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import ( @@ -14,6 +15,7 @@ from esphome.const import ( CONF_FRAMEWORK, CONF_PLATFORM_VERSION, CONF_SOURCE, + CONF_TOOLCHAIN, CONF_VERSION, KEY_CORE, KEY_FRAMEWORK_VERSION, @@ -21,6 +23,7 @@ from esphome.const import ( KEY_TARGET_PLATFORM, PLATFORM_ESP8266, ThreadModel, + Toolchain, ) from esphome.core import ( CORE, @@ -43,6 +46,7 @@ from .const import ( CONF_ENABLE_SERIAL1, CONF_RESTORE_FROM_FLASH, KEY_BOARD, + KEY_DECODE_TOOLS, KEY_ESP8266, KEY_PIN_INITIAL_STATES, KEY_SCANF_FLOAT, @@ -105,6 +109,47 @@ def set_core_data(config: ConfigType) -> ConfigType: return config +_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ARDUINO) +_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS) +_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.PLATFORMIO) + + +def _validate_native_toolchain(config: ConfigType) -> ConfigType: + """Constraints of the native (non-PlatformIO) Arduino toolchain.""" + if not CORE.using_toolchain_arduino: + return config + from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION + + conf = config[CONF_FRAMEWORK] + version = cv.Version.parse(conf[CONF_VERSION]) + if version < MIN_FRAMEWORK_VERSION: + raise cv.Invalid( + "'toolchain: arduino' requires framework version " + f"{MIN_FRAMEWORK_VERSION} or newer" + ) + # platform_version is a PlatformIO concept; drop it, warning when a + # custom pin is discarded + if ( + conf.pop(CONF_PLATFORM_VERSION, _ARDUINO_4_PLATFORM_SPEC) + != _ARDUINO_4_PLATFORM_SPEC + ): + _LOGGER.warning( + "'platform_version' is ignored by 'toolchain: arduino'; the native " + "toolchain downloads the framework and compiler directly" + ) + if conf[CONF_SOURCE] != _format_framework_arduino_version(version): + raise cv.Invalid( + "'toolchain: arduino' does not support a custom framework source; " + "use 'toolchain: platformio'" + ) + if config[CONF_BOARD] not in BOARDS: + raise cv.Invalid( + f"Board '{config[CONF_BOARD]}' is not supported by " + "'toolchain: arduino'; use 'toolchain: platformio'" + ) + return config + + def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: """Binary-download entries for a built ESP8266 firmware. @@ -134,7 +179,7 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 # Same encoding the native toolchain uses for its package download, so a - # version bump cannot drift between the two paths. + # custom-source check against this value cannot drift from what it fetches. from esphome.arduino8266.framework import framework_package_version try: @@ -196,7 +241,7 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: platform_version = value.get(CONF_PLATFORM_VERSION) if platform_version is None: if version >= cv.Version(3, 1, 0): - platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) + platform_version = _ARDUINO_4_PLATFORM_SPEC else: platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version @@ -219,6 +264,10 @@ def _parse_platform_version(value: Any) -> str: return value +# The platform_version derived for every core >= 3.1.0 config +_ARDUINO_4_PLATFORM_SPEC = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) + + ARDUINO_FRAMEWORK_SCHEMA = cv.All( cv.Schema( { @@ -251,11 +300,13 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_SERIAL1): cv.boolean, cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean, cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean, + cv.Optional( + CONF_TOOLCHAIN, visibility=cv.Visibility.ADVANCED + ): _validate_toolchain, } ), - # Until the native toolchain lands, PlatformIO is the only backend; - # reject a --toolchain this platform cannot serve yet. - cv.require_platformio_toolchain("ESP8266"), + _resolve_toolchain, + _validate_native_toolchain, set_core_data, ) @@ -295,12 +346,13 @@ def _choose_ld_script(board: str) -> str: @coroutine_with_priority(CoroPriority.PLATFORM) async def to_code(config: ConfigType) -> None: + use_platformio = CORE.using_toolchain_platformio cg.add(esp8266_ns.setup_preferences()) - cg.add_platformio_option("lib_ldf_mode", "off") - cg.add_platformio_option("lib_compat_mode", "strict") - - cg.add_platformio_option("board", config[CONF_BOARD]) + if use_platformio: + cg.add_platformio_option("lib_ldf_mode", "off") + cg.add_platformio_option("lib_compat_mode", "strict") + cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_build_flag("-DUSE_ESP8266") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) @@ -318,29 +370,30 @@ async def to_code(config: ConfigType) -> None: # The native generator reads the same decision (KEY_SCANF_FLOAT) CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = bool(enable_scanf_float) - - extra_scripts = [ - "pre:ccache.py", - "pre:testing_mode.py", - "pre:exclude_updater.py", - "pre:exclude_waveform.py", - "pre:relocate_ratetable.py", - ] - if not enable_scanf_float: - extra_scripts.append("pre:remove_float_scanf.py") - extra_scripts.append("post:post_build.py") - cg.add_platformio_option("extra_scripts", extra_scripts) + if use_platformio: + extra_scripts = [ + "pre:ccache.py", + "pre:testing_mode.py", + "pre:exclude_updater.py", + "pre:exclude_waveform.py", + "pre:relocate_ratetable.py", + ] + if not enable_scanf_float: + extra_scripts.append("pre:remove_float_scanf.py") + extra_scripts.append("post:post_build.py") + cg.add_platformio_option("extra_scripts", extra_scripts) conf = config[CONF_FRAMEWORK] - cg.add_platformio_option("framework", "arduino") cg.add_build_flag("-DUSE_ARDUINO") cg.add_build_flag("-DUSE_ESP8266_FRAMEWORK_ARDUINO") cg.add_build_flag("-Wno-nonnull-compare") - cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION]) - cg.add_platformio_option( - "platform_packages", - [f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"], - ) + if use_platformio: + cg.add_platformio_option("framework", "arduino") + cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION]) + cg.add_platformio_option( + "platform_packages", + [f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"], + ) # Default for platformio is LWIP2_LOW_MEMORY with: # - MSS=536 @@ -369,8 +422,8 @@ async def to_code(config: ConfigType) -> None: # Force-include inline std::__throw_* overrides so GCC dead-strips the unused # libstdc++ error message strings (e.g. "basic_string::_M_create") from DRAM. - # See throw_stubs.h for details. Must be prepended before , so this - # uses build_src_flags with -include. + # See throw_stubs.h. Unconditional: the native build generator reads + # the same option, keeping one source of truth. cg.add_platformio_option( "build_src_flags", "-include esphome/components/esp8266/throw_stubs.h" ) @@ -400,6 +453,8 @@ async def to_code(config: ConfigType) -> None: # implementation in the Arduino ESP8266 core. cg.add_build_flag("-Wl,--wrap=millis") + # Unconditional: the native build generator reads the same option, + # keeping one source of truth cg.add_platformio_option("board_build.flash_mode", config[CONF_BOARD_FLASH_MODE]) ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] @@ -408,7 +463,7 @@ async def to_code(config: ConfigType) -> None: cg.RawExpression(f"VERSION_CODE({ver.major}, {ver.minor}, {ver.patch})"), ) - if config[CONF_BOARD] in BOARDS: + if use_platformio and config[CONF_BOARD] in BOARDS: cg.add_platformio_option( "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) ) @@ -449,8 +504,24 @@ async def finalize_serial_config() -> None: cg.add_build_flag("-DNO_GLOBAL_SERIAL1") +# Called by __main__.compile_program; returning False falls through to the +# PlatformIO toolchain. +def run_compile(args, config: ConfigType) -> bool: + # Positive check: the native backend only runs when explicitly resolved + toolchain = native_backend() + if toolchain is None: + return False + if toolchain.run_compile(config, CORE.verbose) != 0: + raise EsphomeError("ESP8266 native build failed") + return True + + # Called by writer.py def copy_files() -> None: + # Native builds skip the PlatformIO extra scripts; the build generator + # carries their logic + if CORE.using_toolchain_arduino: + return dir = Path(__file__).parent for script in ( "post_build", @@ -507,22 +578,54 @@ ESP8266_EXCEPTION_CODES = { } -def _decode_pc(config: ConfigType, addr: str) -> None: +def _resolve_decode_tools(config: ConfigType) -> tuple[str, str] | None: + """``(addr2line, elf)`` for this build, or None after warning why.""" + if (native_toolchain := native_backend()) is not None: + addr2line = native_toolchain.get_addr2line_path() + elf = native_toolchain.get_elf_path() + for path in (addr2line, elf): + if not path.is_file(): + _LOGGER.warning("Cannot decode crash addresses: %s missing", path) + return None + return str(addr2line), str(elf) from esphome.platformio import toolchain idedata = toolchain.get_idedata(config) if not idedata.addr2line_path or not idedata.firmware_elf_path: - _LOGGER.debug("decode_pc no addr2line") + _LOGGER.warning("Cannot decode crash addresses: no addr2line or ELF in idedata") + return None + return idedata.addr2line_path, idedata.firmware_elf_path + + +def _decode_tools(config: ConfigType) -> tuple[str, str] | None: + """Resolved once per run: the tools are a property of the build, not of + the address, so a stack dump cannot repeat the failure warning.""" + data = CORE.data.setdefault(KEY_ESP8266, {}) + if KEY_DECODE_TOOLS not in data: + data[KEY_DECODE_TOOLS] = _resolve_decode_tools(config) + return data[KEY_DECODE_TOOLS] + + +def _decode_pc(config: ConfigType, addr: str, *, bulk: bool = False) -> None: + """Decode one crash address. ``bulk``: the caller is scanning every + 8-hex stack word, most of which are not code addresses -- unmappable + ones log at debug so real frames are not buried.""" + # Bulk stack words are ~150 per dump; only named registers warn + log = _LOGGER.debug if bulk else _LOGGER.warning + if (tools := _decode_tools(config)) is None: + log("Not decoded %s (no addr2line or ELF)", addr) return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + addr2line, elf = tools + command = [addr2line, "-pfiaC", "-e", elf, addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() - except Exception: # noqa: BLE001 # pylint: disable=broad-except + except Exception as err: # noqa: BLE001 # pylint: disable=broad-except + log("Could not decode crash address %s (%s)", addr, err) _LOGGER.debug("Caught exception for command %s", command, exc_info=1) return if "?? ??:0" in translation: - # Nothing useful + log("Not decoded %s (address not in %s)", addr, elf) return translation = translation.replace(" at ??:?", "").replace(":?", "") _LOGGER.warning("Decoded %s", translation) @@ -592,6 +695,6 @@ def process_stacktrace(config: ConfigType, line: str, backtrace_state: bool) -> if backtrace_state: for addr in re.finditer(STACKTRACE_ESP8266_BACKTRACE_PC_RE, line): - _decode_pc(config, addr.group()) + _decode_pc(config, addr.group(), bulk=True) return backtrace_state diff --git a/esphome/components/esp8266/const.py b/esphome/components/esp8266/const.py index ac2fe0aa3b..bf4dd1a358 100644 --- a/esphome/components/esp8266/const.py +++ b/esphome/components/esp8266/const.py @@ -19,6 +19,8 @@ KEY_SERIAL1_REQUIRED = "serial1_required" KEY_SCANF_FLOAT = "scanf_float" # Per-board flash-layout override consumed by board_ld_script() KEY_LDSCRIPT = "ldscript" +# Crash-decode tools, resolved once per run by __init__ +KEY_DECODE_TOOLS = "decode_tools" # esp8266 namespace is already defined by arduino, manually prefix esphome esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266") diff --git a/esphome/core/config.py b/esphome/core/config.py index 4d3fb83312..bb3f03d366 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -555,7 +555,13 @@ NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscri # that is stored rather than translated away. Consumed by the esp8266 native # backend (later in this chain) for its ignored-option warning; defined here # so it stays adjacent to the routing. -NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | {"lib_ignore"} +# build_src_flags and board_build.flash_mode are not user-routable, so +# not in the set above +NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | { + "lib_ignore", + "build_src_flags", + "board_build.flash_mode", +} @coroutine_with_priority(CoroPriority.FINAL) diff --git a/esphome/writer.py b/esphome/writer.py index 9ab5d644c5..bddb6a0f5f 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -609,10 +609,10 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False): _LOGGER.info("Deleting %s", idf_path) rmtree(idf_path) - # The idedata cache is derived from the build but lives under the data dir, - # not the build path, so it must be removed separately in both modes. - idedata_cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json") - if idedata_cache.is_file(): + # idedata caches live under the data dir, not the build path; globbed + # so a future backend suffix cannot drift out of clean-all + idedata_dir = CORE.relative_internal_path("idedata") + for idedata_cache in idedata_dir.glob(f"{CORE.name}*.json"): _LOGGER.info("Deleting %s", idedata_cache) idedata_cache.unlink() diff --git a/tests/unit_tests/components/esp8266/test_toolchain_validation.py b/tests/unit_tests/components/esp8266/test_toolchain_validation.py new file mode 100644 index 0000000000..951217c8a1 --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_toolchain_validation.py @@ -0,0 +1,197 @@ +"""Tests for the native (non-PlatformIO) toolchain config validation.""" + +from __future__ import annotations + +from collections.abc import Generator +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from esphome.components import esp8266 +from esphome.components.esp8266 import ( + ARDUINO_FRAMEWORK_SCHEMA, + _resolve_toolchain, + _validate_native_toolchain, +) +import esphome.config_validation as cv +from esphome.const import ( + CONF_BOARD, + CONF_FRAMEWORK, + CONF_PLATFORM_VERSION, + CONF_SOURCE, + CONF_TOOLCHAIN, + CONF_VERSION, + KEY_CORE, + KEY_TARGET_PLATFORM, + PLATFORM_ESP8266, + Toolchain, +) +from esphome.core import CORE, EsphomeError +from esphome.types import ConfigType + + +@pytest.fixture(autouse=True) +def _arduino_toolchain() -> Generator[None]: + # The suite-wide reset_core fixture clears both after each test; the + # shared backend resolver reads the platform as well as the toolchain, + # and the decode-tool cache lives in CORE.data + CORE.toolchain = Toolchain.ARDUINO + CORE.data.setdefault(KEY_CORE, {})[KEY_TARGET_PLATFORM] = PLATFORM_ESP8266 + yield + + +def _config( + board: str = "nodemcuv2", + version: str = "3.1.2", + source: str | None = None, + platform_version: str | None = None, +) -> ConfigType: + framework: dict[str, str] = {CONF_VERSION: version} + if source is not None: + framework[CONF_SOURCE] = source + if platform_version is not None: + framework[CONF_PLATFORM_VERSION] = platform_version + # The real schema fills the source/platform_version defaults, so these + # tests validate against what config validation actually emits + return { + CONF_FRAMEWORK: ARDUINO_FRAMEWORK_SCHEMA(framework), + CONF_BOARD: board, + } + + +def test_valid_config_passes() -> None: + config = _config() + assert _validate_native_toolchain(config) is config + + +def test_platformio_toolchain_skips_checks() -> None: + # 3.0.2 is pio-legal (>= the global 3.0.0 floor) but below the native + # toolchain's own 3.1.1 floor; the bogus board only the native path checks + CORE.toolchain = Toolchain.PLATFORMIO + config = _config(board="not_a_board", version="3.0.2") + assert _validate_native_toolchain(config) is config + + +def test_version_below_floor_rejected() -> None: + # 3.1.0 has no registry package, so the native floor is 3.1.1 + with pytest.raises(cv.Invalid, match="3.1.1 or newer"): + _validate_native_toolchain(_config(version="3.1.0")) + + +def test_version_at_floor_accepted() -> None: + _validate_native_toolchain(_config(version="3.1.1")) + + +def test_custom_platform_version_warns_and_is_dropped( + caplog: pytest.LogCaptureFixture, +) -> None: + config = _config(platform_version="platformio/espressif8266@4.0.1") + _validate_native_toolchain(config) + assert "'platform_version' is ignored" in caplog.text + assert CONF_PLATFORM_VERSION not in config[CONF_FRAMEWORK] + + +def test_default_platform_version_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + config = _config() + _validate_native_toolchain(config) + assert "'platform_version' is ignored" not in caplog.text + assert CONF_PLATFORM_VERSION not in config[CONF_FRAMEWORK] + + +def test_custom_source_rejected() -> None: + with pytest.raises(cv.Invalid, match="custom framework source"): + _validate_native_toolchain( + _config(source="https://github.com/esp8266/Arduino.git") + ) + + +def test_unsupported_board_rejected() -> None: + with pytest.raises(cv.Invalid, match="not supported by"): + _validate_native_toolchain(_config(board="not_a_board")) + + +def test_yaml_toolchain_key_resolves() -> None: + """The documented `toolchain: arduino` YAML key selects the native path.""" + CORE.toolchain = None + _resolve_toolchain({CONF_TOOLCHAIN: Toolchain.ARDUINO}) + assert CORE.toolchain == Toolchain.ARDUINO + assert CORE.using_toolchain_arduino + + +def test_yaml_toolchain_key_defaults_to_platformio() -> None: + CORE.toolchain = None + _resolve_toolchain({}) + assert CORE.toolchain == Toolchain.PLATFORMIO + + +def test_decode_pc_native_missing_tools_warns_once( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A stack dump of many addresses produces one missing-tool warning.""" + + with ( + patch( + "esphome.arduino8266.toolchain.get_addr2line_path", + return_value=tmp_path / "missing-addr2line", + ), + patch( + "esphome.arduino8266.toolchain.get_elf_path", + return_value=tmp_path / "missing.elf", + ), + ): + esp8266._decode_pc({}, "40201234") + esp8266._decode_pc({}, "40201238") + assert caplog.text.count("Cannot decode crash addresses") == 1 + + +def test_decode_pc_platformio_missing_tools_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + """The PlatformIO branch reports a missing addr2line/ELF at the same + warning level as the native one; raw undecoded addresses with no + stated reason are undiagnosable at default log level.""" + + CORE.toolchain = Toolchain.PLATFORMIO + idedata = SimpleNamespace(addr2line_path=None, firmware_elf_path=None) + with patch("esphome.platformio.toolchain.get_idedata", return_value=idedata): + esp8266._decode_pc({}, "40201234") + esp8266._decode_pc({}, "40201238") + assert caplog.text.count("Cannot decode crash addresses") == 1 + + +def test_resolve_toolchain_rejects_unsupported() -> None: + """ESP8266 rejects a CLI toolchain it cannot serve, like every platform.""" + + CORE.toolchain = Toolchain.SDK_NRF + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'sdk-nrf'"): + _resolve_toolchain({}) + + +def test_run_compile_platformio_falls_through() -> None: + """Under toolchain: platformio the hook returns False without touching + the native backend; this is what keeps existing users on PlatformIO.""" + CORE.toolchain = Toolchain.PLATFORMIO + with patch("esphome.arduino8266.toolchain.run_compile") as mock_native: + assert esp8266.run_compile(SimpleNamespace(), {}) is False + mock_native.assert_not_called() + + +def test_run_compile_arduino_failure_raises() -> None: + """A non-zero native build fails by name instead of returning success.""" + CORE.verbose = False + with ( + patch("esphome.arduino8266.toolchain.run_compile", return_value=1), + pytest.raises(EsphomeError, match="native build failed"), + ): + esp8266.run_compile(SimpleNamespace(), {}) + + +def test_copy_files_native_skips_platformio_scripts(tmp_path: Path) -> None: + """The native build writes no PlatformIO extra scripts.""" + CORE.build_path = tmp_path + esp8266.copy_files() + assert list(tmp_path.iterdir()) == [] diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 0db7039668..189c024b3a 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1480,12 +1480,3 @@ async def test_add_platformio_options_native_arduino( assert "board_build.ldscript is ignored" in caplog.text assert "'arduino' toolchain" in caplog.text assert "upload_speed" not in caplog.text - - -def test_esp8266_rejects_unsupported_cli_toolchain() -> None: - """Until the native backend lands, ESP8266 serves only PlatformIO.""" - from esphome.components.esp8266 import CONFIG_SCHEMA - - CORE.toolchain = Toolchain.ARDUINO - with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): - CONFIG_SCHEMA({"board": "nodemcuv2"}) diff --git a/tests/unit_tests/fixtures/lazy_imports/native_upload_fast_path.py b/tests/unit_tests/fixtures/lazy_imports/native_upload_fast_path.py new file mode 100644 index 0000000000..ee9751de80 --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/native_upload_fast_path.py @@ -0,0 +1,65 @@ +"""Run the native-toolchain serial-upload path and report heavy modules. + +Executed as a subprocess by test_lazy_imports.py: heavy module names come +in on argv, the ones found in sys.modules afterwards go out on stdout. +``upload_using_esptool`` dispatches native toolchains through a +toolchain-keyed table; if someone routes it back through the platform +component packages (esp32 or esp8266), this reports the leak. +""" + +import os +from pathlib import Path +import sys +import tempfile +from unittest.mock import patch + +from _leak_report import print_leaked_modules + +from esphome.__main__ import upload_using_esptool +from esphome.const import ( + CONF_ESPHOME, + KEY_CORE, + KEY_ESP32, + KEY_TARGET_PLATFORM, + KEY_VARIANT, + Toolchain, +) +from esphome.core import CORE + +# An ambient ESPHOME_USE_SUBPROCESS would route past the patched +# run_external_command into run_external_process and confuse the checks. +os.environ.pop("ESPHOME_USE_SUBPROCESS", None) + +config = {CONF_ESPHOME: {"platformio_options": {}}} + +with tempfile.TemporaryDirectory() as build_dir: + CORE.name = "leaktest" + CORE.build_path = build_dir + + for platform, toolchain, backend in ( + ("esp8266", Toolchain.ARDUINO, "esphome.arduino8266.toolchain"), + ("esp32", Toolchain.ESP_IDF, "esphome.espidf.toolchain"), + ): + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} + if platform == "esp32": + CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32S3"} + CORE.toolchain = toolchain + + import importlib + + image = importlib.import_module(backend).get_factory_firmware_path() + image.parent.mkdir(parents=True, exist_ok=True) + image.write_bytes(b"\x00") + + with patch("esphome.__main__.run_external_command", return_value=0) as mock_run: + rc = upload_using_esptool(config, "/dev/ttyUSB0", None, None) + + # Fail loudly if the upload path stopped doing its work; otherwise + # an empty leak list could just mean nothing ran. + if rc != 0: + sys.exit(f"upload_using_esptool({platform}) returned {rc}") + cmd = list(mock_run.call_args[0][1:]) + if str(image) not in [str(Path(c)) for c in cmd]: + sys.exit(f"native factory image did not reach esptool: {cmd}") + +print_leaked_modules() diff --git a/tests/unit_tests/test_arduino8266_toolchain.py b/tests/unit_tests/test_arduino8266_toolchain.py new file mode 100644 index 0000000000..a9c0546b4b --- /dev/null +++ b/tests/unit_tests/test_arduino8266_toolchain.py @@ -0,0 +1,548 @@ +"""Tests for esphome.arduino8266.toolchain (the ninja build driver).""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.arduino8266 import framework, toolchain +import esphome.config_validation as cv +from esphome.const import ( + CONF_COMPILE_PROCESS_LIMIT, + CONF_ESPHOME, + KEY_CORE, + KEY_FRAMEWORK_VERSION, +) +from esphome.core import CORE, EsphomeError + +_SIZE_OUTPUT = """\ +firmware.elf : +section size addr +.data 1924 1073643520 +.noinit 56 1073645444 +.text 496 1074790400 +.irom0.text 342804 1075843088 +.text1 27489 1074790896 +.rodata 2588 1073645504 +.bss 26504 1073648096 +Total 401861 +""" + + +@pytest.fixture(autouse=True) +def _setup_core(tmp_path: Path) -> None: + CORE.name = "test8266" + CORE.config_path = tmp_path / "test8266.yaml" + CORE.build_path = tmp_path + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 1, 2)} + # run_compile verifies the produced artifacts; give every test a build + # that "produced" them (tests for the guard delete them again). The + # manifest comes first: artifacts must not be older than build.ninja. + build_dir = CORE.relative_pioenvs_path("test8266") + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("# manifest") + for artifact in ( + "firmware.elf", + "firmware.bin", + "firmware.factory.bin", + "firmware.ota.bin", + ): + (build_dir / artifact).write_bytes(b"") + + +def _paths(tmp_path: Path) -> framework.InstalledPaths: + return framework.InstalledPaths( + framework=tmp_path / "framework", + toolchain=tmp_path / "toolchain", + ninja=tmp_path / "ninja", + ) + + +def test_path_getters(tmp_path: Path) -> None: + assert toolchain.get_build_dir() == CORE.relative_pioenvs_path("test8266") + assert toolchain.get_elf_path().name == "firmware.elf" + # The framework accessor owns the layout and the Windows suffix + suffix = ".exe" if os.name == "nt" else "" + assert toolchain.get_addr2line_path().name == f"xtensa-lx106-elf-addr2line{suffix}" + assert toolchain.get_objdump_path().name == f"xtensa-lx106-elf-objdump{suffix}" + assert toolchain.get_readelf_path().name == f"xtensa-lx106-elf-readelf{suffix}" + + +def test_run_compile_build_failure(tmp_path: Path) -> None: + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project"), + patch.object( + toolchain.subprocess, "run", return_value=MagicMock(returncode=2) + ) as mock_run, + patch.object(toolchain, "_write_compile_commands") as mock_compdb, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=True) == 2 + cmd = mock_run.call_args[0][0] + assert "-v" in cmd + # The compile database is generated before the build runs, so a failed + # build cannot leave a stale database behind. + mock_compdb.assert_called_once() + + +def test_run_compile_success(tmp_path: Path) -> None: + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project", return_value=False), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ) as mock_run, + patch.object(toolchain, "_write_compile_commands") as mock_compdb, + patch.object(toolchain, "_print_size_summary") as mock_size, + patch.object(toolchain, "get_idedata") as mock_idedata, + ): + rc = toolchain.run_compile( + {CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 4}}, verbose=False + ) + assert rc == 0 + call = next(c for c in mock_run.call_args_list if "ninja" in str(c[0][0][0])) + cmd = call[0][0] + # Explicit targets: a manifest missing them fails as "unknown target" + assert {"firmware.factory.bin", "firmware.ota.bin"} <= set(cmd) + assert cmd[cmd.index("-j") + 1] == "4" + # cwd, not -C, so ninja prints no "Entering directory" banner + assert "-C" not in cmd + assert call[1]["cwd"] is not None + mock_compdb.assert_called_once() + mock_size.assert_called_once() + mock_idedata.assert_called_once() + + +def test_run_compile_regenerates_stale_compdb(tmp_path: Path) -> None: + """An interrupted run can leave build.ninja newer than the compile DB; + mere existence must not skip regeneration.""" + build_dir = toolchain.get_build_dir() + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("") + compdb = build_dir / "compile_commands.json" + compdb.write_text("[]") + os.utime(compdb, ((build_dir / "build.ninja").stat().st_mtime - 5,) * 2) + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project", return_value=False), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock( + returncode=0, stdout="ninja: no work to do.\n", stderr="" + ), + ), + patch.object(toolchain, "_write_compile_commands") as mock_compdb, + patch.object(toolchain, "_print_size_summary"), + patch.object(toolchain, "get_idedata"), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + mock_compdb.assert_called_once() + + +def test_run_compile_missing_artifact_fails( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A zero ninja exit that produced no firmware must not be a green + build (size summary and idedata only warn).""" + (toolchain.get_build_dir() / "firmware.elf").unlink() + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project"), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "_print_size_summary") as mock_size, + patch.object(toolchain, "get_idedata"), + ): + rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) + assert rc == 1 + assert "Build produced no" in caplog.text + mock_size.assert_not_called() + + +def test_run_compile_warns_when_idedata_fails( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed idedata generation right after a successful build is visible, + not deferred to a misleading error in a later command.""" + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project"), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "_print_size_summary"), + patch.object(toolchain, "get_idedata", return_value=None), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert "No idedata was generated for this build" in caplog.text + + +def test_write_compile_commands(tmp_path: Path) -> None: + build_dir = tmp_path / "build" + build_dir.mkdir() + entries = '[{"file": "a.cpp", "command": "cc"}]\n' + with patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=entries), + ): + toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {}) + assert (build_dir / "compile_commands.json").read_text() == entries + + +@pytest.mark.parametrize( + ("stdout", "match"), + [ + ("[]\n", "empty compile database"), + # A parse failure names its cause, not the rule-name story + ("not json", "unparsable compile database.*not json"), + ], +) +def test_write_compile_commands_bad_db_raises( + tmp_path: Path, stdout: str, match: str +) -> None: + """An empty or unparsable compile database fails the build with its + actual cause and drops any stale database.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + (build_dir / "compile_commands.json").write_text("[stale]") + with ( + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=stdout), + ), + pytest.raises(EsphomeError, match=match), + ): + toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {}) + assert not (build_dir / "compile_commands.json").exists() + + +def test_write_compile_commands_failure_removes_stale_db(tmp_path: Path) -> None: + """A failed compdb run must not leave a stale database behind.""" + stale = tmp_path / "compile_commands.json" + stale.write_text("[]") + with ( + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=1, stderr="boom"), + ), + pytest.raises(EsphomeError, match="compile_commands"), + ): + toolchain._write_compile_commands(tmp_path / "ninja", tmp_path, {}) + assert not stale.exists() + + +def test_parse_app_size(tmp_path: Path) -> None: + ld = tmp_path / "eagle.flash.4m.ld" + ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0xfeff0\n}\n") + with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld): + assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) == 0xFEFF0 + + ld.write_text("MEMORY { }\n") + with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld): + assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None + + # A zero-length segment is bad data, not a budget; warn and drop it + ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0x0\n}\n") + with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld): + assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None + + with patch( + "esphome.build_gen.arduino8266.get_flash_ld_path", + return_value=tmp_path / "missing.ld", + ): + assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None + + +def test_print_size_summary(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with ( + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=_SIZE_OUTPUT), + ), + patch.object(toolchain, "_parse_app_size", return_value=1044464), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + out = capsys.readouterr().out + # Exact PlatformIO shape so script/ci_memory_impact_extract.py can parse it + assert "RAM: [==== ] 37.9% (used 31016 bytes from 81920 bytes)" in out + assert "Flash: [==== ] 35.9% (used 375301 bytes from 1044464 bytes)" in out + + +def test_print_size_summary_missing_size_tool_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A missing size binary degrades to a warning; the firmware already + linked and must not be discarded.""" + with patch.object( + toolchain.subprocess, "run", side_effect=FileNotFoundError("no size") + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + assert "Could not summarize firmware size" in caplog.text + + +def test_print_size_summary_no_app_size( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + with ( + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=_SIZE_OUTPUT), + ), + patch.object(toolchain, "_parse_app_size", return_value=None), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + out = capsys.readouterr().out + # Both lines are skipped together: a RAM line without Flash would skew + # CI's memory-impact sums across builds + assert out == "" + + +def test_print_size_summary_size_tool_failure( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, +) -> None: + with patch.object( + toolchain.subprocess, + "run", + side_effect=subprocess.CalledProcessError(1, "size", stderr="bad elf"), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + assert capsys.readouterr().out == "" + assert "Could not summarize firmware size" in caplog.text + + +def test_get_idedata_delegates(tmp_path: Path) -> None: + with ( + patch( + "esphome.build_helpers.idedata.load_or_build_idedata", + return_value={"cc_path": "x"}, + ) as mock_load, + patch.object(toolchain, "resolve_ccache_path", return_value="/cc/ccache"), + ): + assert toolchain.get_idedata() == {"cc_path": "x"} + compile_commands, elf, cache = mock_load.call_args[0] + assert compile_commands.name == "compile_commands.json" + assert elf.name == "firmware.elf" + assert cache.name == "test8266.arduino.json" + # The exact configured launcher string is passed for compile DB parsing + # (resolve_ccache_path returns a str, untouched on every platform) + assert mock_load.call_args.kwargs["launcher"] == "/cc/ccache" + + +def test_get_idedata_no_ccache(tmp_path: Path) -> None: + with ( + patch( + "esphome.build_helpers.idedata.load_or_build_idedata", return_value={} + ) as mock_load, + patch.object(toolchain, "resolve_ccache_path", return_value=None), + ): + toolchain.get_idedata() + assert mock_load.call_args.kwargs["launcher"] is None + + +def test_run_compile_skips_compdb_when_ninja_unchanged(tmp_path: Path) -> None: + """An unchanged build.ninja means the compile DB is already current.""" + build_dir = toolchain.get_build_dir() + build_dir.mkdir(parents=True, exist_ok=True) + # write_project (stubbed below) always leaves a build.ninja behind + (build_dir / "build.ninja").write_text("# manifest") + + def run(regenerate_expected: bool) -> None: + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project", return_value=False), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "_write_compile_commands") as mock_compdb, + patch.object(toolchain, "_print_size_summary"), + patch.object(toolchain, "get_idedata"), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert mock_compdb.called == regenerate_expected + + # Missing compile DB: regenerated even though build.ninja is unchanged + run(regenerate_expected=True) + # Present compile DB + unchanged build.ninja: skipped + (build_dir / "compile_commands.json").write_text("[]") + run(regenerate_expected=False) + + +def test_print_size_summary_unparsable_section( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A totals-relevant section that fails to parse must not produce a + confident wrong number; an irrelevant one is ignored.""" + bad = _SIZE_OUTPUT.replace(".bss 26504", ".bss abc") + with patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=bad), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + assert capsys.readouterr().out == "" + assert "missing section(s) .bss" in caplog.text + + caplog.clear() + harmless = _SIZE_OUTPUT + ".broken abc 0\n" + with ( + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=harmless), + ), + patch.object(toolchain, "_parse_app_size", return_value=1044464), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + assert "RAM:" in capsys.readouterr().out + assert not caplog.text + + +def test_print_size_summary_missing_section_skips_summary( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, +) -> None: + """A totals section absent from the output must not default to zero.""" + without_bss = "\n".join( + line for line in _SIZE_OUTPUT.splitlines() if ".bss" not in line + ) + with patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout=without_bss), + ): + toolchain._print_size_summary(tmp_path, _paths(tmp_path)) + assert capsys.readouterr().out == "" + assert "missing section(s) .bss" in caplog.text + + +def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None: + """Component-added options the native build drops are warned by name; + the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags, + flash_mode) stay quiet.""" + CORE.platformio_options = { + "board_build.ldscript": "eagle.flash.4m2m.ld", + "board_build.f_cpu": "160000000L", + "board_build.filesystem": "littlefs", + "board_build.flash_mode": "dio", + "build_src_flags": "-include throw_stubs.h", + "lib_ignore": ["Updater"], + "upload_speed": "460800", + } + toolchain._warn_ignored_platformio_options() + assert "platformio_options->board_build.filesystem is ignored" in caplog.text + assert "native 'arduino' toolchain" in caplog.text + assert "board_build.ldscript is ignored" not in caplog.text + assert "board_build.f_cpu is ignored" not in caplog.text + assert "lib_ignore" not in caplog.text + assert "build_src_flags" not in caplog.text + assert "flash_mode" not in caplog.text + # Component-added upload_speed never gets read under the native + # toolchain, so it must warn + assert "upload_speed" in caplog.text + + +def test_run_compile_idedata_error_does_not_fail_build( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unusable compile DB after a successful build warns, never fails.""" + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project"), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "_print_size_summary"), + patch.object( + toolchain, + "get_idedata", + side_effect=EsphomeError("compile database is unusable"), + ), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert "Could not generate idedata: compile database is unusable" in caplog.text + + +def test_run_compile_skipped_size_summary_names_consequence( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A skipped RAM/Flash summary warns about the missing metric; the + per-cause warnings alone are invisible to CI harnesses.""" + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project"), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "_print_size_summary", return_value=False), + patch.object(toolchain, "get_idedata", return_value=None), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert "Firmware size summary unavailable for this build" in caplog.text + + +def test_parse_app_size_non_utf8_ld_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A corrupt (non-UTF-8) linker script degrades to the same warning as + an unreadable one, never a traceback after a successful link.""" + paths = _paths(tmp_path) + ld = tmp_path / "corrupt.ld" + ld.write_bytes(b"\xff\xfe not utf8") + with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld): + assert toolchain._parse_app_size(tmp_path, paths) is None + assert "Cannot read linker script" in caplog.text + + +def test_get_idedata_accepts_preresolved_ccache() -> None: + """run_compile threads its resolved ccache through; the probe must not + run again.""" + with ( + patch( + "esphome.build_helpers.idedata.load_or_build_idedata", + return_value={"ok": True}, + ) as mock_build, + patch.object(toolchain, "resolve_ccache_path") as mock_resolve, + ): + assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True} + mock_resolve.assert_not_called() + assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache" diff --git a/tests/unit_tests/test_lazy_imports.py b/tests/unit_tests/test_lazy_imports.py index b6878c33a2..1eeddd80ea 100644 --- a/tests/unit_tests/test_lazy_imports.py +++ b/tests/unit_tests/test_lazy_imports.py @@ -34,7 +34,12 @@ HEAVY_MODULES = ( # Everything the storage fast path must keep out of sys.modules; the # existence guard and the leak check must watch the same list. -FAST_PATH_HEAVY_MODULES = HEAVY_MODULES + ("esphome.components.esp32",) +FAST_PATH_HEAVY_MODULES = HEAVY_MODULES + ( + "esphome.components.esp32", + # A native backend module resolves artifact paths on this path; its + # framework half pulls in the whole package-download stack + "esphome.platformio.registry", +) # Heavy only for modules that must not know about the API transport; # in the existence guard so a rename can't silently no-op its check. @@ -175,6 +180,27 @@ def test_esptool_upload_fast_path_does_not_import_heavy_modules( ) +def test_native_upload_fast_path_does_not_import_heavy_modules( + fixture_path: Path, + probe_env: dict[str, str], +) -> None: + """The native-toolchain serial upload dispatches through a + toolchain-keyed backend table; resolving the flash image must not drag + in the esp32 or esp8266 component package or the validation stack. + """ + leaked = _leaked_from_fixture( + fixture_path, + probe_env, + "native_upload_fast_path.py", + extra=("esphome.components.esp8266",), + ) + assert not leaked, ( + f"native upload_using_esptool pulls in heavy modules: {leaked}. " + "The upload fast path skips validation; importing a platform " + "component package executes its codegen module by design." + ) + + def test_api_client_does_not_import_heavy_modules() -> None: """``esphome.api_client`` is on the logs fast path and must stay light. diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 956ba86502..b5e7181269 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -13,7 +13,7 @@ import sys import time from types import SimpleNamespace from typing import Any, Self -from unittest.mock import AsyncMock, MagicMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch import pytest from pytest import CaptureFixture @@ -7212,7 +7212,7 @@ def test_command_run_rp2040_bootsel_redetects_serial_port() -> None: def test_command_idedata_esp_idf_prints_json(capsys: CaptureFixture) -> None: """Under the native ESP-IDF toolchain, idedata is emitted as JSON.""" - setup_core() + setup_core(platform=PLATFORM_ESP32) CORE.toolchain = Toolchain.ESP_IDF data = {"cxx_path": "g++", "prog_path": "/build/firmware.elf"} @@ -7226,7 +7226,7 @@ def test_command_idedata_esp_idf_prints_json(capsys: CaptureFixture) -> None: def test_command_idedata_esp_idf_no_build_errors() -> None: """Under ESP-IDF, a missing build (no idedata) returns an error, not a crash.""" - setup_core() + setup_core(platform=PLATFORM_ESP32) CORE.toolchain = Toolchain.ESP_IDF with patch("esphome.espidf.toolchain.get_idedata", return_value=None): @@ -7400,6 +7400,201 @@ def test_warn_source_tree_mismatch_falls_back_when_stat_fails( assert not caplog.text +def test_upload_using_esptool_arduino_toolchain( + tmp_path: Path, + mock_run_external_command_main: Mock, +) -> None: + """The native ESP8266 Arduino toolchain flashes its factory image at + 0x0, resolved from the toolchain-keyed backend table (deliberately not + the platform hook: that import would break the upload fast path).""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") + CORE.toolchain = Toolchain.ARDUINO + from esphome.arduino8266 import toolchain as native + + factory = native.get_factory_firmware_path() + factory.parent.mkdir(parents=True, exist_ok=True) + factory.touch() + + config = {CONF_ESPHOME: {"platformio_options": {}}} + result = upload_using_esptool(config, "/dev/ttyUSB0", None, None) + + assert result == 0 + cmd_list = list(mock_run_external_command_main.call_args[0][1:]) + firmware_offset_idx = cmd_list.index("write-flash") + 4 + assert cmd_list[firmware_offset_idx] == "0x0" + assert cmd_list[firmware_offset_idx + 1] == str(factory) + + +@pytest.mark.parametrize( + ("toolchain", "pio_project_written"), + [ + # The native toolchain generates its project at compile time, so + # write_cpp_file must not write a platformio.ini; the default + # toolchain writes the PlatformIO project files. + (Toolchain.ARDUINO, False), + (None, True), + ], +) +def test_write_cpp_file_project_generation_follows_toolchain( + tmp_path: Path, toolchain: Toolchain | None, pio_project_written: bool +) -> None: + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") + CORE.toolchain = toolchain + + with ( + patch("esphome.writer.write_cpp") as mock_write_cpp, + patch("esphome.build_gen.platformio.write_project") as mock_pio_project, + patch.object( + type(CORE), "cpp_main_section", new_callable=PropertyMock + ) as mock_section, + ): + mock_section.return_value = "" + assert main.write_cpp_file() == 0 + + mock_write_cpp.assert_called_once() + assert mock_pio_project.called is pio_project_written + + +def test_command_idedata_arduino_prints_json( + tmp_path: Path, capsys: CaptureFixture +) -> None: + """Under the native ESP8266 Arduino toolchain, idedata is emitted as JSON.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path) + CORE.toolchain = Toolchain.ARDUINO + data = {"cxx_path": "g++", "prog_path": "/build/firmware.elf"} + + with patch( + "esphome.arduino8266.toolchain.get_idedata", return_value=data + ) as mock_get: + result = command_idedata(MagicMock(), CORE.config) + + assert result == 0 + mock_get.assert_called_once_with() + assert json.loads(capsys.readouterr().out) == data + + +def test_command_idedata_arduino_no_build_errors(tmp_path: Path) -> None: + """A missing native build (no idedata) returns an error, not a crash.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path) + CORE.toolchain = Toolchain.ARDUINO + + with patch("esphome.arduino8266.toolchain.get_idedata", return_value=None): + result = command_idedata(MagicMock(), CORE.config) + + assert result == 1 + + +@pytest.mark.parametrize( + ("platform", "toolchain", "module"), + [ + (PLATFORM_ESP8266, Toolchain.ARDUINO, "esphome.arduino8266.toolchain"), + (PLATFORM_ESP32, Toolchain.ESP_IDF, "esphome.espidf.toolchain"), + ], +) +def test_command_analyze_memory_native_toolchains( + tmp_path: Path, + mock_write_cpp: Mock, + mock_compile_program: Mock, + mock_get_esphome_components: Mock, + mock_memory_analyzer_cli: Mock, + mock_ram_strings_analyzer: Mock, + platform: str, + toolchain: Toolchain, + module: str, +) -> None: + """analyze-memory uses the native toolchain's binutils instead of + falling into the PlatformIO branch.""" + setup_core(platform=platform, tmp_path=tmp_path, name="test_device") + CORE.toolchain = toolchain + + config = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + # The tools must exist: a missing binutils now fails by name instead of + # silently falling back to host tools + objdump = tmp_path / "objdump" + readelf = tmp_path / "readelf" + objdump.write_text("") + readelf.write_text("") + # The ELF must exist too: the analyzer swallows tool failures, so a + # missing image would report zeroes with exit 0 + firmware_elf = tmp_path / "firmware.elf" + firmware_elf.write_text("") + with ( + patch(f"{module}.get_objdump_path", return_value=objdump), + patch(f"{module}.get_readelf_path", return_value=readelf), + patch(f"{module}.get_elf_path", return_value=firmware_elf), + ): + result = command_analyze_memory(MockArgs(), config) + + assert result == 0 + mock_memory_analyzer_cli.assert_called_once_with( + str(firmware_elf), + str(objdump), + str(readelf), + set(), + idedata=None, + ) + + +def test_command_analyze_memory_native_missing_elf_fails( + tmp_path: Path, + mock_write_cpp: Mock, + mock_compile_program: Mock, + mock_get_esphome_components: Mock, + mock_memory_analyzer_cli: Mock, + mock_ram_strings_analyzer: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A missing firmware.elf fails by name instead of an exit-0 zeroed + report.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO + + config = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + objdump = tmp_path / "objdump" + readelf = tmp_path / "readelf" + objdump.write_text("") + readelf.write_text("") + module = "esphome.arduino8266.toolchain" + with ( + patch(f"{module}.get_objdump_path", return_value=objdump), + patch(f"{module}.get_readelf_path", return_value=readelf), + patch(f"{module}.get_elf_path", return_value=tmp_path / "missing.elf"), + ): + result = command_analyze_memory(MockArgs(), config) + + assert result == 1 + assert "compile the configuration first" in caplog.text + mock_memory_analyzer_cli.assert_not_called() + + +def test_command_analyze_memory_missing_binutils_fails_by_name( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A truncated toolchain install fails naming the missing tool instead + of silently analyzing with host binutils.""" + setup_core(platform="esp8266", tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO + config = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + module = "esphome.arduino8266.toolchain" + with ( + patch(f"{module}.get_objdump_path", return_value=tmp_path / "missing-objdump"), + patch(f"{module}.get_readelf_path", return_value=tmp_path / "readelf"), + patch("esphome.__main__.write_cpp", return_value=0), + patch("esphome.__main__.compile_program", return_value=0), + ): + assert command_analyze_memory(MockArgs(), config) == 1 + assert "missing-objdump" in caplog.text + assert "toolchain install may be incomplete" in caplog.text + + +def test_command_idedata_incompatible_toolchain(tmp_path: Path) -> None: + """A non-native, non-platformio toolchain errors out cleanly.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + CORE.toolchain = Toolchain.SDK_NRF + + assert command_idedata(MagicMock(), CORE.config) == 1 + + @pytest.mark.parametrize( "error", [ @@ -7476,6 +7671,49 @@ def test_compile_program_espidf_idedata_none_warns( assert "No idedata was generated" in caplog.text +def test_native_toolchain_table_serves_every_native_toolchain() -> None: + """Every member of NATIVE_TOOLCHAINS has a backend entry; a gap would + surface as a targeted EsphomeError on the one affected config, and this + pin keeps the table from drifting when a toolchain is added.""" + from esphome.build_helpers.native import NATIVE_TOOLCHAIN_MODULES + from esphome.const import NATIVE_TOOLCHAINS + + assert {tc for _, tc in NATIVE_TOOLCHAIN_MODULES} == set(NATIVE_TOOLCHAINS) + + +def test_native_toolchain_module_missing_backend_raises(tmp_path: Path) -> None: + """A native toolchain missing from the backend table is a bug and must + fail, not silently degrade to the PlatformIO path.""" + from esphome.build_helpers import native + + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO + with ( + patch.dict(native.NATIVE_TOOLCHAIN_MODULES, clear=True), + pytest.raises(EsphomeError, match="no native build backend"), + ): + native.native_backend() + + +def test_command_analyze_memory_unsupported_toolchain( + tmp_path: Path, + mock_write_cpp: Mock, + mock_compile_program: Mock, + caplog: pytest.LogCaptureFixture, +) -> None: + """A hook-less non-PlatformIO toolchain is refused by name, never routed + into the PlatformIO branch.""" + setup_core(platform=PLATFORM_NRF52, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.SDK_NRF + mock_write_cpp.return_value = 0 + mock_compile_program.return_value = 0 + + result = command_analyze_memory(MockArgs(), {CONF_ESPHOME: {CONF_NAME: "t"}}) + + assert result == 1 + assert "analyze-memory is not supported" in caplog.text + + def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None: """An explicit --toolchain must run the per-platform validators, so the upload/logs fast path becomes a cache miss.""" @@ -7491,6 +7729,29 @@ def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None: mock_read.assert_called_once() +def test_upload_using_esptool_native_missing_firmware_raises( + tmp_path: Path, +) -> None: + """A stale or absent firmware.bin fails by name instead of flashing air.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(EsphomeError, match="compile the configuration first"): + upload_using_esptool( + {CONF_ESPHOME: {"platformio_options": {}}}, "/dev/ttyUSB0", None, None + ) + + +def test_compile_program_unclaimed_native_toolchain_raises( + tmp_path: Path, +) -> None: + """A resolved native toolchain no platform backend claims must fail, + never fall through to the PlatformIO project path.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO # esp32 has no arduino-native backend + with pytest.raises(EsphomeError, match="no platform backend claimed"): + compile_program(MockArgs(), {}) + + def test_cli_toolchain_still_refreshes_the_validated_config_cache( tmp_path: Path, ) -> None: diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 47feae3e3c..3dcc4b12b8 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -516,10 +516,13 @@ def test_clean_build( dependencies_lock = tmp_path / "dependencies.lock" dependencies_lock.write_text("lock file") - # idedata cache lives under the data dir, not the build path. + # idedata caches live under the data dir, not the build path; the + # .arduino.json variant is the native esp8266 toolchain's. idedata_cache = tmp_path / "idedata" / "test.json" idedata_cache.parent.mkdir() idedata_cache.write_text("{}") + arduino_idedata_cache = tmp_path / "idedata" / "test.arduino.json" + arduino_idedata_cache.write_text("{}") # Native ESP-IDF toolchain artifacts. idf_build_dir = tmp_path / "build" @@ -580,6 +583,7 @@ def test_clean_build( assert not piolibdeps_dir.exists() assert not dependencies_lock.exists() assert not idedata_cache.exists() + assert not arduino_idedata_cache.exists() assert not idf_build_dir.exists() assert not managed_components_dir.exists() assert not pio_components_dir.exists()