diff --git a/esphome/__main__.py b/esphome/__main__.py index a2cd561e1d..dadebe9a0d 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -978,20 +978,15 @@ def upload_using_esptool( if file is not None: flash_images = [FlashImage(path=file, offset="0x0")] - elif CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain - - flash_images = [ - FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0") - ] - elif CORE.using_native_toolchain: - # The native backend writes PlatformIO-compatible output paths, so the - # shared property already points at the right file. - if not CORE.firmware_bin.is_file(): + elif (native := _native_toolchain_module()) is not None: + # Every native backend supplies its own 0x0 flash image (bootloader + # and partitions included where the target needs them) + image = native.get_factory_firmware_path() + if not image.is_file(): raise EsphomeError( - f"{CORE.firmware_bin} does not exist; compile the configuration first" + f"{image} does not exist; compile the configuration first" ) - flash_images = [FlashImage(path=CORE.firmware_bin, offset="0x0")] + flash_images = [FlashImage(path=image, offset="0x0")] else: from esphome.platformio import toolchain diff --git a/esphome/arduino8266/toolchain.py b/esphome/arduino8266/toolchain.py index 8d9292e501..87b51466c2 100644 --- a/esphome/arduino8266/toolchain.py +++ b/esphome/arduino8266/toolchain.py @@ -61,6 +61,12 @@ def _toolchain_tool(name: str) -> Path: return framework.toolchain_tool(framework.get_toolchain_path(), name) +def get_factory_firmware_path() -> Path: + """The image to serial-flash at 0x0 (same bytes as firmware.bin: the + 8266 factory copy exists for artifact-contract parity, not content).""" + return get_build_dir() / "firmware.factory.bin" + + def get_addr2line_path() -> Path: return _toolchain_tool("addr2line") @@ -98,22 +104,27 @@ def run_compile(config: ConfigType, verbose: bool) -> int: cmd += ["-j", str(jobs)] # A dry-run probe keeps a no-op rebuild quiet: ninja would only print - # "no work to do". cwd instead of -C also drops the "Entering - # directory" banner on real builds. - probe = subprocess.run( - [str(paths.ninja), "-n"], - cwd=build_dir, - env=env, - capture_output=True, - text=True, - check=False, - close_fds=False, - ) - if probe.stderr.strip(): - # A load-time diagnostic (e.g. "multiple rules generate X") flags a - # generator bug; the skip branch would otherwise swallow it forever - _LOGGER.warning("ninja: %s", probe.stderr.strip()) - if probe.returncode == 0 and "no work to do" in probe.stdout: + # "no work to do". A freshly rewritten manifest all but guarantees work, + # so skip the probe (and its full stat pass) on that path. cwd instead + # of -C also drops the "Entering directory" banner on real builds. + skip_build = False + if not ninja_changed: + probe = subprocess.run( + [str(paths.ninja), "-n"], + cwd=build_dir, + env=env, + capture_output=True, + text=True, + check=False, + close_fds=False, + ) + if probe.stderr.strip(): + # A load-time diagnostic (e.g. "multiple rules generate X") + # flags a generator bug; the skip branch would otherwise + # swallow it forever + _LOGGER.warning("ninja: %s", probe.stderr.strip()) + skip_build = probe.returncode == 0 and "no work to do" in probe.stdout + if skip_build: _LOGGER.debug("ninja: nothing to rebuild") else: _LOGGER.debug("Running: %s", " ".join(cmd)) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 946fb88a5b..d507fff1d7 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -52,6 +52,7 @@ import argparse from collections import Counter from collections.abc import Callable from enum import StrEnum +import functools from functools import cache import json import os @@ -525,6 +526,13 @@ def _path_or_file_trigger( ) +@functools.lru_cache +def _cached_components_closure(files: tuple[str, ...]) -> frozenset[str]: + """The dependency closure walk is expensive; every toolchain smoke-test + job asks for the same file list, so compute it once per run.""" + return frozenset(_changed_components_closure(list(files))) + + def _changed_components_closure(files: list[str]) -> set[str]: """Dependency closure of the changed components, from the changed files.""" component_files = [f for f in files if filter_component_and_test_files(f)] @@ -545,14 +553,19 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool: # compile. When they change we fold the `esp32` component into the matrix so # the default native-IDF build path is still compiled on an infra-only PR. ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/") -ESP_IDF_INFRA_TRIGGER_FILES = frozenset( +# Shared library-conversion modules every native build imports; a new shared +# module added to one native trigger set must not silently skip the other's +# smoke test, so both sets union this one. +_NATIVE_SHARED_TRIGGER_FILES = frozenset( { - "esphome/build_gen/espidf.py", "esphome/framework_helpers.py", "esphome/platformio/library.py", "esphome/platformio/extra_script.py", } ) +ESP_IDF_INFRA_TRIGGER_FILES = _NATIVE_SHARED_TRIGGER_FILES | { + "esphome/build_gen/espidf.py", +} def _esp_idf_infra_changed(files: list[str]) -> bool: @@ -613,7 +626,7 @@ def _toolchain_components_to_test( if core_changed(files) or infra_trigger(files): return sorted(test_set) - return sorted(test_set & _changed_components_closure(files)) + return sorted(test_set & _cached_components_closure(tuple(files))) def should_run_esp32_platformio(branch: str | None = None) -> bool: @@ -658,21 +671,18 @@ ESP8266_NATIVE_TRIGGER_PATH_PREFIXES = ( "esphome/arduino/", "esphome/build_helpers/", ) -ESP8266_NATIVE_TRIGGER_FILES = frozenset( - { - "esphome/build_gen/arduino8266.py", - "esphome/build_gen/build_tool.py", - "esphome/platformio/extra_script.py", - "esphome/components/esp8266/build_surgery.py", - "esphome/components/esp8266/boards.py", - "esphome/platformio/library.py", - "esphome/platformio/registry.py", - "esphome/platformio/toolchain.py", - "script/test_build_components.py", - ".github/workflows/ci.yml", - ".github/actions/cache-arduino8266/action.yml", - } -) +ESP8266_NATIVE_TRIGGER_FILES = _NATIVE_SHARED_TRIGGER_FILES | { + "esphome/build_gen/arduino8266.py", + "esphome/build_gen/build_tool.py", + "esphome/components/esp8266/build_surgery.py", + "esphome/components/esp8266/boards.py", + "esphome/platformio/registry.py", + # esp8266/__init__.py imports copy_ccache_script from it + "esphome/platformio/toolchain.py", + "script/test_build_components.py", + ".github/workflows/ci.yml", + ".github/actions/cache-arduino8266/action.yml", +} def _esp8266_native_path_or_file_trigger(files: list[str]) -> bool: diff --git a/tests/unit_tests/test_arduino8266_toolchain.py b/tests/unit_tests/test_arduino8266_toolchain.py index a949dccc4b..aa32ff26f1 100644 --- a/tests/unit_tests/test_arduino8266_toolchain.py +++ b/tests/unit_tests/test_arduino8266_toolchain.py @@ -86,7 +86,8 @@ def test_run_compile_success(tmp_path: Path) -> None: with ( patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), patch.object(framework, "get_build_env", return_value={}), - patch("esphome.build_gen.arduino8266.write_project"), + # An unchanged manifest is what makes the -n probe run + patch("esphome.build_gen.arduino8266.write_project", return_value=False), patch.object( toolchain.subprocess, "run", @@ -118,7 +119,8 @@ def test_run_compile_noop_skips_the_build_spawn(tmp_path: Path) -> None: with ( patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), patch.object(framework, "get_build_env", return_value={}), - patch("esphome.build_gen.arduino8266.write_project"), + # An unchanged manifest is what makes the -n probe run + patch("esphome.build_gen.arduino8266.write_project", return_value=False), patch.object( toolchain.subprocess, "run", @@ -145,7 +147,8 @@ def test_run_compile_surfaces_probe_diagnostics( with ( patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), patch.object(framework, "get_build_env", return_value={}), - patch("esphome.build_gen.arduino8266.write_project"), + # An unchanged manifest is what makes the -n probe run + patch("esphome.build_gen.arduino8266.write_project", return_value=False), patch.object( toolchain.subprocess, "run", diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index d64a1c4877..bfad6708a0 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -7124,11 +7124,15 @@ def test_upload_using_esptool_arduino_toolchain( tmp_path: Path, mock_run_external_command_main: Mock, ) -> None: - """The native ESP8266 Arduino toolchain flashes CORE.firmware_bin at 0x0.""" + """The native ESP8266 Arduino toolchain flashes its factory image at + 0x0, dispatched through the platform's native_toolchain_module hook.""" setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test") CORE.toolchain = Toolchain.ARDUINO - CORE.firmware_bin.parent.mkdir(parents=True, exist_ok=True) - CORE.firmware_bin.touch() + from esphome.arduino8266 import toolchain as native + + factory = native.get_factory_firmware_path() + factory.parent.mkdir(parents=True, exist_ok=True) + factory.touch() config = {CONF_ESPHOME: {"platformio_options": {}}} result = upload_using_esptool(config, "/dev/ttyUSB0", None, None) @@ -7137,7 +7141,7 @@ def test_upload_using_esptool_arduino_toolchain( cmd_list = list(mock_run_external_command_main.call_args[0][1:]) firmware_offset_idx = cmd_list.index("write-flash") + 4 assert cmd_list[firmware_offset_idx] == "0x0" - assert cmd_list[firmware_offset_idx + 1] == str(CORE.firmware_bin) + assert cmd_list[firmware_offset_idx + 1] == str(factory) @pytest.mark.parametrize(