diff --git a/esphome/__main__.py b/esphome/__main__.py index 01b33eb8ac..2894eb6742 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -881,7 +881,15 @@ def upload_using_esptool( elif CORE.using_toolchain_esp_idf: from esphome.espidf import api - flash_images = [FlashImage(path=api.get_factory_firmware_path(), offset="0x0")] + # For ESP-IDF the upload is a single factory image at 0x0 (it bundles + # bootloader + partitions + app). The prebuilt-dir form ships that + # single file at the canonical name so the dashboard can flash it + # without re-deriving the ESP-IDF build path. + factory_path = ( + CORE.prebuilt_artifact_path("firmware.factory.bin") + or api.get_factory_firmware_path() + ) + flash_images = [FlashImage(path=factory_path, offset="0x0")] else: from esphome import platformio_api @@ -960,6 +968,23 @@ def upload_using_esptool( def upload_using_platformio(config: ConfigType, port: str) -> int: from esphome import platformio_api + # --prebuilt-dir for libretiny / RP2040 serial routes PlatformIO at the + # prebuilt tree by repointing CORE.build_path. The dashboard must ship a + # build tree shape (platformio.ini plus .pioenvs//...) under + # --prebuilt-dir for this path because PlatformIO needs platformio.ini and + # the env-specific .pioenvs/ directory to run -t upload -t nobuild. + # Upload is terminal so mutating build_path here has no downstream effect. + if CORE.prebuilt_dir is not None: + platformio_ini = CORE.prebuilt_dir / "platformio.ini" + if not platformio_ini.is_file(): + raise EsphomeError( + f"--prebuilt-dir {CORE.prebuilt_dir} is missing platformio.ini. " + "Uploads on this platform re-invoke PlatformIO, so the prebuilt " + "directory must contain platformio.ini plus the env-specific " + ".pioenvs// build tree." + ) + CORE.build_path = CORE.prebuilt_dir + # RP2040 platform-raspberrypi build recipe expects firmware.bin.signed for # the upload target, but 'nobuild' skips the build phase that creates it. # Create it here so the upload doesn't fail. @@ -998,13 +1023,23 @@ def upload_using_picotool(config: ConfigType) -> int: from esphome import platformio_api idedata = platformio_api.get_idedata(config) - firmware_elf = Path(idedata.firmware_elf_path) - if not firmware_elf.is_file(): + # --prebuilt-dir ships canonical artifacts at the root of the directory, + # not the full PlatformIO build tree, so the ELF may not be present. + # picotool's "load" target accepts .uf2 / .bin / .elf, so fall back to + # CORE.firmware_bin (which resolves to the prebuilt firmware.uf2 when set) + # when no ELF is available. + firmware_file: Path + elf_path = Path(idedata.firmware_elf_path) + if elf_path.is_file(): + firmware_file = elf_path + elif CORE.prebuilt_dir is not None and CORE.firmware_bin.is_file(): + firmware_file = CORE.firmware_bin + else: _LOGGER.error( "Firmware ELF file not found at %s. " "Make sure the project has been compiled first.", - firmware_elf, + elf_path, ) return 1 @@ -1023,7 +1058,7 @@ def upload_using_picotool(config: ConfigType) -> int: # so progress bars display in real-time with \r updates. # Capture stderr only so we can detect permission errors. result = subprocess.run( - [str(picotool), "load", "-v", "-x", str(firmware_elf)], + [str(picotool), "load", "-v", "-x", str(firmware_file)], stderr=subprocess.PIPE, timeout=60, check=False, @@ -1113,6 +1148,19 @@ def upload_program( config: ConfigType, args: ArgsProtocol, devices: list[str] ) -> tuple[int, str | None]: host = devices[0] + + # --prebuilt-dir routes every per-platform upload helper at a directory of + # prebuilt artifacts instead of the local build tree. Validate once here + # so failures surface before we dispatch into platform-specific code that + # would otherwise produce confusing "file not found" errors deep in the + # esptool / picotool / PlatformIO call stacks. + prebuilt_dir = getattr(args, "prebuilt_dir", None) + if prebuilt_dir is not None: + prebuilt_path = Path(prebuilt_dir).expanduser() + if not prebuilt_path.is_dir(): + raise EsphomeError(f"--prebuilt-dir {prebuilt_dir} is not a directory") + CORE.prebuilt_dir = prebuilt_path + try: module = importlib.import_module("esphome.components." + CORE.target_platform) if getattr(module, "upload_program")(config, args, host): @@ -2118,6 +2166,17 @@ def parse_args(argv): "--file", help="Manually specify the binary file to upload.", ) + parser_upload.add_argument( + "--prebuilt-dir", + help=( + "Directory of prebuilt artifacts to flash instead of re-deriving " + "paths from the local build tree. The configuration is still read " + "(to load OTA settings, target platform, esp32 variant, etc.) but " + "the upload path reads bytes from this directory. Layout is " + "documented at " + "https://developers.esphome.io/architecture/upload-prebuilt-dir/." + ), + ) parser_upload.add_argument( "--ota-platform", choices=[CONF_ESPHOME, CONF_WEB_SERVER], diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index ef0eddc603..cdb12c129a 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -567,6 +567,11 @@ class EsphomeCore: self.config_path: Path | None = None # The relative path to where all build files are stored self.build_path: Path | None = None + # Directory of prebuilt artifacts for `esphome upload --prebuilt-dir`. + # When set, firmware/partition/bootloader resolution and the idedata + # cache prefer files under this directory over the local build tree. + # See docs/architecture/upload-prebuilt-dir. + self.prebuilt_dir: Path | None = None # The validated configuration, this is None until the config has been validated self.config: ConfigType | None = None # The pending tasks in the task queue (mostly for C++ generation) @@ -633,6 +638,7 @@ class EsphomeCore: self.data = {} self.config_path = None self.build_path = None + self.prebuilt_dir = None self.config = None self.event_loop = _FakeEventLoop() self.task_counter = 0 @@ -775,8 +781,28 @@ class EsphomeCore: def relative_piolibdeps_path(self, *path: str | Path) -> Path: return self.relative_build_path(".piolibdeps", *path) + def prebuilt_artifact_path(self, *names: str) -> Path | None: + # Return the first existing prebuilt artifact among ``names``, or None + # when no prebuilt dir is configured or none of the candidates exist. + # Callers pass canonical basenames (e.g. "firmware.bin", "firmware.uf2") + # and the upload helpers fall back to the local build tree when this + # returns None. + if self.prebuilt_dir is None: + return None + for name in names: + candidate = self.prebuilt_dir / name + if candidate.is_file(): + return candidate + return None + @property def firmware_bin(self) -> Path: + # Prebuilt override: prefer firmware.bin then firmware.uf2 so the + # dashboard can ship a single canonical name per platform. + if ( + prebuilt := self.prebuilt_artifact_path("firmware.bin", "firmware.uf2") + ) is not None: + return prebuilt # Check if using ESP-IDF toolchain if self.using_toolchain_esp_idf: return self.relative_build_path("build", f"{self.name}.bin") @@ -789,6 +815,12 @@ class EsphomeCore: # Native ESP-IDF (--toolchain esp-idf): the partition table image is emitted under # build/partition_table/partition-table.bin alongside firmware.bin. PlatformIO writes the # equivalent file as partitions.bin in the env-specific .pioenvs directory. + if ( + prebuilt := self.prebuilt_artifact_path( + "partitions.bin", "partition-table.bin" + ) + ) is not None: + return prebuilt if self.using_toolchain_esp_idf: return self.relative_build_path( "build", "partition_table", "partition-table.bin" @@ -797,6 +829,8 @@ class EsphomeCore: @property def bootloader_bin(self) -> Path: + if (prebuilt := self.prebuilt_artifact_path("bootloader.bin")) is not None: + return prebuilt if self.using_toolchain_esp_idf: return self.relative_build_path("build", "bootloader", "bootloader.bin") return self.relative_pioenvs_path(self.name, "bootloader.bin") diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index 81ff01306a..86c7cdfd4f 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -102,6 +102,16 @@ def _run_idedata(config): def _load_idedata(config): + # `esphome upload --prebuilt-dir` ships a pre-rendered idedata.json next + # to the artifacts. When present we use it verbatim: ``firmware_bin_path`` + # and ``extra.flash_images[*].path`` already point at absolute paths under + # the prebuilt directory, so the esptool / picotool helpers find the right + # bytes without re-running PlatformIO or consulting the local build tree. + if CORE.prebuilt_dir is not None: + prebuilt_idedata = CORE.prebuilt_dir / "idedata.json" + if prebuilt_idedata.is_file(): + return json.loads(prebuilt_idedata.read_text(encoding="utf-8")) + platformio_ini = CORE.relative_build_path("platformio.ini") temp_idedata = CORE.relative_internal_path("idedata", f"{CORE.name}.json") diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py index 1a52e6b29e..9cbd6cc120 100644 --- a/tests/unit_tests/test_core.py +++ b/tests/unit_tests/test_core.py @@ -870,6 +870,89 @@ class TestEsphomeCore: "foo/build/.pioenvs/test-device/bootloader.bin" ) + def test_prebuilt_artifact_path__none_when_unset(self, target): + """prebuilt_artifact_path is the gate on every prebuilt-dir override. + When --prebuilt-dir is not set, every consumer must fall back to the + local build tree.""" + assert target.prebuilt_artifact_path("firmware.bin") is None + + def test_prebuilt_artifact_path__returns_first_existing(self, target, tmp_path): + """firmware.bin (ESP) and firmware.uf2 (RP2040/libretiny) are both + canonical names, so callers pass them in priority order and get the + first one that actually exists.""" + target.prebuilt_dir = tmp_path + (tmp_path / "firmware.uf2").write_bytes(b"uf2") + + assert ( + target.prebuilt_artifact_path("firmware.bin", "firmware.uf2") + == tmp_path / "firmware.uf2" + ) + + def test_prebuilt_artifact_path__none_when_no_candidate_exists( + self, target, tmp_path + ): + """If --prebuilt-dir is set but the directory is empty for the asked + names, return None so the caller falls through to the local build + tree rather than asserting.""" + target.prebuilt_dir = tmp_path + + assert target.prebuilt_artifact_path("firmware.bin") is None + + def test_firmware_bin__prebuilt_override(self, target, tmp_path): + """CORE.firmware_bin is the single resolution point every OTA path + reads. With --prebuilt-dir set and firmware.bin present, it must + return the prebuilt path instead of the (non-existent) local build + path.""" + target.name = "test-device" + target.toolchain = const.Toolchain.PLATFORMIO + target.prebuilt_dir = tmp_path + (tmp_path / "firmware.bin").write_bytes(b"fw") + + assert target.firmware_bin == tmp_path / "firmware.bin" + + def test_firmware_bin__prebuilt_override_uf2(self, target, tmp_path): + """RP2040 / libretiny ship firmware.uf2; firmware_bin returns it when + firmware.bin is absent so the dashboard only needs to ship one file.""" + target.name = "test-device" + target.toolchain = const.Toolchain.PLATFORMIO + target.prebuilt_dir = tmp_path + (tmp_path / "firmware.uf2").write_bytes(b"uf2") + + assert target.firmware_bin == tmp_path / "firmware.uf2" + + def test_partition_table_bin__prebuilt_override(self, target, tmp_path): + target.name = "test-device" + target.toolchain = const.Toolchain.PLATFORMIO + target.prebuilt_dir = tmp_path + (tmp_path / "partitions.bin").write_bytes(b"pt") + + assert target.partition_table_bin == tmp_path / "partitions.bin" + + def test_bootloader_bin__prebuilt_override(self, target, tmp_path): + target.name = "test-device" + target.toolchain = const.Toolchain.PLATFORMIO + target.prebuilt_dir = tmp_path + (tmp_path / "bootloader.bin").write_bytes(b"bl") + + assert target.bootloader_bin == tmp_path / "bootloader.bin" + + def test_firmware_bin__prebuilt_dir_set_but_file_missing_falls_through( + self, target, tmp_path + ): + """Setting --prebuilt-dir alone must not break devices that don't ship + every artifact (e.g. OTA-only uploads with no bootloader.bin). When + the canonical file isn't in the directory, fall back to the local + build path so the caller's existing error messages still apply.""" + target.name = "test-device" + target.toolchain = const.Toolchain.PLATFORMIO + target.data[const.KEY_CORE] = {const.KEY_TARGET_PLATFORM: "esp32"} + target.prebuilt_dir = tmp_path + # No firmware.bin in tmp_path on purpose. + + assert target.firmware_bin == Path( + "foo/build/.pioenvs/test-device/firmware.bin" + ) + def test_add_library__extracts_short_name_from_path(self, target): """Test add_library extracts short name from library paths like owner/lib.""" target.data[const.KEY_CORE] = { diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 3eb50de76b..ff8394e2dd 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -1135,6 +1135,7 @@ class MockArgs: ota_platform: str | None = None partition_table: bool = False bootloader: bool = False + prebuilt_dir: str | None = None def test_upload_program_serial_esp32( @@ -1412,6 +1413,95 @@ def test_upload_using_platformio_skips_signed_bin_for_non_rp2040( assert result == 0 +def test_upload_program_prebuilt_dir_sets_core_attr( + mock_upload_using_esptool: Mock, + mock_get_port_type: Mock, + mock_check_permissions: Mock, + tmp_path: Path, +) -> None: + """--prebuilt-dir must stash the validated path on CORE before dispatching + so that all per-platform helpers and CORE.firmware_bin / etc. resolve to + the prebuilt artifacts.""" + setup_core(platform=PLATFORM_ESP32) + mock_get_port_type.return_value = "SERIAL" + mock_upload_using_esptool.return_value = 0 + + prebuilt = tmp_path / "artifacts" + prebuilt.mkdir() + + config = {} + args = MockArgs(prebuilt_dir=str(prebuilt)) + devices = ["/dev/ttyUSB0"] + + exit_code, _ = upload_program(config, args, devices) + + assert exit_code == 0 + assert CORE.prebuilt_dir == prebuilt + + +def test_upload_program_prebuilt_dir_missing_raises( + mock_get_port_type: Mock, + mock_check_permissions: Mock, + tmp_path: Path, +) -> None: + """Catch a missing --prebuilt-dir before dispatching to a per-platform + helper; deferring would surface as a confusing "file not found" deep in + esptool / picotool / PlatformIO.""" + setup_core(platform=PLATFORM_ESP32) + mock_get_port_type.return_value = "SERIAL" + + missing = tmp_path / "does-not-exist" + + config = {} + args = MockArgs(prebuilt_dir=str(missing)) + devices = ["/dev/ttyUSB0"] + + with pytest.raises(EsphomeError, match="not a directory"): + upload_program(config, args, devices) + + +def test_upload_using_picotool_falls_back_to_firmware_bin_when_elf_missing( + tmp_path: Path, +) -> None: + """`--prebuilt-dir` ships a flat firmware.uf2, not the build tree's ELF. + picotool load accepts uf2/bin/elf, so when the idedata ELF is missing + fall through to CORE.firmware_bin (the prebuilt .uf2) instead of failing. + """ + setup_core(platform=PLATFORM_RP2040, tmp_path=tmp_path) + + prebuilt = tmp_path / "prebuilt" + prebuilt.mkdir() + firmware_uf2 = prebuilt / "firmware.uf2" + firmware_uf2.write_bytes(b"uf2-data") + CORE.prebuilt_dir = prebuilt + + # idedata points at an ELF that doesn't exist on disk (typical for + # prebuilt-dir flat layouts). + mock_idedata = MagicMock() + mock_idedata.firmware_elf_path = str(tmp_path / "build" / "firmware.elf") + mock_idedata.cc_path = "/fake/path/gcc" + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stderr = b"" + + # Stub the picotool lookup to short-circuit the toolchain probe. + with ( + patch("esphome.platformio_api.get_idedata", return_value=mock_idedata), + patch( + "esphome.__main__.get_picotool_path", + return_value=tmp_path / "picotool", + ), + patch("subprocess.run", return_value=mock_result) as mock_run, + ): + exit_code = upload_using_picotool({}) + + assert exit_code == 0 + # Verify picotool was handed the prebuilt .uf2, not the missing ELF. + cmd = mock_run.call_args[0][0] + assert str(firmware_uf2) in cmd + + def test_upload_program_serial_upload_failed( mock_upload_using_esptool: Mock, mock_get_port_type: Mock, diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index 7a88ec4d9e..ac826d3a67 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -236,6 +236,61 @@ def test_load_idedata_regenerates_on_corrupted_cache( assert result["prog_path"] == "/new/firmware.elf" +def test_load_idedata_uses_prebuilt_dir_when_set( + setup_core: Path, mock_run_platformio_cli_run: Mock +) -> None: + """`esphome upload --prebuilt-dir ` is expected to ship the rendered + idedata.json next to the artifacts and bypass PlatformIO entirely. Verify + that _load_idedata returns the prebuilt copy verbatim without consulting + platformio.ini mtime or invoking ``platformio run -t idedata``.""" + CORE.build_path = str(setup_core / "build" / "test") + CORE.name = "test" + + prebuilt_dir = setup_core / "prebuilt" + prebuilt_dir.mkdir() + prebuilt_idedata = prebuilt_dir / "idedata.json" + prebuilt_idedata.write_text( + json.dumps({"prog_path": str(prebuilt_dir / "firmware.elf")}) + ) + + CORE.prebuilt_dir = prebuilt_dir + + result = platformio_api._load_idedata({"name": "test"}) + + assert result["prog_path"] == str(prebuilt_dir / "firmware.elf") + # Never re-runs PlatformIO when prebuilt idedata is supplied: the dashboard + # ships these artifacts from a paired build server with no local PIO tree. + mock_run_platformio_cli_run.assert_not_called() + + +def test_load_idedata_falls_back_when_prebuilt_idedata_missing( + setup_core: Path, mock_run_platformio_cli_run: Mock +) -> None: + """If --prebuilt-dir is set but the directory has no idedata.json, the + normal local-build-tree path runs. Lets the dashboard skip idedata for + OTA-only uploads (where firmware.bin alone is enough) without breaking.""" + CORE.build_path = str(setup_core / "build" / "test") + CORE.name = "test" + + prebuilt_dir = setup_core / "prebuilt" + prebuilt_dir.mkdir() + # Intentionally no idedata.json in prebuilt_dir. + CORE.prebuilt_dir = prebuilt_dir + + platformio_ini = setup_core / "build" / "test" / "platformio.ini" + platformio_ini.parent.mkdir(parents=True, exist_ok=True) + platformio_ini.write_text("content") + + mock_run_platformio_cli_run.return_value = json.dumps( + {"prog_path": "/local/firmware.elf"} + ) + + result = platformio_api._load_idedata({"name": "test"}) + + assert result["prog_path"] == "/local/firmware.elf" + mock_run_platformio_cli_run.assert_called_once() + + def test_run_idedata_parses_json_from_output( setup_core: Path, mock_run_platformio_cli_run: Mock ) -> None: