[cli] Add esphome upload --prebuilt-dir <path>

Adds a --prebuilt-dir flag to esphome upload that points the per-platform
upload helpers at a directory of prebuilt artifacts shipped from a paired
build server, instead of re-deriving paths from the local build tree.

Covers every upload-dispatch shape:
- ESP32 / ESP8266 serial (esptool) reads firmware.bin + extras via the
  prebuilt idedata.json the dashboard ships next to the artifacts.
- ESP32 / ESP8266 OTA (native API + web_server) reads CORE.firmware_bin,
  CORE.partition_table_bin and CORE.bootloader_bin which now consult the
  prebuilt-dir first.
- RP2040 BOOTSEL (picotool) falls back from the idedata ELF (absent in
  the flat layout) to the prebuilt firmware.uf2.
- RP2040 serial / libretiny serial / OTA (PlatformIO upload -t nobuild)
  point platformio at the prebuilt build tree via CORE.build_path so the
  -t upload -t nobuild path finds platformio.ini and .pioenvs/<name>/.

Issue: esphome/device-builder#572

esphome/__main__.py

@@ -881,7 +881,15 @@ def upload_using_esptool(
     elif CORE.using_toolchain_esp_idf:
         from esphome.espidf import api
 
-        flash_images = [FlashImage(path=api.get_factory_firmware_path(), offset="0x0")]
+        # For ESP-IDF the upload is a single factory image at 0x0 (it bundles
+        # bootloader + partitions + app). The prebuilt-dir form ships that
+        # single file at the canonical name so the dashboard can flash it
+        # without re-deriving the ESP-IDF build path.
+        factory_path = (
+            CORE.prebuilt_artifact_path("firmware.factory.bin")
+            or api.get_factory_firmware_path()
+        )
+        flash_images = [FlashImage(path=factory_path, offset="0x0")]
     else:
         from esphome import platformio_api
 
@@ -960,6 +968,23 @@ def upload_using_esptool(
 def upload_using_platformio(config: ConfigType, port: str) -> int:
     from esphome import platformio_api
 
+    # --prebuilt-dir for libretiny / RP2040 serial routes PlatformIO at the
+    # prebuilt tree by repointing CORE.build_path. The dashboard must ship a
+    # build tree shape (platformio.ini plus .pioenvs/<name>/...) under
+    # --prebuilt-dir for this path because PlatformIO needs platformio.ini and
+    # the env-specific .pioenvs/<name> directory to run -t upload -t nobuild.
+    # Upload is terminal so mutating build_path here has no downstream effect.
+    if CORE.prebuilt_dir is not None:
+        platformio_ini = CORE.prebuilt_dir / "platformio.ini"
+        if not platformio_ini.is_file():
+            raise EsphomeError(
+                f"--prebuilt-dir {CORE.prebuilt_dir} is missing platformio.ini. "
+                "Uploads on this platform re-invoke PlatformIO, so the prebuilt "
+                "directory must contain platformio.ini plus the env-specific "
+                ".pioenvs/<name>/ build tree."
+            )
+        CORE.build_path = CORE.prebuilt_dir
+
     # RP2040 platform-raspberrypi build recipe expects firmware.bin.signed for
     # the upload target, but 'nobuild' skips the build phase that creates it.
     # Create it here so the upload doesn't fail.
@@ -998,13 +1023,23 @@ def upload_using_picotool(config: ConfigType) -> int:
     from esphome import platformio_api
 
     idedata = platformio_api.get_idedata(config)
-    firmware_elf = Path(idedata.firmware_elf_path)
 
-    if not firmware_elf.is_file():
+    # --prebuilt-dir ships canonical artifacts at the root of the directory,
+    # not the full PlatformIO build tree, so the ELF may not be present.
+    # picotool's "load" target accepts .uf2 / .bin / .elf, so fall back to
+    # CORE.firmware_bin (which resolves to the prebuilt firmware.uf2 when set)
+    # when no ELF is available.
+    firmware_file: Path
+    elf_path = Path(idedata.firmware_elf_path)
+    if elf_path.is_file():
+        firmware_file = elf_path
+    elif CORE.prebuilt_dir is not None and CORE.firmware_bin.is_file():
+        firmware_file = CORE.firmware_bin
+    else:
         _LOGGER.error(
             "Firmware ELF file not found at %s. "
             "Make sure the project has been compiled first.",
-            firmware_elf,
+            elf_path,
         )
         return 1
 
@@ -1023,7 +1058,7 @@ def upload_using_picotool(config: ConfigType) -> int:
         # so progress bars display in real-time with \r updates.
         # Capture stderr only so we can detect permission errors.
         result = subprocess.run(
-            [str(picotool), "load", "-v", "-x", str(firmware_elf)],
+            [str(picotool), "load", "-v", "-x", str(firmware_file)],
             stderr=subprocess.PIPE,
             timeout=60,
             check=False,
@@ -1113,6 +1148,19 @@ def upload_program(
     config: ConfigType, args: ArgsProtocol, devices: list[str]
 ) -> tuple[int, str | None]:
     host = devices[0]
+
+    # --prebuilt-dir routes every per-platform upload helper at a directory of
+    # prebuilt artifacts instead of the local build tree. Validate once here
+    # so failures surface before we dispatch into platform-specific code that
+    # would otherwise produce confusing "file not found" errors deep in the
+    # esptool / picotool / PlatformIO call stacks.
+    prebuilt_dir = getattr(args, "prebuilt_dir", None)
+    if prebuilt_dir is not None:
+        prebuilt_path = Path(prebuilt_dir).expanduser()
+        if not prebuilt_path.is_dir():
+            raise EsphomeError(f"--prebuilt-dir {prebuilt_dir} is not a directory")
+        CORE.prebuilt_dir = prebuilt_path
+
     try:
         module = importlib.import_module("esphome.components." + CORE.target_platform)
         if getattr(module, "upload_program")(config, args, host):
@@ -2118,6 +2166,17 @@ def parse_args(argv):
         "--file",
         help="Manually specify the binary file to upload.",
     )
+    parser_upload.add_argument(
+        "--prebuilt-dir",
+        help=(
+            "Directory of prebuilt artifacts to flash instead of re-deriving "
+            "paths from the local build tree. The configuration is still read "
+            "(to load OTA settings, target platform, esp32 variant, etc.) but "
+            "the upload path reads bytes from this directory. Layout is "
+            "documented at "
+            "https://developers.esphome.io/architecture/upload-prebuilt-dir/."
+        ),
+    )
     parser_upload.add_argument(
         "--ota-platform",
         choices=[CONF_ESPHOME, CONF_WEB_SERVER],

esphome/core/__init__.py

@@ -567,6 +567,11 @@ class EsphomeCore:
         self.config_path: Path | None = None
         # The relative path to where all build files are stored
         self.build_path: Path | None = None
+        # Directory of prebuilt artifacts for `esphome upload --prebuilt-dir`.
+        # When set, firmware/partition/bootloader resolution and the idedata
+        # cache prefer files under this directory over the local build tree.
+        # See docs/architecture/upload-prebuilt-dir.
+        self.prebuilt_dir: Path | None = None
         # The validated configuration, this is None until the config has been validated
         self.config: ConfigType | None = None
         # The pending tasks in the task queue (mostly for C++ generation)
@@ -633,6 +638,7 @@ class EsphomeCore:
         self.data = {}
         self.config_path = None
         self.build_path = None
+        self.prebuilt_dir = None
         self.config = None
         self.event_loop = _FakeEventLoop()
         self.task_counter = 0
@@ -775,8 +781,28 @@ class EsphomeCore:
     def relative_piolibdeps_path(self, *path: str | Path) -> Path:
         return self.relative_build_path(".piolibdeps", *path)
 
+    def prebuilt_artifact_path(self, *names: str) -> Path | None:
+        # Return the first existing prebuilt artifact among ``names``, or None
+        # when no prebuilt dir is configured or none of the candidates exist.
+        # Callers pass canonical basenames (e.g. "firmware.bin", "firmware.uf2")
+        # and the upload helpers fall back to the local build tree when this
+        # returns None.
+        if self.prebuilt_dir is None:
+            return None
+        for name in names:
+            candidate = self.prebuilt_dir / name
+            if candidate.is_file():
+                return candidate
+        return None
+
     @property
     def firmware_bin(self) -> Path:
+        # Prebuilt override: prefer firmware.bin then firmware.uf2 so the
+        # dashboard can ship a single canonical name per platform.
+        if (
+            prebuilt := self.prebuilt_artifact_path("firmware.bin", "firmware.uf2")
+        ) is not None:
+            return prebuilt
         # Check if using ESP-IDF toolchain
         if self.using_toolchain_esp_idf:
             return self.relative_build_path("build", f"{self.name}.bin")
@@ -789,6 +815,12 @@ class EsphomeCore:
         # Native ESP-IDF (--toolchain esp-idf): the partition table image is emitted under
         # build/partition_table/partition-table.bin alongside firmware.bin. PlatformIO writes the
         # equivalent file as partitions.bin in the env-specific .pioenvs directory.
+        if (
+            prebuilt := self.prebuilt_artifact_path(
+                "partitions.bin", "partition-table.bin"
+            )
+        ) is not None:
+            return prebuilt
         if self.using_toolchain_esp_idf:
             return self.relative_build_path(
                 "build", "partition_table", "partition-table.bin"
@@ -797,6 +829,8 @@ class EsphomeCore:
 
     @property
     def bootloader_bin(self) -> Path:
+        if (prebuilt := self.prebuilt_artifact_path("bootloader.bin")) is not None:
+            return prebuilt
         if self.using_toolchain_esp_idf:
             return self.relative_build_path("build", "bootloader", "bootloader.bin")
         return self.relative_pioenvs_path(self.name, "bootloader.bin")

esphome/platformio_api.py

@@ -102,6 +102,16 @@ def _run_idedata(config):
 
 
 def _load_idedata(config):
+    # `esphome upload --prebuilt-dir` ships a pre-rendered idedata.json next
+    # to the artifacts. When present we use it verbatim: ``firmware_bin_path``
+    # and ``extra.flash_images[*].path`` already point at absolute paths under
+    # the prebuilt directory, so the esptool / picotool helpers find the right
+    # bytes without re-running PlatformIO or consulting the local build tree.
+    if CORE.prebuilt_dir is not None:
+        prebuilt_idedata = CORE.prebuilt_dir / "idedata.json"
+        if prebuilt_idedata.is_file():
+            return json.loads(prebuilt_idedata.read_text(encoding="utf-8"))
+
     platformio_ini = CORE.relative_build_path("platformio.ini")
     temp_idedata = CORE.relative_internal_path("idedata", f"{CORE.name}.json")
 

tests/unit_tests/test_core.py

@@ -870,6 +870,89 @@ class TestEsphomeCore:
             "foo/build/.pioenvs/test-device/bootloader.bin"
         )
 
+    def test_prebuilt_artifact_path__none_when_unset(self, target):
+        """prebuilt_artifact_path is the gate on every prebuilt-dir override.
+        When --prebuilt-dir is not set, every consumer must fall back to the
+        local build tree."""
+        assert target.prebuilt_artifact_path("firmware.bin") is None
+
+    def test_prebuilt_artifact_path__returns_first_existing(self, target, tmp_path):
+        """firmware.bin (ESP) and firmware.uf2 (RP2040/libretiny) are both
+        canonical names, so callers pass them in priority order and get the
+        first one that actually exists."""
+        target.prebuilt_dir = tmp_path
+        (tmp_path / "firmware.uf2").write_bytes(b"uf2")
+
+        assert (
+            target.prebuilt_artifact_path("firmware.bin", "firmware.uf2")
+            == tmp_path / "firmware.uf2"
+        )
+
+    def test_prebuilt_artifact_path__none_when_no_candidate_exists(
+        self, target, tmp_path
+    ):
+        """If --prebuilt-dir is set but the directory is empty for the asked
+        names, return None so the caller falls through to the local build
+        tree rather than asserting."""
+        target.prebuilt_dir = tmp_path
+
+        assert target.prebuilt_artifact_path("firmware.bin") is None
+
+    def test_firmware_bin__prebuilt_override(self, target, tmp_path):
+        """CORE.firmware_bin is the single resolution point every OTA path
+        reads. With --prebuilt-dir set and firmware.bin present, it must
+        return the prebuilt path instead of the (non-existent) local build
+        path."""
+        target.name = "test-device"
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.prebuilt_dir = tmp_path
+        (tmp_path / "firmware.bin").write_bytes(b"fw")
+
+        assert target.firmware_bin == tmp_path / "firmware.bin"
+
+    def test_firmware_bin__prebuilt_override_uf2(self, target, tmp_path):
+        """RP2040 / libretiny ship firmware.uf2; firmware_bin returns it when
+        firmware.bin is absent so the dashboard only needs to ship one file."""
+        target.name = "test-device"
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.prebuilt_dir = tmp_path
+        (tmp_path / "firmware.uf2").write_bytes(b"uf2")
+
+        assert target.firmware_bin == tmp_path / "firmware.uf2"
+
+    def test_partition_table_bin__prebuilt_override(self, target, tmp_path):
+        target.name = "test-device"
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.prebuilt_dir = tmp_path
+        (tmp_path / "partitions.bin").write_bytes(b"pt")
+
+        assert target.partition_table_bin == tmp_path / "partitions.bin"
+
+    def test_bootloader_bin__prebuilt_override(self, target, tmp_path):
+        target.name = "test-device"
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.prebuilt_dir = tmp_path
+        (tmp_path / "bootloader.bin").write_bytes(b"bl")
+
+        assert target.bootloader_bin == tmp_path / "bootloader.bin"
+
+    def test_firmware_bin__prebuilt_dir_set_but_file_missing_falls_through(
+        self, target, tmp_path
+    ):
+        """Setting --prebuilt-dir alone must not break devices that don't ship
+        every artifact (e.g. OTA-only uploads with no bootloader.bin). When
+        the canonical file isn't in the directory, fall back to the local
+        build path so the caller's existing error messages still apply."""
+        target.name = "test-device"
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.data[const.KEY_CORE] = {const.KEY_TARGET_PLATFORM: "esp32"}
+        target.prebuilt_dir = tmp_path
+        # No firmware.bin in tmp_path on purpose.
+
+        assert target.firmware_bin == Path(
+            "foo/build/.pioenvs/test-device/firmware.bin"
+        )
+
     def test_add_library__extracts_short_name_from_path(self, target):
         """Test add_library extracts short name from library paths like owner/lib."""
         target.data[const.KEY_CORE] = {

tests/unit_tests/test_main.py

@@ -1135,6 +1135,7 @@ class MockArgs:
     ota_platform: str | None = None
     partition_table: bool = False
     bootloader: bool = False
+    prebuilt_dir: str | None = None
 
 
 def test_upload_program_serial_esp32(
@@ -1412,6 +1413,95 @@ def test_upload_using_platformio_skips_signed_bin_for_non_rp2040(
     assert result == 0
 
 
+def test_upload_program_prebuilt_dir_sets_core_attr(
+    mock_upload_using_esptool: Mock,
+    mock_get_port_type: Mock,
+    mock_check_permissions: Mock,
+    tmp_path: Path,
+) -> None:
+    """--prebuilt-dir must stash the validated path on CORE before dispatching
+    so that all per-platform helpers and CORE.firmware_bin / etc. resolve to
+    the prebuilt artifacts."""
+    setup_core(platform=PLATFORM_ESP32)
+    mock_get_port_type.return_value = "SERIAL"
+    mock_upload_using_esptool.return_value = 0
+
+    prebuilt = tmp_path / "artifacts"
+    prebuilt.mkdir()
+
+    config = {}
+    args = MockArgs(prebuilt_dir=str(prebuilt))
+    devices = ["/dev/ttyUSB0"]
+
+    exit_code, _ = upload_program(config, args, devices)
+
+    assert exit_code == 0
+    assert CORE.prebuilt_dir == prebuilt
+
+
+def test_upload_program_prebuilt_dir_missing_raises(
+    mock_get_port_type: Mock,
+    mock_check_permissions: Mock,
+    tmp_path: Path,
+) -> None:
+    """Catch a missing --prebuilt-dir before dispatching to a per-platform
+    helper; deferring would surface as a confusing "file not found" deep in
+    esptool / picotool / PlatformIO."""
+    setup_core(platform=PLATFORM_ESP32)
+    mock_get_port_type.return_value = "SERIAL"
+
+    missing = tmp_path / "does-not-exist"
+
+    config = {}
+    args = MockArgs(prebuilt_dir=str(missing))
+    devices = ["/dev/ttyUSB0"]
+
+    with pytest.raises(EsphomeError, match="not a directory"):
+        upload_program(config, args, devices)
+
+
+def test_upload_using_picotool_falls_back_to_firmware_bin_when_elf_missing(
+    tmp_path: Path,
+) -> None:
+    """`--prebuilt-dir` ships a flat firmware.uf2, not the build tree's ELF.
+    picotool load accepts uf2/bin/elf, so when the idedata ELF is missing
+    fall through to CORE.firmware_bin (the prebuilt .uf2) instead of failing.
+    """
+    setup_core(platform=PLATFORM_RP2040, tmp_path=tmp_path)
+
+    prebuilt = tmp_path / "prebuilt"
+    prebuilt.mkdir()
+    firmware_uf2 = prebuilt / "firmware.uf2"
+    firmware_uf2.write_bytes(b"uf2-data")
+    CORE.prebuilt_dir = prebuilt
+
+    # idedata points at an ELF that doesn't exist on disk (typical for
+    # prebuilt-dir flat layouts).
+    mock_idedata = MagicMock()
+    mock_idedata.firmware_elf_path = str(tmp_path / "build" / "firmware.elf")
+    mock_idedata.cc_path = "/fake/path/gcc"
+
+    mock_result = MagicMock()
+    mock_result.returncode = 0
+    mock_result.stderr = b""
+
+    # Stub the picotool lookup to short-circuit the toolchain probe.
+    with (
+        patch("esphome.platformio_api.get_idedata", return_value=mock_idedata),
+        patch(
+            "esphome.__main__.get_picotool_path",
+            return_value=tmp_path / "picotool",
+        ),
+        patch("subprocess.run", return_value=mock_result) as mock_run,
+    ):
+        exit_code = upload_using_picotool({})
+
+    assert exit_code == 0
+    # Verify picotool was handed the prebuilt .uf2, not the missing ELF.
+    cmd = mock_run.call_args[0][0]
+    assert str(firmware_uf2) in cmd
+
+
 def test_upload_program_serial_upload_failed(
     mock_upload_using_esptool: Mock,
     mock_get_port_type: Mock,

tests/unit_tests/test_platformio_api.py

@@ -236,6 +236,61 @@ def test_load_idedata_regenerates_on_corrupted_cache(
     assert result["prog_path"] == "/new/firmware.elf"
 
 
+def test_load_idedata_uses_prebuilt_dir_when_set(
+    setup_core: Path, mock_run_platformio_cli_run: Mock
+) -> None:
+    """`esphome upload --prebuilt-dir <path>` is expected to ship the rendered
+    idedata.json next to the artifacts and bypass PlatformIO entirely. Verify
+    that _load_idedata returns the prebuilt copy verbatim without consulting
+    platformio.ini mtime or invoking ``platformio run -t idedata``."""
+    CORE.build_path = str(setup_core / "build" / "test")
+    CORE.name = "test"
+
+    prebuilt_dir = setup_core / "prebuilt"
+    prebuilt_dir.mkdir()
+    prebuilt_idedata = prebuilt_dir / "idedata.json"
+    prebuilt_idedata.write_text(
+        json.dumps({"prog_path": str(prebuilt_dir / "firmware.elf")})
+    )
+
+    CORE.prebuilt_dir = prebuilt_dir
+
+    result = platformio_api._load_idedata({"name": "test"})
+
+    assert result["prog_path"] == str(prebuilt_dir / "firmware.elf")
+    # Never re-runs PlatformIO when prebuilt idedata is supplied: the dashboard
+    # ships these artifacts from a paired build server with no local PIO tree.
+    mock_run_platformio_cli_run.assert_not_called()
+
+
+def test_load_idedata_falls_back_when_prebuilt_idedata_missing(
+    setup_core: Path, mock_run_platformio_cli_run: Mock
+) -> None:
+    """If --prebuilt-dir is set but the directory has no idedata.json, the
+    normal local-build-tree path runs. Lets the dashboard skip idedata for
+    OTA-only uploads (where firmware.bin alone is enough) without breaking."""
+    CORE.build_path = str(setup_core / "build" / "test")
+    CORE.name = "test"
+
+    prebuilt_dir = setup_core / "prebuilt"
+    prebuilt_dir.mkdir()
+    # Intentionally no idedata.json in prebuilt_dir.
+    CORE.prebuilt_dir = prebuilt_dir
+
+    platformio_ini = setup_core / "build" / "test" / "platformio.ini"
+    platformio_ini.parent.mkdir(parents=True, exist_ok=True)
+    platformio_ini.write_text("content")
+
+    mock_run_platformio_cli_run.return_value = json.dumps(
+        {"prog_path": "/local/firmware.elf"}
+    )
+
+    result = platformio_api._load_idedata({"name": "test"})
+
+    assert result["prog_path"] == "/local/firmware.elf"
+    mock_run_platformio_cli_run.assert_called_once()
+
+
 def test_run_idedata_parses_json_from_output(
     setup_core: Path, mock_run_platformio_cli_run: Mock
 ) -> None: