Route the registry prefetch through the shared downloader; register tools caches in one table

esphome/build_helpers/tools_cache.py

@@ -20,3 +20,13 @@ def tools_cache_path(env_var: str, subdir: str) -> Path:
     return (
         Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir
     ).resolve()
+
+
+# (env override, cache subdir) per native backend. writer.clean_all wipes
+# every entry via tools_cache_path, so listing a cache here is the single
+# step that registers it for removal; the backends' own path getters use
+# the same named pairs so the two cannot drift.
+IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf")
+SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf")
+ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266")
+TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE)

esphome/components/nrf52/framework.py

@@ -7,7 +7,7 @@ import shutil
 import sys
 import tempfile
 
-from esphome.build_helpers.tools_cache import tools_cache_path
+from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path
 import esphome.config_validation as cv
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
@@ -51,7 +51,7 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str(
 def get_sdk_nrf_tools_path() -> Path:
     # Machine-global (OS user cache dir) so all projects share one install;
     # see espidf.framework.get_idf_tools_path for the location rationale.
-    return tools_cache_path("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf")
+    return tools_cache_path(*SDK_NRF_TOOLS_CACHE)
 
 
 def _needs_venv_rebuild(

esphome/espidf/framework.py

@@ -16,7 +16,7 @@ from esphome.build_helpers.ccache import (
     parse_enable_env,
     resolve_ccache_path,
 )
-from esphome.build_helpers.tools_cache import tools_cache_path
+from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path
 from esphome.core import Version
 from esphome.framework_helpers import (
     BatchDownloadProgress,
@@ -97,7 +97,7 @@ def get_idf_tools_path() -> Path:
     # Machine-global so all projects share the multi-GB install instead of
     # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path
     # for the env-override and normalization rules.
-    return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf")
+    return tools_cache_path(*IDF_TOOLS_CACHE)
 
 
 # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply

esphome/platformio/registry.py

@@ -4,7 +4,6 @@ platformio package (identical bits, esphome's own download machinery)."""
 from __future__ import annotations
 
 from collections.abc import Collection
-from concurrent.futures import ThreadPoolExecutor
 import io
 import json
 import logging
@@ -19,6 +18,7 @@ from esphome.framework_helpers import (
     download_from_mirrors,
     download_with_resume,
     rmdir,
+    run_batch_downloads,
 )
 
 _LOGGER = logging.getLogger(__name__)
@@ -187,14 +187,11 @@ def prefetch_packages(
         len(pending),
         ", ".join(name for name, *_ in pending),
     )
-    progress = BatchDownloadProgress(
-        "Downloading packages", sum(size for *_, size in pending)
-    )
 
-    def _fetch(entry: tuple[str, str, Path, str, str, int]) -> None:
+    def _fetch(entry: tuple[str, str, Path, str, str, int]):
         name, version, dest, url, sha256, size = entry
-        tracker = progress.tracker()
-        try:
+
+        def fetch(tracker):
             dest.parent.mkdir(parents=True, exist_ok=True)
             with FileLock(f"{dest}.lock", fallback_to_soft=False):
                 download_with_resume(
@@ -204,17 +201,18 @@ def prefetch_packages(
                     size=size,
                     progress=tracker,
                 )
-        except Exception as err:  # noqa: BLE001  # pylint: disable=broad-exception-caught
-            # install_package retries this one itself, with a visible bar
-            _LOGGER.debug("Prefetch of %s failed: %s", name, err)
 
-    ex = ThreadPoolExecutor(max_workers=len(pending))
-    try:
-        for future in [ex.submit(_fetch, entry) for entry in pending]:
-            future.result()
-    finally:
-        ex.shutdown(wait=True, cancel_futures=True)
-        progress.done()
+        return fetch
+
+    failures = run_batch_downloads(
+        BatchDownloadProgress(
+            "Downloading packages", sum(size for *_, size in pending)
+        ),
+        [(entry[0], _fetch(entry)) for entry in pending],
+    )
+    for name, err in failures:
+        # install_package retries this one itself, with a visible bar
+        _LOGGER.debug("Prefetch of %s failed: %s", name, err)
 
 
 def install_package(

esphome/writer.py

@@ -690,14 +690,17 @@ def clean_all(configuration: list[str]):
     # the per-config loop above can't reach. Wipe the default cache root
     # (also catches leftovers from older install layouts), then the resolved
     # install paths for the ESPHOME_*_PREFIX overrides (docker/add-on/CI)
-    # that live outside it.
+    # that live outside it. Every backend's cache is listed in
+    # TOOLS_CACHE_SPECS, so registering one there is the only step.
     import platformdirs
 
-    from esphome.components.nrf52.framework import get_sdk_nrf_tools_path
-    from esphome.espidf.framework import get_idf_tools_path
+    from esphome.build_helpers.tools_cache import TOOLS_CACHE_SPECS, tools_cache_path
 
     cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve()
-    for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()):
+    install_paths = [cache_root] + [
+        tools_cache_path(*spec) for spec in TOOLS_CACHE_SPECS
+    ]
+    for install_path in install_paths:
         if install_path.is_dir():
             _LOGGER.info("Deleting %s", install_path)
             rmtree(install_path)