From e557e8446887ba022a20a0d473bbd8ff06101571 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 22 Aug 2026 21:14:18 -0500 Subject: [PATCH] Route the registry prefetch through the shared downloader; register tools caches in one table --- esphome/build_helpers/tools_cache.py | 10 +++++++++ esphome/components/nrf52/framework.py | 4 ++-- esphome/espidf/framework.py | 4 ++-- esphome/platformio/registry.py | 32 +++++++++++++-------------- esphome/writer.py | 11 +++++---- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py index 1960db458f..f23c859b34 100644 --- a/esphome/build_helpers/tools_cache.py +++ b/esphome/build_helpers/tools_cache.py @@ -20,3 +20,13 @@ def tools_cache_path(env_var: str, subdir: str) -> Path: return ( Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir ).resolve() + + +# (env override, cache subdir) per native backend. writer.clean_all wipes +# every entry via tools_cache_path, so listing a cache here is the single +# step that registers it for removal; the backends' own path getters use +# the same named pairs so the two cannot drift. +IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf") +SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") +ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") +TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE) diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 578e91fe40..cffc630fcc 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -7,7 +7,7 @@ import shutil import sys import tempfile -from esphome.build_helpers.tools_cache import tools_cache_path +from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -51,7 +51,7 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str( def get_sdk_nrf_tools_path() -> Path: # Machine-global (OS user cache dir) so all projects share one install; # see espidf.framework.get_idf_tools_path for the location rationale. - return tools_cache_path("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") + return tools_cache_path(*SDK_NRF_TOOLS_CACHE) def _needs_venv_rebuild( diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index ff02da2832..296d4f6fca 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -16,7 +16,7 @@ from esphome.build_helpers.ccache import ( parse_enable_env, resolve_ccache_path, ) -from esphome.build_helpers.tools_cache import tools_cache_path +from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path from esphome.core import Version from esphome.framework_helpers import ( BatchDownloadProgress, @@ -97,7 +97,7 @@ def get_idf_tools_path() -> Path: # Machine-global so all projects share the multi-GB install instead of # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path # for the env-override and normalization rules. - return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf") + return tools_cache_path(*IDF_TOOLS_CACHE) # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index a5c47e9199..d7c9a92a2c 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -4,7 +4,6 @@ platformio package (identical bits, esphome's own download machinery).""" from __future__ import annotations from collections.abc import Collection -from concurrent.futures import ThreadPoolExecutor import io import json import logging @@ -19,6 +18,7 @@ from esphome.framework_helpers import ( download_from_mirrors, download_with_resume, rmdir, + run_batch_downloads, ) _LOGGER = logging.getLogger(__name__) @@ -187,14 +187,11 @@ def prefetch_packages( len(pending), ", ".join(name for name, *_ in pending), ) - progress = BatchDownloadProgress( - "Downloading packages", sum(size for *_, size in pending) - ) - def _fetch(entry: tuple[str, str, Path, str, str, int]) -> None: + def _fetch(entry: tuple[str, str, Path, str, str, int]): name, version, dest, url, sha256, size = entry - tracker = progress.tracker() - try: + + def fetch(tracker): dest.parent.mkdir(parents=True, exist_ok=True) with FileLock(f"{dest}.lock", fallback_to_soft=False): download_with_resume( @@ -204,17 +201,18 @@ def prefetch_packages( size=size, progress=tracker, ) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught - # install_package retries this one itself, with a visible bar - _LOGGER.debug("Prefetch of %s failed: %s", name, err) - ex = ThreadPoolExecutor(max_workers=len(pending)) - try: - for future in [ex.submit(_fetch, entry) for entry in pending]: - future.result() - finally: - ex.shutdown(wait=True, cancel_futures=True) - progress.done() + return fetch + + failures = run_batch_downloads( + BatchDownloadProgress( + "Downloading packages", sum(size for *_, size in pending) + ), + [(entry[0], _fetch(entry)) for entry in pending], + ) + for name, err in failures: + # install_package retries this one itself, with a visible bar + _LOGGER.debug("Prefetch of %s failed: %s", name, err) def install_package( diff --git a/esphome/writer.py b/esphome/writer.py index 847dbe2b2d..955dcba14a 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -690,14 +690,17 @@ def clean_all(configuration: list[str]): # the per-config loop above can't reach. Wipe the default cache root # (also catches leftovers from older install layouts), then the resolved # install paths for the ESPHOME_*_PREFIX overrides (docker/add-on/CI) - # that live outside it. + # that live outside it. Every backend's cache is listed in + # TOOLS_CACHE_SPECS, so registering one there is the only step. import platformdirs - from esphome.components.nrf52.framework import get_sdk_nrf_tools_path - from esphome.espidf.framework import get_idf_tools_path + from esphome.build_helpers.tools_cache import TOOLS_CACHE_SPECS, tools_cache_path cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve() - for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()): + install_paths = [cache_root] + [ + tools_cache_path(*spec) for spec in TOOLS_CACHE_SPECS + ] + for install_path in install_paths: if install_path.is_dir(): _LOGGER.info("Deleting %s", install_path) rmtree(install_path)