diff --git a/esphome/__main__.py b/esphome/__main__.py index c17471dbbb..308d0d6653 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -857,18 +857,14 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: toolchain.create_factory_bin() toolchain.create_ota_bin() toolchain.create_elf_copy() + from esphome.build_helpers.idedata import IDEDATA_BEST_EFFORT_ERRORS + try: if toolchain.get_idedata() is None: _LOGGER.warning("No idedata was generated for this build") - except ( - EsphomeError, - LookupError, - OSError, - RuntimeError, - ValueError, - ) as err: - # Broad on purpose: the firmware already built; an idedata - # failure must not fail a successful build. + except IDEDATA_BEST_EFFORT_ERRORS as err: + # The firmware already built; an idedata failure must not fail + # a successful build. _LOGGER.warning("Could not generate idedata: %s", err) _LOGGER.debug("Idedata failure detail", exc_info=True) else: diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 125238a30a..1edbe4b36f 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -15,11 +15,11 @@ from __future__ import annotations import os from pathlib import Path -from typing import Any, NamedTuple +from typing import NamedTuple -from esphome.build_helpers.ccache import ccache_defaults_env, resolve_ccache_path +from esphome.build_helpers.ccache import ccache_defaults_env from esphome.build_helpers.ninja import find_ninja -from esphome.build_helpers.tools_cache import tools_cache_path +from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path from esphome.core import EsphomeError, Version from esphome.framework_helpers import str_to_lst_of_str from esphome.platformio.registry import install_package, prefetch_packages @@ -41,7 +41,7 @@ ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( def get_arduino8266_tools_path() -> Path: # Machine-global so all projects share one install; see # espidf.framework.get_idf_tools_path for the location rationale. - return tools_cache_path("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") + return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) # 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the @@ -90,8 +90,8 @@ class InstalledPaths(NamedTuple): def check_and_install(framework_version: Version) -> InstalledPaths: """Ensure framework, toolchain, and ninja are installed; return their paths.""" if framework_version < MIN_FRAMEWORK_VERSION: - # Config validation will enforce this once the native backend is - # wired in; keep the module honest when called directly. + # Config validation enforces this too; keep the module honest when + # called directly. raise EsphomeError( f"The native toolchain requires the Arduino core " f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" @@ -102,50 +102,33 @@ def check_and_install(framework_version: Version) -> InstalledPaths: framework_path = get_framework_path(package_version) downloads_dir = get_arduino8266_tools_path() / "downloads" toolchain_path = get_toolchain_path() + # One spec per package: the prefetch and the installs must agree + specs = ( + ( + FRAMEWORK_PACKAGE, + package_version, + framework_path, + ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ("cores/esp8266", "tools/sdk", "libraries"), + ), + ( + TOOLCHAIN_PACKAGE, + TOOLCHAIN_VERSION, + toolchain_path, + ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + # xtensa-lx106-elf pins the target: every gcc package has a bin/ + ("bin", "xtensa-lx106-elf"), + ), + ) # Fetch both archives at once; the installs below verify and extract - prefetch_packages( - [ - ( - FRAMEWORK_PACKAGE, - package_version, - framework_path, - ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, - ), - ( - TOOLCHAIN_PACKAGE, - TOOLCHAIN_VERSION, - toolchain_path, - ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, - ), - ], - downloads_dir, - ) - install_package( - FRAMEWORK_PACKAGE, - package_version, - framework_path, - ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, - downloads_dir, - expect=("cores/esp8266", "tools/sdk", "libraries"), - ) - install_package( - TOOLCHAIN_PACKAGE, - TOOLCHAIN_VERSION, - toolchain_path, - ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, - downloads_dir, - # xtensa-lx106-elf pins the target: every gcc package has a bin/ - expect=("bin", "xtensa-lx106-elf"), - ) + prefetch_packages([spec[:4] for spec in specs], downloads_dir) + for name, version, dest, mirrors, expect in specs: + install_package(name, version, dest, mirrors, downloads_dir, expect=expect) return InstalledPaths( framework=framework_path, toolchain=toolchain_path, ninja=ninja_path ) -# Sentinel: "resolve for me"; None is a real value meaning disabled. -CCACHE_UNRESOLVED: Any = object() - - def toolchain_tool(toolchain_path: Path, name: str) -> Path: """Path to one toolchain tool (gcc, g++, ar, size, addr2line, ...). @@ -156,9 +139,7 @@ def toolchain_tool(toolchain_path: Path, name: str) -> Path: return toolchain_path / "bin" / f"xtensa-lx106-elf-{name}{suffix}" -def get_build_env( - toolchain_path: Path, ccache: str | None = CCACHE_UNRESOLVED -) -> dict[str, str]: +def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]: env = os.environ.copy() # Drop empty entries: a trailing separator from an absent PATH would # make the shell search the current directory for tools @@ -171,22 +152,13 @@ def get_build_env( return env -def ccache_path() -> str | None: - """The ccache binary to prefix compiles with, or None when disabled. - - Deliberately uncached: env/PATH can change between builds in a - long-lived host process. - """ - return resolve_ccache_path() - - -def ccache_env(ccache: str | None = CCACHE_UNRESOLVED) -> dict[str, str]: +def ccache_env(ccache: str | None) -> dict[str, str]: """Return ccache settings for the build subprocess (not os.environ). - Values the user already set in the environment are respected. + ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None + when disabled. Values the user already set in the environment are + respected. """ - if ccache is CCACHE_UNRESOLVED: - ccache = ccache_path() if ccache is None: return {} return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") diff --git a/esphome/build_helpers/idedata.py b/esphome/build_helpers/idedata.py index f58b0ec8f8..567a5fa29e 100644 --- a/esphome/build_helpers/idedata.py +++ b/esphome/build_helpers/idedata.py @@ -21,6 +21,17 @@ import subprocess from esphome.core import EsphomeError from esphome.helpers import write_file +# Everything idedata generation may raise after a successful link. Broad on +# purpose, and shared by every consumer: idedata is a bonus artifact, so +# these must be caught and warned about, never allowed to fail the build. +IDEDATA_BEST_EFFORT_ERRORS = ( + EsphomeError, + LookupError, + OSError, + RuntimeError, + ValueError, +) + _LOGGER = logging.getLogger(__name__) # C++ translation-unit suffixes used to identify ESPHome source files. @@ -305,9 +316,24 @@ def idedata_from_build(compile_commands: Path, launcher: str | None = None) -> d build_includes: dict[str, None] = dict.fromkeys( rep_includes if _is_esphome_src(representative["file"]) else () ) + + def _shape(entry: dict) -> str: + # The command minus its TU-specific paths: entries sharing a shape + # carry identical include sets (one ninja rule), so tokenize once + # per shape instead of once per TU + return ( + entry["command"] + .replace(entry.get("file", ""), "") + .replace(entry.get("output", ""), "") + ) + + seen_shapes = {_shape(representative)} for entry in entries: if entry is representative or not _is_esphome_src(entry["file"]): continue + if (shape := _shape(entry)) in seen_shapes: + continue + seen_shapes.add(shape) for inc in parse_entry(entry, launcher)[2]: build_includes.setdefault(inc, None) diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py index 1960db458f..f23c859b34 100644 --- a/esphome/build_helpers/tools_cache.py +++ b/esphome/build_helpers/tools_cache.py @@ -20,3 +20,13 @@ def tools_cache_path(env_var: str, subdir: str) -> Path: return ( Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir ).resolve() + + +# (env override, cache subdir) per native backend. writer.clean_all wipes +# every entry via tools_cache_path, so listing a cache here is the single +# step that registers it for removal; the backends' own path getters use +# the same named pairs so the two cannot drift. +IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf") +SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") +ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") +TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE) diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 578e91fe40..cffc630fcc 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -7,7 +7,7 @@ import shutil import sys import tempfile -from esphome.build_helpers.tools_cache import tools_cache_path +from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -51,7 +51,7 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str( def get_sdk_nrf_tools_path() -> Path: # Machine-global (OS user cache dir) so all projects share one install; # see espidf.framework.get_idf_tools_path for the location rationale. - return tools_cache_path("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") + return tools_cache_path(*SDK_NRF_TOOLS_CACHE) def _needs_venv_rebuild( diff --git a/esphome/core/config.py b/esphome/core/config.py index ef34b02385..472ca64c9a 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -561,9 +561,9 @@ def _add_library_str(lib: str) -> None: NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscript"}) # The full set that survives into CORE.platformio_options under the native # arduino toolchain: lib_ignore is the only specially-translated key below -# that is stored rather than translated away. Intentionally unused until the -# esp8266 native backend (the final PR in this chain) consumes it for its -# ignored-option warning; defined here so it stays adjacent to the routing. +# that is stored rather than translated away. Consumed by the esp8266 native +# backend (later in this chain) for its ignored-option warning; defined here +# so it stays adjacent to the routing. NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | {"lib_ignore"} diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 8d6d09e0f6..296d4f6fca 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -1,7 +1,6 @@ """ESP-IDF framework tools for ESPHome.""" from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor from ctypes.util import find_library import json import logging @@ -17,7 +16,7 @@ from esphome.build_helpers.ccache import ( parse_enable_env, resolve_ccache_path, ) -from esphome.build_helpers.tools_cache import tools_cache_path +from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path from esphome.core import Version from esphome.framework_helpers import ( BatchDownloadProgress, @@ -29,6 +28,7 @@ from esphome.framework_helpers import ( get_python_env_executable_path, get_system_python_path, rmdir, + run_batch_downloads, run_command, run_command_ok, str_to_lst_of_str, @@ -97,7 +97,7 @@ def get_idf_tools_path() -> Path: # Machine-global so all projects share the multi-GB install instead of # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path # for the env-override and normalization rules. - return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf") + return tools_cache_path(*IDF_TOOLS_CACHE) # Windows' default MAX_PATH is 260 characters. ESP-IDF toolchains nest deeply @@ -684,12 +684,6 @@ def _patch_tools_json_demote_unused_tools(framework_path: Path) -> None: ) -# Tool archives are large (tens to hundreds of MB) and served by GitHub / -# dl.espressif.com; a few streams at once saturate most links without -# hammering the host. Smaller than external_files' 8: those are tiny files. -_PREFETCH_WORKERS = 4 - - def _prefetch_idf_tool_archives( framework_path: Path, targets_str: str, @@ -727,17 +721,16 @@ def _prefetch_idf_tool_archives( ) return dist_path = get_idf_tools_path() / "dist" - pending = [ - entry - for entry in json.loads(stdout) - if not (dist_path / entry["dest"]).is_file() - ] - # tools.json always carries sha256 and size; an entry missing either - # must not be downloaded unverified here, so leave it to the - # installer (which fails loudly on a bad archive). - entries = [e for e in pending if e.get("sha256") and e.get("size")] - for entry in pending: - if entry not in entries: + entries = [] + for entry in json.loads(stdout): + if (dist_path / entry["dest"]).is_file(): + continue + # tools.json always carries sha256 and size; an entry missing + # either must not be downloaded unverified here, so leave it to + # the installer (which fails loudly on a bad archive). + if entry.get("sha256") and entry.get("size"): + entries.append(entry) + else: _LOGGER.warning( "Tool %s has no sha256/size in the download list; " "leaving it to the installer", @@ -750,42 +743,28 @@ def _prefetch_idf_tool_archives( len(entries), ", ".join(entry["name"] for entry in entries), ) + # Every entry carries a size (checked above), so the combined bar can # be trusted. Unlike the library prefetch there is no sequential # fallback: per-file bars from several threads would interleave, and # skipping the prefetch would lose the resume workaround for #17703. - progress = BatchDownloadProgress( - "Downloading ESP-IDF tools", sum(entry["size"] for entry in entries) + def _download(entry: dict): + return lambda tracker: download_with_resume( + entry["url"], + dist_path / entry["dest"], + sha256=entry["sha256"], + size=entry["size"], + progress=tracker, + ) + + # A failed archive is retried by the installer itself (without + # resume); keep prefetching the rest. + failures = run_batch_downloads( + BatchDownloadProgress( + "Downloading ESP-IDF tools", sum(entry["size"] for entry in entries) + ), + [(entry["name"], _download(entry)) for entry in entries], ) - # Reported after the bar is done so the warnings do not land on - # its row; list.append is atomic under the GIL. - failures: list[tuple[str, Exception]] = [] - - def _download(entry: dict) -> None: - tracker = progress.tracker() - try: - download_with_resume( - entry["url"], - dist_path / entry["dest"], - sha256=entry["sha256"], - size=entry["size"], - progress=tracker, - ) - except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught - # Keep prefetching the remaining archives; the installer - # will retry this one itself (without resume). - tracker(0) - failures.append((entry["name"], e)) - - ex = ThreadPoolExecutor(max_workers=min(_PREFETCH_WORKERS, len(entries))) - try: - for future in [ex.submit(_download, entry) for entry in entries]: - future.result() - finally: - # On Ctrl-C drop the queued archives instead of downloading them - # all before the process can exit; in-flight ones still finish. - ex.shutdown(wait=True, cancel_futures=True) - progress.done() for name, e in failures: _LOGGER.warning("Could not prefetch %s: %s", name, e) except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index fc9346bed9..7911acb111 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -1,6 +1,7 @@ """Generic toolchain installation helpers shared across framework implementations.""" from collections.abc import Callable, Iterable +from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack import hashlib import io @@ -759,6 +760,46 @@ def _stream_response_to_file( own_bar.update(1) +# Concurrent downloads per batch; enough to hide latency without +# hammering the host or the mirrors. +BATCH_DOWNLOAD_WORKERS = 4 + + +def run_batch_downloads( + progress: "BatchDownloadProgress", + jobs: list[tuple[str, Callable[[Callable[[int], None]], None]]], + max_workers: int = BATCH_DOWNLOAD_WORKERS, +) -> list[tuple[str, Exception]]: + """Run download jobs concurrently, reporting into one combined bar. + + ``jobs`` holds ``(name, fetch)`` pairs where ``fetch(tracker)`` performs + one download reporting absolute byte counts to ``tracker``. Failures are + collected (list.append is atomic under the GIL) and returned after the + bar is done, so the caller's warnings never land on the bar's row; a + failed job credits its tracker 0 so the bar can still complete. Ctrl-C + drops queued jobs instead of downloading them all before the process + can exit; in-flight ones still finish. + """ + failures: list[tuple[str, Exception]] = [] + + def _run(name: str, fetch: Callable[[Callable[[int], None]], None]) -> None: + tracker = progress.tracker() + try: + fetch(tracker) + except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught + failures.append((name, err)) + tracker(0) + + ex = ThreadPoolExecutor(max_workers=min(max_workers, len(jobs))) + try: + for future in [ex.submit(_run, name, fetch) for name, fetch in jobs]: + future.result() + finally: + ex.shutdown(wait=True, cancel_futures=True) + progress.done() + return failures + + class BatchDownloadProgress: """One progress bar across several concurrent ``download_with_resume`` calls. diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 0e3a942724..12f076edd4 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -32,10 +32,12 @@ from urllib.request import url2pathname from esphome import git from esphome.core import CORE, EsphomeError, Library from esphome.framework_helpers import ( + BATCH_DOWNLOAD_WORKERS, BatchDownloadProgress, archive_extract_all, download_from_mirrors, rmdir, + run_batch_downloads, ) _LOGGER = logging.getLogger(__name__) @@ -949,10 +951,6 @@ def _warn_unsatisfied_versionless( ) -# A few streams saturate most links without hammering the registry -_DOWNLOAD_WORKERS = 4 - - def _content_lengths(urls: list[str]) -> list[int | None]: """Content-Length per URL via HEAD requests; None when unknown.""" import requests @@ -968,7 +966,7 @@ def _content_lengths(urls: list[str]) -> list[int | None]: _LOGGER.debug("HEAD %s failed: %s", url, err) return None - with ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(urls))) as ex: + with ThreadPoolExecutor(max_workers=min(BATCH_DOWNLOAD_WORKERS, len(urls))) as ex: return list(ex.map(head, urls)) @@ -1025,28 +1023,16 @@ def _prefetch_wave( ), ) return - progress = BatchDownloadProgress("Downloading libraries", sum(sizes)) - # Reported after the bar is done so the warnings do not land on its - # row; list.append is atomic under the GIL. - failures: list[tuple[str, Exception]] = [] - def _fetch(component: ConvertedLibrary) -> None: - tracker = progress.tracker() - try: - component.download(salt=salt, namespace=namespace, progress=tracker) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught - failures.append((component.name, err)) - tracker(0) + def _fetch(component: ConvertedLibrary): + return lambda tracker: component.download( + salt=salt, namespace=namespace, progress=tracker + ) - ex = ThreadPoolExecutor(max_workers=min(_DOWNLOAD_WORKERS, len(components))) - try: - for future in [ex.submit(_fetch, component) for component in components]: - future.result() - finally: - # On Ctrl-C drop the queued archives instead of downloading them - # all before the process can exit; in-flight ones still finish. - ex.shutdown(wait=True, cancel_futures=True) - progress.done() + failures = run_batch_downloads( + BatchDownloadProgress("Downloading libraries", sum(sizes)), + [(component.name, _fetch(component)) for component in components], + ) for name, err in failures: # The sequential call below retries and raises the real error _LOGGER.warning("Prefetch of %s failed (retrying sequentially): %s", name, err) diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index a5c47e9199..d7c9a92a2c 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -4,7 +4,6 @@ platformio package (identical bits, esphome's own download machinery).""" from __future__ import annotations from collections.abc import Collection -from concurrent.futures import ThreadPoolExecutor import io import json import logging @@ -19,6 +18,7 @@ from esphome.framework_helpers import ( download_from_mirrors, download_with_resume, rmdir, + run_batch_downloads, ) _LOGGER = logging.getLogger(__name__) @@ -187,14 +187,11 @@ def prefetch_packages( len(pending), ", ".join(name for name, *_ in pending), ) - progress = BatchDownloadProgress( - "Downloading packages", sum(size for *_, size in pending) - ) - def _fetch(entry: tuple[str, str, Path, str, str, int]) -> None: + def _fetch(entry: tuple[str, str, Path, str, str, int]): name, version, dest, url, sha256, size = entry - tracker = progress.tracker() - try: + + def fetch(tracker): dest.parent.mkdir(parents=True, exist_ok=True) with FileLock(f"{dest}.lock", fallback_to_soft=False): download_with_resume( @@ -204,17 +201,18 @@ def prefetch_packages( size=size, progress=tracker, ) - except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught - # install_package retries this one itself, with a visible bar - _LOGGER.debug("Prefetch of %s failed: %s", name, err) - ex = ThreadPoolExecutor(max_workers=len(pending)) - try: - for future in [ex.submit(_fetch, entry) for entry in pending]: - future.result() - finally: - ex.shutdown(wait=True, cancel_futures=True) - progress.done() + return fetch + + failures = run_batch_downloads( + BatchDownloadProgress( + "Downloading packages", sum(size for *_, size in pending) + ), + [(entry[0], _fetch(entry)) for entry in pending], + ) + for name, err in failures: + # install_package retries this one itself, with a visible bar + _LOGGER.debug("Prefetch of %s failed: %s", name, err) def install_package( diff --git a/esphome/writer.py b/esphome/writer.py index 6e70f320a9..955dcba14a 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -690,20 +690,17 @@ def clean_all(configuration: list[str]): # the per-config loop above can't reach. Wipe the default cache root # (also catches leftovers from older install layouts), then the resolved # install paths for the ESPHOME_*_PREFIX overrides (docker/add-on/CI) - # that live outside it. + # that live outside it. Every backend's cache is listed in + # TOOLS_CACHE_SPECS, so registering one there is the only step. import platformdirs - from esphome.arduino8266.framework import get_arduino8266_tools_path - from esphome.components.nrf52.framework import get_sdk_nrf_tools_path - from esphome.espidf.framework import get_idf_tools_path + from esphome.build_helpers.tools_cache import TOOLS_CACHE_SPECS, tools_cache_path cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve() - for install_path in ( - cache_root, - get_idf_tools_path(), - get_sdk_nrf_tools_path(), - get_arduino8266_tools_path(), - ): + install_paths = [cache_root] + [ + tools_cache_path(*spec) for spec in TOOLS_CACHE_SPECS + ] + for install_path in install_paths: if install_path.is_dir(): _LOGGER.info("Deleting %s", install_path) rmtree(install_path) diff --git a/tests/unit_tests/build_helpers/test_idedata.py b/tests/unit_tests/build_helpers/test_idedata.py index 491daa0ec0..ff91ca34c1 100644 --- a/tests/unit_tests/build_helpers/test_idedata.py +++ b/tests/unit_tests/build_helpers/test_idedata.py @@ -151,6 +151,40 @@ def test_is_esphome_src_handles_backslash_paths() -> None: assert not idedata._is_esphome_src(r"C:\b\src\esphome\core\app.h") +def test_idedata_from_build_dedupes_identical_command_shapes( + tmp_path: Path, +) -> None: + """Translation units sharing one ninja rule (same command modulo + file/output) carry + identical includes, so only one per shape is tokenized; a differing + shape still contributes its includes.""" + + def _tu(name: str, inc: str) -> dict: + # ninja's compdb embeds the file and output strings verbatim + file = f"{ABS}build/src/esphome/core/{name}.cpp" + return _entry( + f"{ABS}build", file, f"/tools/g++ -I{ABS}inc/{inc} -c {file} -o {name}.o" + ) | {"output": f"{name}.o"} + + entries = [_tu(name, "shared") for name in ("application", "component", "helpers")] + entries.append(_tu("extra", "extra")) + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text(json.dumps(entries)) + with ( + patch.object(idedata, "get_toolchain_includes", return_value=[]), + patch.object(idedata, "parse_entry", wraps=idedata.parse_entry) as spy, + ): + data = idedata.idedata_from_build(compile_commands) + includes = set(data["includes"]["build"]) + assert f"{ABS}inc/shared".replace("\\", "/") in { + i.replace("\\", "/") for i in includes + } + assert any("inc/extra" in i for i in includes) + # Representative + one distinct shape; the two same-shape duplicates + # are never tokenized + assert spy.call_count == 2 + + def test_idedata_from_build(tmp_path: Path) -> None: """Full transform: representative entry + include union + toolchain dirs.""" compile_commands = tmp_path / "compile_commands.json" diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 932958d667..bd0a620e10 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -112,33 +112,15 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: with patch.object(framework, "ccache_env", return_value={"CCACHE_DIR": "x"}): - env = framework.get_build_env(tmp_path) + env = framework.get_build_env(tmp_path, None) assert env["PATH"].startswith(str(tmp_path / "bin") + os.pathsep) assert env["CCACHE_DIR"] == "x" -def test_ccache_path_delegates_uncached( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Delegates on every call; the env/PATH decision must not freeze for - the process lifetime.""" - monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False) - with patch.object( - framework, "resolve_ccache_path", return_value="/usr/bin/ccache" - ) as mock_resolve: - assert framework.ccache_path() == "/usr/bin/ccache" - assert framework.ccache_path() == "/usr/bin/ccache" - assert mock_resolve.call_count == 2 - - def test_ccache_env(tmp_path: Path) -> None: - with patch.object(framework, "ccache_path", return_value=None): - assert framework.ccache_env() == {} - with ( - patch.object(framework, "ccache_path", return_value="/usr/bin/ccache"), - patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True), - ): - env = framework.ccache_env() + assert framework.ccache_env(None) == {} + with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True): + env = framework.ccache_env("/usr/bin/ccache") # User-set values are respected; the rest get defaults assert "CCACHE_NOHASHDIR" not in env assert env["CCACHE_DEPEND"] == "1" @@ -159,7 +141,7 @@ def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: patch.dict(os.environ, {}, clear=True), patch.object(framework, "ccache_env", return_value={}), ): - env = framework.get_build_env(tmp_path) + env = framework.get_build_env(tmp_path, None) assert env["PATH"] == str(tmp_path / "bin") with ( patch.dict( @@ -167,20 +149,16 @@ def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: ), patch.object(framework, "ccache_env", return_value={}), ): - env = framework.get_build_env(tmp_path) + env = framework.get_build_env(tmp_path, None) assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"] def test_ccache_env_accepts_a_preresolved_path() -> None: - """A caller that already resolved ccache threads it through; the probe - must not run again (None means resolved-and-disabled).""" - with ( - patch.dict(os.environ, {}, clear=True), - patch.object(framework, "ccache_path") as mock_resolve, - ): + """The caller resolves ccache once and threads it through; None means + resolved-and-disabled.""" + with patch.dict(os.environ, {}, clear=True): assert framework.ccache_env(None) == {} env = framework.ccache_env("/usr/bin/ccache") - mock_resolve.assert_not_called() assert env["CCACHE_DIR"].endswith("ccache") diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index a604a9ecfb..cf62b48071 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -989,7 +989,7 @@ def test_prefetch_downloads_archives_concurrently(tmp_path: Path) -> None: patch("esphome.espidf.framework.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch( - "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor + "esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor ) as pool, ): _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) @@ -1008,7 +1008,7 @@ def test_prefetch_single_archive_uses_one_worker(tmp_path: Path) -> None: patch("esphome.espidf.framework.download_with_resume") as download, patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch( - "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor + "esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor ) as pool, ): _prefetch_idf_tool_archives(tmp_path, "esp32", ["required"], None) @@ -1108,7 +1108,7 @@ def test_prefetch_finishes_progress_bar_and_cancels_queue(tmp_path: Path) -> Non patch("esphome.espidf.framework.get_system_python_path", return_value="python"), patch("esphome.espidf.framework.BatchDownloadProgress") as progress_cls, patch( - "esphome.espidf.framework.ThreadPoolExecutor", wraps=ThreadPoolExecutor + "esphome.framework_helpers.ThreadPoolExecutor", wraps=ThreadPoolExecutor ) as pool_cls, ): pool = MagicMock(wraps=ThreadPoolExecutor(max_workers=2))