diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index e7a2dc1536..759d9a0bef 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -237,6 +237,35 @@ def idf_macro_matches(idf_path: Path) -> bool: return live == _EXPECTED_MACRO +def _cmake_args_for_write(minimal: bool) -> dict[str, str]: + """The cg.add_cmake_arg() values for one project write. + + The discovery (minimal) write lets every built-in component register so + the list it finds, cached per target, serves any exclusion set; the full + write drops the excluded names from that list and passes all of them to + IDF. Names outside the framework's components directory (managed + components) stay excluded on the discovery write too, since the list + only holds built-in ones. A checkout supplied through IDF_PATH caches + nothing, so its discovery keeps every exclusion. + """ + from esphome.espidf.toolchain import _esphome_manages_idf, _get_idf_path + + args = dict(CORE.cmake_args) + if ( + not minimal + or not _esphome_manages_idf() + or not (excluded := args.get("EXCLUDE_COMPONENTS")) + ): + return args + root = _get_idf_path() / "components" + outside = [name for name in excluded.split(";") if not (root / name).is_dir()] + if outside: + args["EXCLUDE_COMPONENTS"] = ";".join(outside) + else: + del args["EXCLUDE_COMPONENTS"] + return args + + def get_project_cmakelists( minimal: bool = False, builtin_components: list[str] | None = None ) -> str: @@ -300,11 +329,10 @@ def get_project_cmakelists( # CMake variables registered via cg.add_cmake_arg(). Emitted before # include(project.cmake) so values like EXCLUDE_COMPONENTS are already - # set when project.cmake seeds the component list, and on minimal - # (discovery) writes too so excluded components never register. + # set when project.cmake seeds the component list. cmake_args = "\n".join( f"set({name} {_cmake_quote(value)})" - for name, value in sorted(CORE.cmake_args.items()) + for name, value in sorted(_cmake_args_for_write(minimal).items()) ) # Per-project list exposed as a CMake variable so converted PIO libs @@ -327,10 +355,10 @@ def get_project_cmakelists( # component's REQUIRES including real IDF components). Referenced by # src/CMakeLists and by each converted PIO lib's CMakeLists. Skipped # on minimal writes because project_description.json may be stale. - # Excluded components are dropped here as well: a stale - # project_description.json from a build without exclusions may still - # list them, and requiring an excluded component pulls it back into - # the build (IDF requirement expansion overrides EXCLUDE_COMPONENTS). + # Excluded components are dropped here as well: the discovery list + # (cached per target) still holds them, and requiring an excluded + # component pulls it back into the build (IDF requirement expansion + # overrides EXCLUDE_COMPONENTS). # Derived from the EXCLUDE_COMPONENTS cmake arg emitted above so the # two can never disagree within one generated file. builtin_components_property = ( @@ -554,10 +582,9 @@ def write_project( ) # Snapshot the exclusion set so has_outdated_files() can trigger a - # discovery reconfigure when it changes. Excluded components never - # register in project_description.json, so re-including one (e.g. a - # config gains mqtt) requires a fresh discovery pass before the - # ESPHOME_PROJECT_BUILTIN_COMPONENTS property can list it. + # reconfigure when it changes. The ESPHOME_PROJECT_BUILTIN_COMPONENTS + # property drops excluded names, so re-including one (e.g. a config + # gains mqtt) needs a reconfigure to rebuild it from the discovery list. write_file_if_changed( CORE.relative_build_path("exclude_components.esphomeinternal"), ";".join(get_excluded_builtin_components()), diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index effb817f35..af3fda6349 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field import fnmatch -import hashlib import json import logging import os @@ -516,23 +515,21 @@ def _sync_component_mirror() -> None: def _builtin_component_cache_path() -> Path | None: - """Cache file for this build's built-in component list. + """Cache file for this target's built-in component list. The file lives inside the extracted framework directory so it is discarded together with that exact checkout (re-extract, source - override, clean-all); the target and the EXCLUDE_COMPONENTS set name it. - The sdkconfig is not part of the key: IDF components register regardless - of CONFIG_* options and only gate their sources on them. A checkout - supplied through IDF_PATH is not managed by ESPHome and is never cached. + override, clean-all). The discovery configure registers every built-in + component, so the list serves any EXCLUDE_COMPONENTS set (the project + write drops the excluded names) and any sdkconfig (IDF components + register regardless of CONFIG_* options and only gate their sources on + them). A checkout supplied through IDF_PATH is not managed by ESPHome + and is never cached. """ if not _esphome_manages_idf(): return None target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) - excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "") - excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12] - return ( - _get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json" - ) + return _get_idf_path() / ".esphome_component_lists" / f"{target}.json" def load_cached_builtin_components() -> list[str] | None: @@ -635,10 +632,9 @@ def has_outdated_files(): already deletes ``dependencies.lock`` on a change but that signal gets lost as soon as the lock is missing. - ``exclude_components.esphomeinternal`` -- the resolved - EXCLUDE_COMPONENTS set. Excluded components never register in - ``project_description.json``, so re-including one needs a fresh - discovery pass before it can appear in the builtin-components - property that ``src`` REQUIRES. + EXCLUDE_COMPONENTS set. The builtin-components property that + ``src`` REQUIRES drops excluded names, so re-including one needs a + reconfigure to rebuild that property from the discovery list. We deliberately don't watch: - The top-level/src ``CMakeLists.txt`` -- ESPHome owns those, and @@ -803,8 +799,7 @@ def run_compile(config, verbose: bool) -> int: Uses two-phase configure to auto-discover available components: 1. If no previous build, configure with minimal REQUIRES to discover - components (skipped when a cached list for this IDF/target/exclusion - set exists) + components (skipped when a cached list for this IDF/target exists) 2. Regenerate CMakeLists.txt with discovered components 3. Run full build """ diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 2c95c61b37..c439b87d16 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import logging +import os from pathlib import Path from unittest.mock import patch @@ -350,17 +351,54 @@ def test_get_project_cmakelists_emits_exclude_components(tmp_path: Path) -> None assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd" not in content -def test_get_project_cmakelists_minimal_emits_exclude_components() -> None: - """The discovery (minimal) write also excludes components so they never - register in project_description.json.""" +def test_get_project_cmakelists_minimal_excludes_only_managed_components( + tmp_path: Path, +) -> None: + """The discovery (minimal) write lets built-in components register so + the cached list serves every exclusion set; managed ones stay excluded.""" + (tmp_path / "components" / "unity").mkdir(parents=True) + CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "espressif__esp_hosted"} + register_exclude_components_cmake_arg() + + with patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path): + minimal = _render(minimal=True) + full = _render() + + assert 'set(EXCLUDE_COMPONENTS "espressif__esp_hosted")' in minimal + assert 'set(EXCLUDE_COMPONENTS "espressif__esp_hosted;unity")' in full + + +def test_get_project_cmakelists_minimal_keeps_exclusions_for_custom_idf_path( + tmp_path: Path, +) -> None: + """Nothing is cached for a user-supplied IDF_PATH, so its discovery write + keeps every exclusion.""" + (tmp_path / "components" / "unity").mkdir(parents=True) CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"} register_exclude_components_cmake_arg() - content = _render(minimal=True) + with ( + patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path), + patch.dict(os.environ, {"IDF_PATH": str(tmp_path)}), + ): + content = _render(minimal=True) assert 'set(EXCLUDE_COMPONENTS "unity")' in content +def test_get_project_cmakelists_minimal_drops_exclude_line_when_all_builtin( + tmp_path: Path, +) -> None: + (tmp_path / "components" / "unity").mkdir(parents=True) + CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"} + register_exclude_components_cmake_arg() + + with patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path): + content = _render(minimal=True) + + assert "EXCLUDE_COMPONENTS" not in content + + def test_get_project_cmakelists_no_exclude_components_line_when_empty() -> None: """No EXCLUDE_COMPONENTS line at all when nothing is excluded.""" register_exclude_components_cmake_arg() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 854c39a9ce..2a799a4d84 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -647,7 +647,7 @@ def test_run_compile_cache_hit_skips_discovery(setup_core: Path) -> None: @contextmanager -def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]: +def _cache_env(tmp_path: Path) -> Iterator[Path]: """Patch everything the cache key derives from onto a temp IDF tree and yield that tree's path.""" idf_path = tmp_path / "idf" @@ -655,7 +655,6 @@ def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]: with ( patch.object(toolchain, "_get_idf_path", return_value=idf_path), patch.dict(CORE.data, {KEY_ESP32: {KEY_VARIANT: "ESP32"}}), - patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": excluded}), ): yield idf_path @@ -663,7 +662,7 @@ def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]: def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None: """A saved list is read back until it is dropped.""" _setup_build(setup_core) - with _cache_env(tmp_path, "fatfs") as idf_path: + with _cache_env(tmp_path) as idf_path: for name in ("lwip", "esp_timer"): (idf_path / "components" / name).mkdir() assert toolchain.load_cached_builtin_components() is None @@ -673,25 +672,28 @@ def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None: assert toolchain.load_cached_builtin_components() is None -def test_component_cache_misses_on_key_change_or_missing_component( +def test_component_cache_is_per_target_and_drops_missing_component( setup_core: Path, tmp_path: Path ) -> None: - """A different exclusion set uses another entry, an entry naming a + """One entry per target serves every exclusion set, an entry naming a component that no longer exists is ignored, and a custom IDF_PATH is never cached.""" _setup_build(setup_core) - with _cache_env(tmp_path, "fatfs") as idf_path: + with _cache_env(tmp_path) as idf_path: (idf_path / "components" / "lwip").mkdir() toolchain.save_cached_builtin_components(["lwip"]) path = toolchain._builtin_component_cache_path() assert path.parent == idf_path / ".esphome_component_lists" - assert path.name.startswith("esp32-") + assert path.name == "esp32.json" assert toolchain.load_cached_builtin_components() == ["lwip"] with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): assert toolchain.load_cached_builtin_components() is None - with _cache_env(tmp_path, "fatfs;unity"): - assert toolchain.load_cached_builtin_components() is None - with _cache_env(tmp_path, "fatfs") as idf_path: + with ( + _cache_env(tmp_path), + patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": "fatfs;unity"}), + ): + assert toolchain.load_cached_builtin_components() == ["lwip"] + with _cache_env(tmp_path) as idf_path: path.write_text(json.dumps(["lwip", "gone"])) assert toolchain.load_cached_builtin_components() is None # A plain file with the right name is not a component directory. @@ -703,7 +705,7 @@ def test_component_cache_save_skips_empty_list_or_custom_idf_path( setup_core: Path, tmp_path: Path ) -> None: _setup_build(setup_core) - with _cache_env(tmp_path, "") as idf_path: + with _cache_env(tmp_path) as idf_path: toolchain.save_cached_builtin_components([]) with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): toolchain.save_cached_builtin_components(["lwip"]) @@ -715,7 +717,7 @@ def test_component_cache_write_failure_is_logged( ) -> None: _setup_build(setup_core) with ( - _cache_env(tmp_path, ""), + _cache_env(tmp_path), patch.object(toolchain, "write_file", side_effect=EsphomeError("disk full")), ): toolchain.save_cached_builtin_components(["lwip"]) @@ -725,7 +727,7 @@ def test_component_cache_write_failure_is_logged( def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) -> None: _setup_build(setup_core) - with _cache_env(tmp_path, ""): + with _cache_env(tmp_path): path = toolchain._builtin_component_cache_path() path.parent.mkdir(parents=True) path.write_text("{not json")