[espidf] Cache the discovered component list once per target (#20432)

esphome/build_gen/espidf.py

@@ -237,6 +237,35 @@ def idf_macro_matches(idf_path: Path) -> bool:
     return live == _EXPECTED_MACRO
 
 
+def _cmake_args_for_write(minimal: bool) -> dict[str, str]:
+    """The cg.add_cmake_arg() values for one project write.
+
+    The discovery (minimal) write lets every built-in component register so
+    the list it finds, cached per target, serves any exclusion set; the full
+    write drops the excluded names from that list and passes all of them to
+    IDF. Names outside the framework's components directory (managed
+    components) stay excluded on the discovery write too, since the list
+    only holds built-in ones. A checkout supplied through IDF_PATH caches
+    nothing, so its discovery keeps every exclusion.
+    """
+    from esphome.espidf.toolchain import _esphome_manages_idf, _get_idf_path
+
+    args = dict(CORE.cmake_args)
+    if (
+        not minimal
+        or not _esphome_manages_idf()
+        or not (excluded := args.get("EXCLUDE_COMPONENTS"))
+    ):
+        return args
+    root = _get_idf_path() / "components"
+    outside = [name for name in excluded.split(";") if not (root / name).is_dir()]
+    if outside:
+        args["EXCLUDE_COMPONENTS"] = ";".join(outside)
+    else:
+        del args["EXCLUDE_COMPONENTS"]
+    return args
+
+
 def get_project_cmakelists(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> str:
@@ -300,11 +329,10 @@ def get_project_cmakelists(
 
     # CMake variables registered via cg.add_cmake_arg(). Emitted before
     # include(project.cmake) so values like EXCLUDE_COMPONENTS are already
-    # set when project.cmake seeds the component list, and on minimal
-    # (discovery) writes too so excluded components never register.
+    # set when project.cmake seeds the component list.
     cmake_args = "\n".join(
         f"set({name} {_cmake_quote(value)})"
-        for name, value in sorted(CORE.cmake_args.items())
+        for name, value in sorted(_cmake_args_for_write(minimal).items())
     )
 
     # Per-project list exposed as a CMake variable so converted PIO libs
@@ -327,10 +355,10 @@ def get_project_cmakelists(
     # component's REQUIRES including real IDF components). Referenced by
     # src/CMakeLists and by each converted PIO lib's CMakeLists. Skipped
     # on minimal writes because project_description.json may be stale.
-    # Excluded components are dropped here as well: a stale
-    # project_description.json from a build without exclusions may still
-    # list them, and requiring an excluded component pulls it back into
-    # the build (IDF requirement expansion overrides EXCLUDE_COMPONENTS).
+    # Excluded components are dropped here as well: the discovery list
+    # (cached per target) still holds them, and requiring an excluded
+    # component pulls it back into the build (IDF requirement expansion
+    # overrides EXCLUDE_COMPONENTS).
     # Derived from the EXCLUDE_COMPONENTS cmake arg emitted above so the
     # two can never disagree within one generated file.
     builtin_components_property = (
@@ -554,10 +582,9 @@ def write_project(
     )
 
     # Snapshot the exclusion set so has_outdated_files() can trigger a
-    # discovery reconfigure when it changes. Excluded components never
-    # register in project_description.json, so re-including one (e.g. a
-    # config gains mqtt) requires a fresh discovery pass before the
-    # ESPHOME_PROJECT_BUILTIN_COMPONENTS property can list it.
+    # reconfigure when it changes. The ESPHOME_PROJECT_BUILTIN_COMPONENTS
+    # property drops excluded names, so re-including one (e.g. a config
+    # gains mqtt) needs a reconfigure to rebuild it from the discovery list.
     write_file_if_changed(
         CORE.relative_build_path("exclude_components.esphomeinternal"),
         ";".join(get_excluded_builtin_components()),

esphome/espidf/toolchain.py

@@ -2,7 +2,6 @@
 
 from dataclasses import dataclass, field
 import fnmatch
-import hashlib
 import json
 import logging
 import os
@@ -516,23 +515,21 @@ def _sync_component_mirror() -> None:
 
 
 def _builtin_component_cache_path() -> Path | None:
-    """Cache file for this build's built-in component list.
+    """Cache file for this target's built-in component list.
 
     The file lives inside the extracted framework directory so it is
     discarded together with that exact checkout (re-extract, source
-    override, clean-all); the target and the EXCLUDE_COMPONENTS set name it.
-    The sdkconfig is not part of the key: IDF components register regardless
-    of CONFIG_* options and only gate their sources on them. A checkout
-    supplied through IDF_PATH is not managed by ESPHome and is never cached.
+    override, clean-all). The discovery configure registers every built-in
+    component, so the list serves any EXCLUDE_COMPONENTS set (the project
+    write drops the excluded names) and any sdkconfig (IDF components
+    register regardless of CONFIG_* options and only gate their sources on
+    them). A checkout supplied through IDF_PATH is not managed by ESPHome
+    and is never cached.
     """
     if not _esphome_manages_idf():
         return None
     target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
-    excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "")
-    excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12]
-    return (
-        _get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json"
-    )
+    return _get_idf_path() / ".esphome_component_lists" / f"{target}.json"
 
 
 def load_cached_builtin_components() -> list[str] | None:
@@ -635,10 +632,9 @@ def has_outdated_files():
       already deletes ``dependencies.lock`` on a change but that signal
       gets lost as soon as the lock is missing.
     - ``exclude_components.esphomeinternal`` -- the resolved
-      EXCLUDE_COMPONENTS set. Excluded components never register in
-      ``project_description.json``, so re-including one needs a fresh
-      discovery pass before it can appear in the builtin-components
-      property that ``src`` REQUIRES.
+      EXCLUDE_COMPONENTS set. The builtin-components property that
+      ``src`` REQUIRES drops excluded names, so re-including one needs a
+      reconfigure to rebuild that property from the discovery list.
 
     We deliberately don't watch:
     - The top-level/src ``CMakeLists.txt`` -- ESPHome owns those, and
@@ -803,8 +799,7 @@ def run_compile(config, verbose: bool) -> int:
 
     Uses two-phase configure to auto-discover available components:
     1. If no previous build, configure with minimal REQUIRES to discover
-       components (skipped when a cached list for this IDF/target/exclusion
-       set exists)
+       components (skipped when a cached list for this IDF/target exists)
     2. Regenerate CMakeLists.txt with discovered components
     3. Run full build
     """

tests/unit_tests/build_gen/test_espidf.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 from pathlib import Path
 from unittest.mock import patch
 
@@ -350,17 +351,54 @@ def test_get_project_cmakelists_emits_exclude_components(tmp_path: Path) -> None
     assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd" not in content
 
 
-def test_get_project_cmakelists_minimal_emits_exclude_components() -> None:
-    """The discovery (minimal) write also excludes components so they never
-    register in project_description.json."""
+def test_get_project_cmakelists_minimal_excludes_only_managed_components(
+    tmp_path: Path,
+) -> None:
+    """The discovery (minimal) write lets built-in components register so
+    the cached list serves every exclusion set; managed ones stay excluded."""
+    (tmp_path / "components" / "unity").mkdir(parents=True)
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "espressif__esp_hosted"}
+    register_exclude_components_cmake_arg()
+
+    with patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path):
+        minimal = _render(minimal=True)
+        full = _render()
+
+    assert 'set(EXCLUDE_COMPONENTS "espressif__esp_hosted")' in minimal
+    assert 'set(EXCLUDE_COMPONENTS "espressif__esp_hosted;unity")' in full
+
+
+def test_get_project_cmakelists_minimal_keeps_exclusions_for_custom_idf_path(
+    tmp_path: Path,
+) -> None:
+    """Nothing is cached for a user-supplied IDF_PATH, so its discovery write
+    keeps every exclusion."""
+    (tmp_path / "components" / "unity").mkdir(parents=True)
     CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"}
     register_exclude_components_cmake_arg()
 
-    content = _render(minimal=True)
+    with (
+        patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path),
+        patch.dict(os.environ, {"IDF_PATH": str(tmp_path)}),
+    ):
+        content = _render(minimal=True)
 
     assert 'set(EXCLUDE_COMPONENTS "unity")' in content
 
 
+def test_get_project_cmakelists_minimal_drops_exclude_line_when_all_builtin(
+    tmp_path: Path,
+) -> None:
+    (tmp_path / "components" / "unity").mkdir(parents=True)
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"}
+    register_exclude_components_cmake_arg()
+
+    with patch("esphome.espidf.toolchain._get_idf_path", return_value=tmp_path):
+        content = _render(minimal=True)
+
+    assert "EXCLUDE_COMPONENTS" not in content
+
+
 def test_get_project_cmakelists_no_exclude_components_line_when_empty() -> None:
     """No EXCLUDE_COMPONENTS line at all when nothing is excluded."""
     register_exclude_components_cmake_arg()

tests/unit_tests/test_espidf_toolchain.py

@@ -647,7 +647,7 @@ def test_run_compile_cache_hit_skips_discovery(setup_core: Path) -> None:
 
 
 @contextmanager
-def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]:
+def _cache_env(tmp_path: Path) -> Iterator[Path]:
     """Patch everything the cache key derives from onto a temp IDF tree and
     yield that tree's path."""
     idf_path = tmp_path / "idf"
@@ -655,7 +655,6 @@ def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]:
     with (
         patch.object(toolchain, "_get_idf_path", return_value=idf_path),
         patch.dict(CORE.data, {KEY_ESP32: {KEY_VARIANT: "ESP32"}}),
-        patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": excluded}),
     ):
         yield idf_path
 
@@ -663,7 +662,7 @@ def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]:
 def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None:
     """A saved list is read back until it is dropped."""
     _setup_build(setup_core)
-    with _cache_env(tmp_path, "fatfs") as idf_path:
+    with _cache_env(tmp_path) as idf_path:
         for name in ("lwip", "esp_timer"):
             (idf_path / "components" / name).mkdir()
         assert toolchain.load_cached_builtin_components() is None
@@ -673,25 +672,28 @@ def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None:
         assert toolchain.load_cached_builtin_components() is None
 
 
-def test_component_cache_misses_on_key_change_or_missing_component(
+def test_component_cache_is_per_target_and_drops_missing_component(
     setup_core: Path, tmp_path: Path
 ) -> None:
-    """A different exclusion set uses another entry, an entry naming a
+    """One entry per target serves every exclusion set, an entry naming a
     component that no longer exists is ignored, and a custom IDF_PATH is
     never cached."""
     _setup_build(setup_core)
-    with _cache_env(tmp_path, "fatfs") as idf_path:
+    with _cache_env(tmp_path) as idf_path:
         (idf_path / "components" / "lwip").mkdir()
         toolchain.save_cached_builtin_components(["lwip"])
         path = toolchain._builtin_component_cache_path()
         assert path.parent == idf_path / ".esphome_component_lists"
-        assert path.name.startswith("esp32-")
+        assert path.name == "esp32.json"
         assert toolchain.load_cached_builtin_components() == ["lwip"]
         with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}):
             assert toolchain.load_cached_builtin_components() is None
-    with _cache_env(tmp_path, "fatfs;unity"):
-        assert toolchain.load_cached_builtin_components() is None
-    with _cache_env(tmp_path, "fatfs") as idf_path:
+    with (
+        _cache_env(tmp_path),
+        patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": "fatfs;unity"}),
+    ):
+        assert toolchain.load_cached_builtin_components() == ["lwip"]
+    with _cache_env(tmp_path) as idf_path:
         path.write_text(json.dumps(["lwip", "gone"]))
         assert toolchain.load_cached_builtin_components() is None
         # A plain file with the right name is not a component directory.
@@ -703,7 +705,7 @@ def test_component_cache_save_skips_empty_list_or_custom_idf_path(
     setup_core: Path, tmp_path: Path
 ) -> None:
     _setup_build(setup_core)
-    with _cache_env(tmp_path, "") as idf_path:
+    with _cache_env(tmp_path) as idf_path:
         toolchain.save_cached_builtin_components([])
         with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}):
             toolchain.save_cached_builtin_components(["lwip"])
@@ -715,7 +717,7 @@ def test_component_cache_write_failure_is_logged(
 ) -> None:
     _setup_build(setup_core)
     with (
-        _cache_env(tmp_path, ""),
+        _cache_env(tmp_path),
         patch.object(toolchain, "write_file", side_effect=EsphomeError("disk full")),
     ):
         toolchain.save_cached_builtin_components(["lwip"])
@@ -725,7 +727,7 @@ def test_component_cache_write_failure_is_logged(
 
 def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) -> None:
     _setup_build(setup_core)
-    with _cache_env(tmp_path, ""):
+    with _cache_env(tmp_path):
         path = toolchain._builtin_component_cache_path()
         path.parent.mkdir(parents=True)
         path.write_text("{not json")