[esp32] Apply IDF component exclusions to native toolchain builds (#18531)

esphome/build_gen/espidf.py

@@ -3,7 +3,12 @@
 import json
 from pathlib import Path
 
-from esphome.components.esp32 import get_esp32_variant, idf_version
+from esphome.components.esp32 import (
+    get_esp32_variant,
+    get_excluded_builtin_components,
+    get_managed_component_require_names,
+    idf_version,
+)
 import esphome.config_validation as cv
 from esphome.core import CORE
 from esphome.framework_helpers import (
@@ -119,24 +124,40 @@ def get_project_cmakelists(minimal: bool = False) -> str:
     # runs as a separate CMake script invocation that doesn't load the
     # project's top-level CMakeLists; without this, ${ESPHOME_PROJECT_
     # MANAGED_COMPONENTS} in a converted-lib REQUIRES expands to empty).
-    from esphome.components.esp32 import get_managed_component_require_names
-
     managed_components_property = "\n".join(
         f"idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS {name} APPEND)"
         for name in get_managed_component_require_names()
     )
 
+    # Components excluded from the build (DEFAULT_EXCLUDED_IDF_COMPONENTS
+    # minus per-component re-includes). project.cmake reads the plain
+    # EXCLUDE_COMPONENTS variable when seeding the component list, so this
+    # must be set before project(). Emitted on minimal writes too so the
+    # discovery reconfigure never registers the excluded components.
+    excluded_components = get_excluded_builtin_components()
+    exclude_components_var = (
+        f'set(EXCLUDE_COMPONENTS "{";".join(excluded_components)}")'
+        if excluded_components
+        else ""
+    )
+
     # Built-in IDF components exposed via our own property (not IDF's
     # __COMPONENT_REQUIRES_COMMON, which would append them to every
     # component's REQUIRES including real IDF components). Referenced by
     # src/CMakeLists and by each converted PIO lib's CMakeLists. Skipped
     # on minimal writes because project_description.json may be stale.
+    # Excluded components are dropped here as well: a stale
+    # project_description.json from a build without exclusions may still
+    # list them, and requiring an excluded component pulls it back into
+    # the build (IDF requirement expansion overrides EXCLUDE_COMPONENTS).
     builtin_components_property = (
         ""
         if minimal
         else "\n".join(
             f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)"
-            for name in sorted(get_available_components() or [])
+            for name in sorted(
+                set(get_available_components() or []).difference(excluded_components)
+            )
         )
     )
 
@@ -165,6 +186,8 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src)
 
 include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
+{exclude_components_var}
+
 {cpp_standard_options}
 
 {cxx_compile_options}
@@ -264,3 +287,13 @@ def write_project(minimal: bool = False) -> None:
         CORE.relative_src_path("CMakeLists.txt"),
         get_component_cmakelists(),
     )
+
+    # Snapshot the exclusion set so has_outdated_files() can trigger a
+    # discovery reconfigure when it changes. Excluded components never
+    # register in project_description.json, so re-including one (e.g. a
+    # config gains mqtt) requires a fresh discovery pass before the
+    # ESPHOME_PROJECT_BUILTIN_COMPONENTS property can list it.
+    write_file_if_changed(
+        CORE.relative_build_path("exclude_components.esphomeinternal"),
+        ";".join(get_excluded_builtin_components()),
+    )

esphome/components/esp32/__init__.py

@@ -738,6 +738,16 @@ def include_builtin_idf_component(name: str) -> None:
     CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS].discard(name)
 
 
+def get_excluded_builtin_components() -> list[str]:
+    """Return the sorted built-in IDF components excluded from the build.
+
+    Single accessor for both build writers: the PlatformIO path passes it as
+    ``-DEXCLUDE_COMPONENTS`` and the native ESP-IDF path emits it into the
+    generated CMakeLists.
+    """
+    return sorted(CORE.data.get(KEY_ESP32, {}).get(KEY_EXCLUDE_COMPONENTS, ()))
+
+
 def _enable_arduino_library(name: str) -> None:
     """Enable an Arduino library that is disabled by default.
 
@@ -2122,13 +2132,10 @@ def _configure_lwip_max_sockets(conf: dict) -> None:
 @coroutine_with_priority(CoroPriority.FINAL)
 async def _write_exclude_components() -> None:
     """Write EXCLUDE_COMPONENTS cmake arg after all components have registered exclusions."""
-    if KEY_ESP32 not in CORE.data:
-        return
-    excluded = CORE.data[KEY_ESP32].get(KEY_EXCLUDE_COMPONENTS)
-    if excluded:
-        exclude_list = ";".join(sorted(excluded))
+    if excluded := get_excluded_builtin_components():
         cg.add_platformio_option(
-            "board_build.cmake_extra_args", f"-DEXCLUDE_COMPONENTS={exclude_list}"
+            "board_build.cmake_extra_args",
+            f"-DEXCLUDE_COMPONENTS={';'.join(excluded)}",
         )
 
 

esphome/espidf/toolchain.py

@@ -273,6 +273,11 @@ def has_outdated_files():
       happen without any sdkconfig impact, and ``_write_idf_component_yml``
       already deletes ``dependencies.lock`` on a change but that signal
       gets lost as soon as the lock is missing.
+    - ``exclude_components.esphomeinternal`` -- the resolved
+      EXCLUDE_COMPONENTS set. Excluded components never register in
+      ``project_description.json``, so re-including one needs a fresh
+      discovery pass before it can appear in the builtin-components
+      property that ``src`` REQUIRES.
 
     We deliberately don't watch:
     - The top-level/src ``CMakeLists.txt`` -- ESPHome owns those, and
@@ -291,6 +296,9 @@ def has_outdated_files():
         f"sdkconfig.{CORE.name}.esphomeinternal"
     )
     idf_component_yml_path = CORE.relative_build_path("src/idf_component.yml")
+    exclude_components_path = CORE.relative_build_path(
+        "exclude_components.esphomeinternal"
+    )
     dependency_lock_path = CORE.relative_build_path("dependencies.lock")
     build_ninja_path = CORE.relative_build_path("build/build.ninja")
 
@@ -309,7 +317,11 @@ def has_outdated_files():
     cmakecache_txt_mtime = cmakecache_txt_path.stat().st_mtime
     return any(
         f.stat().st_mtime > cmakecache_txt_mtime
-        for f in [sdkconfig_internal_path, idf_component_yml_path]
+        for f in [
+            sdkconfig_internal_path,
+            idf_component_yml_path,
+            exclude_components_path,
+        ]
         if f.exists()
     )
 
@@ -386,6 +398,16 @@ def run_compile(config, verbose: bool) -> int:
             return rc
         _LOGGER.info("Regenerating CMakeLists.txt with discovered components...")
         write_project(minimal=False)
+        # Restamp the reference file has_outdated_files() compares against.
+        # A reconfigure that only changes properties or plain variables
+        # (sdkconfig options, the exclusion set) does not rewrite
+        # CMakeCache.txt, so without this the watched inputs stay newer
+        # forever and every subsequent build repeats the discovery pass.
+        # Done after the full write so an interrupt cannot leave a minimal
+        # CMakeLists behind that is already marked fresh.
+        cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
+        if cmakecache.is_file():
+            os.utime(cmakecache)
         if CORE.testing_mode:
             # Reconfigure again so cmake is up to date with the full
             # component list before the build's idf.py invocation runs --

tests/unit_tests/build_gen/test_espidf.py

@@ -11,6 +11,7 @@ import pytest
 from esphome.components.esp32 import (
     KEY_COMPONENTS,
     KEY_ESP32,
+    KEY_EXCLUDE_COMPONENTS,
     KEY_IDF_VERSION,
     KEY_PATH,
     KEY_REF,
@@ -28,6 +29,7 @@ def _reset_core(tmp_path: Path) -> None:
     CORE.data.setdefault(KEY_CORE, {})
     CORE.data[KEY_ESP32] = {
         KEY_COMPONENTS: {},
+        KEY_EXCLUDE_COMPONENTS: set(),
         KEY_IDF_VERSION: cv.Version(5, 5, 4),
     }
 
@@ -47,6 +49,17 @@ def _write_project_description(tmp_path: Path, components: dict[str, str]) -> No
     )
 
 
+def _render(minimal: bool = False) -> str:
+    """Render the top-level CMakeLists with the standard variant/name patches."""
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        return get_project_cmakelists(minimal=minimal)
+
+
 def test_get_available_components_returns_none_without_build_path() -> None:
     """No build_path set yet: must not raise on Path(None)."""
     CORE.build_path = None
@@ -88,13 +101,7 @@ def test_get_project_cmakelists_minimal_omits_builtin_components_property(
     first write before the discovery pass refreshes it)."""
     _write_project_description(tmp_path, {"esp_lcd": "/idf/components/esp_lcd"})
 
-    with (
-        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
-        patch.object(CORE, "name", "test"),
-    ):
-        from esphome.build_gen.espidf import get_project_cmakelists
-
-        content = get_project_cmakelists(minimal=True)
+    content = _render(minimal=True)
 
     assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS" not in content
 
@@ -115,13 +122,7 @@ def test_get_project_cmakelists_full_emits_builtin_components_property(
         },
     )
 
-    with (
-        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
-        patch.object(CORE, "name", "test"),
-    ):
-        from esphome.build_gen.espidf import get_project_cmakelists
-
-        content = get_project_cmakelists(minimal=False)
+    content = _render()
 
     assert (
         "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd APPEND)"
@@ -136,6 +137,92 @@ def test_get_project_cmakelists_full_emits_builtin_components_property(
     assert "JPEGDEC APPEND" not in content
 
 
+def test_get_project_cmakelists_emits_exclude_components(tmp_path: Path) -> None:
+    """Excluded components are passed to IDF via EXCLUDE_COMPONENTS and are
+    dropped from ESPHOME_PROJECT_BUILTIN_COMPONENTS even when a stale
+    project_description.json still lists them (requiring an excluded
+    component would pull it back into the build)."""
+    _write_project_description(
+        tmp_path,
+        {
+            "esp_lcd": "/idf/components/esp_lcd",
+            "freertos": "/idf/components/freertos",
+            "unity": "/idf/components/unity",
+        },
+    )
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"}
+
+    content = _render()
+
+    assert 'set(EXCLUDE_COMPONENTS "esp_lcd;unity")' in content
+    # Must be set before project() so project.cmake sees it.
+    assert content.index("set(EXCLUDE_COMPONENTS") < content.index("project(test)")
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS freertos APPEND)"
+        in content
+    )
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS unity" not in content
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd" not in content
+
+
+def test_get_project_cmakelists_minimal_emits_exclude_components() -> None:
+    """The discovery (minimal) write also excludes components so they never
+    register in project_description.json."""
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"}
+
+    content = _render(minimal=True)
+
+    assert 'set(EXCLUDE_COMPONENTS "unity")' in content
+
+
+def test_get_project_cmakelists_no_exclude_components_line_when_empty() -> None:
+    """No EXCLUDE_COMPONENTS line at all when nothing is excluded."""
+    content = _render()
+
+    assert "EXCLUDE_COMPONENTS" not in content
+
+
+def test_include_builtin_idf_component_removes_exclusion() -> None:
+    """include_builtin_idf_component() drops a name from the exclusion set so
+    a component a config actually uses is not passed to EXCLUDE_COMPONENTS."""
+    from esphome.components.esp32 import (
+        exclude_builtin_idf_component,
+        get_excluded_builtin_components,
+        include_builtin_idf_component,
+    )
+
+    exclude_builtin_idf_component("esp_eth")
+    exclude_builtin_idf_component("unity")
+    include_builtin_idf_component("esp_eth")
+
+    assert get_excluded_builtin_components() == ["unity"]
+
+    content = _render()
+
+    assert 'set(EXCLUDE_COMPONENTS "unity")' in content
+    assert "esp_eth" not in content
+
+
+def test_write_project_writes_exclude_components_stamp(tmp_path: Path) -> None:
+    """write_project() snapshots the exclusion set; the toolchain watches the
+    stamp to trigger a discovery reconfigure when the set changes (excluded
+    components never register in project_description.json)."""
+    CORE.build_flags = set()
+    CORE.build_path = tmp_path
+    CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"}
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import write_project
+
+        write_project()
+
+    stamp = tmp_path / "exclude_components.esphomeinternal"
+    assert stamp.read_text() == "esp_lcd;unity"
+
+
 def test_get_component_cmakelists_no_link_flags() -> None:
     """With no -Wl, flags the target_link_options block is emitted with an empty body."""
     CORE.build_flags = set()

tests/unit_tests/test_espidf_toolchain.py

@@ -100,6 +100,33 @@ def _setup_build(setup_core: Path) -> tuple[Path, Path]:
     return compile_commands, cache
 
 
+def test_has_outdated_files_detects_exclusion_change(setup_core: Path) -> None:
+    """A newer exclude_components.esphomeinternal stamp forces a reconfigure
+    so components that leave the exclusion set get rediscovered."""
+    CORE.build_path = setup_core
+    build = setup_core / "build"
+    (build / "config").mkdir(parents=True)
+    (build / "config" / "sdkconfig.h").write_text("")
+    cmakecache = build / "CMakeCache.txt"
+    cmakecache.write_text("")
+    (build / "build.ninja").write_text("")
+
+    with patch.object(CORE, "name", "test"):
+        assert not toolchain.has_outdated_files()
+
+        stamp = setup_core / "exclude_components.esphomeinternal"
+        stamp.write_text("unity")
+        os.utime(stamp, (cmakecache.stat().st_mtime + 10,) * 2)
+
+        assert toolchain.has_outdated_files()
+
+        # The flag must clear once the reference file is restamped (as
+        # run_compile does after a successful discovery reconfigure);
+        # otherwise every later build would repeat the discovery pass.
+        os.utime(cmakecache, (stamp.stat().st_mtime + 10,) * 2)
+        assert not toolchain.has_outdated_files()
+
+
 def test_get_idedata_returns_none_without_compile_commands(setup_core: Path) -> None:
     """No compile DB yet -> None (rather than an error)."""
     _setup_build(setup_core)
@@ -373,6 +400,48 @@ def test_run_idf_py_jobs_sets_build_jobs_env(setup_core: Path) -> None:
         assert "IDF_PY_BUILD_JOBS" not in env
 
 
+def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> None:
+    """After a successful discovery reconfigure the reference CMakeCache.txt
+    is restamped; cmake does not rewrite it when only properties or plain
+    variables change, so the staleness flag would otherwise never clear."""
+    _setup_build(setup_core)
+    config = {CONF_ESPHOME: {}}
+    cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
+    cmakecache.parent.mkdir(parents=True, exist_ok=True)
+    cmakecache.write_text("")
+    old = cmakecache.stat().st_mtime - 100
+    os.utime(cmakecache, (old, old))
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch("esphome.build_gen.espidf.write_project"),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "run_idf_py", return_value=0),
+        patch.object(toolchain, "print_summary"),
+    ):
+        assert toolchain.run_compile(config, verbose=False) == 0
+
+    assert cmakecache.stat().st_mtime > old
+
+
+def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None:
+    """A discovery pass that produced no CMakeCache.txt (nothing to restamp)
+    still completes normally."""
+    _setup_build(setup_core)
+    config = {CONF_ESPHOME: {}}
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch("esphome.build_gen.espidf.write_project"),
+        patch.object(toolchain, "run_reconfigure", return_value=0),
+        patch.object(toolchain, "run_idf_py", return_value=0),
+        patch.object(toolchain, "print_summary"),
+    ):
+        assert toolchain.run_compile(config, verbose=False) == 0
+
+    assert not CORE.relative_build_path("build/CMakeCache.txt").exists()
+
+
 def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None:
     """compile_process_limit is forwarded to run_idf_py as the job limit."""
     _setup_build(setup_core)