From a8e721abebdda3a42b1b6ecf391a90938b2187ac Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:03:49 -0500 Subject: [PATCH] [esp32] Apply IDF component exclusions to native toolchain builds (#18531) --- esphome/build_gen/espidf.py | 41 +++++++- esphome/components/esp32/__init__.py | 19 ++-- esphome/espidf/toolchain.py | 24 ++++- tests/unit_tests/build_gen/test_espidf.py | 115 +++++++++++++++++++--- tests/unit_tests/test_espidf_toolchain.py | 69 +++++++++++++ 5 files changed, 243 insertions(+), 25 deletions(-) diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index cf476555e7..b65ce23307 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -3,7 +3,12 @@ import json from pathlib import Path -from esphome.components.esp32 import get_esp32_variant, idf_version +from esphome.components.esp32 import ( + get_esp32_variant, + get_excluded_builtin_components, + get_managed_component_require_names, + idf_version, +) import esphome.config_validation as cv from esphome.core import CORE from esphome.framework_helpers import ( @@ -119,24 +124,40 @@ def get_project_cmakelists(minimal: bool = False) -> str: # runs as a separate CMake script invocation that doesn't load the # project's top-level CMakeLists; without this, ${ESPHOME_PROJECT_ # MANAGED_COMPONENTS} in a converted-lib REQUIRES expands to empty). - from esphome.components.esp32 import get_managed_component_require_names - managed_components_property = "\n".join( f"idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS {name} APPEND)" for name in get_managed_component_require_names() ) + # Components excluded from the build (DEFAULT_EXCLUDED_IDF_COMPONENTS + # minus per-component re-includes). project.cmake reads the plain + # EXCLUDE_COMPONENTS variable when seeding the component list, so this + # must be set before project(). Emitted on minimal writes too so the + # discovery reconfigure never registers the excluded components. + excluded_components = get_excluded_builtin_components() + exclude_components_var = ( + f'set(EXCLUDE_COMPONENTS "{";".join(excluded_components)}")' + if excluded_components + else "" + ) + # Built-in IDF components exposed via our own property (not IDF's # __COMPONENT_REQUIRES_COMMON, which would append them to every # component's REQUIRES including real IDF components). Referenced by # src/CMakeLists and by each converted PIO lib's CMakeLists. Skipped # on minimal writes because project_description.json may be stale. + # Excluded components are dropped here as well: a stale + # project_description.json from a build without exclusions may still + # list them, and requiring an excluded component pulls it back into + # the build (IDF requirement expansion overrides EXCLUDE_COMPONENTS). builtin_components_property = ( "" if minimal else "\n".join( f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)" - for name in sorted(get_available_components() or []) + for name in sorted( + set(get_available_components() or []).difference(excluded_components) + ) ) ) @@ -165,6 +186,8 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{exclude_components_var} + {cpp_standard_options} {cxx_compile_options} @@ -264,3 +287,13 @@ def write_project(minimal: bool = False) -> None: CORE.relative_src_path("CMakeLists.txt"), get_component_cmakelists(), ) + + # Snapshot the exclusion set so has_outdated_files() can trigger a + # discovery reconfigure when it changes. Excluded components never + # register in project_description.json, so re-including one (e.g. a + # config gains mqtt) requires a fresh discovery pass before the + # ESPHOME_PROJECT_BUILTIN_COMPONENTS property can list it. + write_file_if_changed( + CORE.relative_build_path("exclude_components.esphomeinternal"), + ";".join(get_excluded_builtin_components()), + ) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 3065cdadad..d6e0890751 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -738,6 +738,16 @@ def include_builtin_idf_component(name: str) -> None: CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS].discard(name) +def get_excluded_builtin_components() -> list[str]: + """Return the sorted built-in IDF components excluded from the build. + + Single accessor for both build writers: the PlatformIO path passes it as + ``-DEXCLUDE_COMPONENTS`` and the native ESP-IDF path emits it into the + generated CMakeLists. + """ + return sorted(CORE.data.get(KEY_ESP32, {}).get(KEY_EXCLUDE_COMPONENTS, ())) + + def _enable_arduino_library(name: str) -> None: """Enable an Arduino library that is disabled by default. @@ -2122,13 +2132,10 @@ def _configure_lwip_max_sockets(conf: dict) -> None: @coroutine_with_priority(CoroPriority.FINAL) async def _write_exclude_components() -> None: """Write EXCLUDE_COMPONENTS cmake arg after all components have registered exclusions.""" - if KEY_ESP32 not in CORE.data: - return - excluded = CORE.data[KEY_ESP32].get(KEY_EXCLUDE_COMPONENTS) - if excluded: - exclude_list = ";".join(sorted(excluded)) + if excluded := get_excluded_builtin_components(): cg.add_platformio_option( - "board_build.cmake_extra_args", f"-DEXCLUDE_COMPONENTS={exclude_list}" + "board_build.cmake_extra_args", + f"-DEXCLUDE_COMPONENTS={';'.join(excluded)}", ) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index bb6452acf2..07ba03e2cf 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -273,6 +273,11 @@ def has_outdated_files(): happen without any sdkconfig impact, and ``_write_idf_component_yml`` already deletes ``dependencies.lock`` on a change but that signal gets lost as soon as the lock is missing. + - ``exclude_components.esphomeinternal`` -- the resolved + EXCLUDE_COMPONENTS set. Excluded components never register in + ``project_description.json``, so re-including one needs a fresh + discovery pass before it can appear in the builtin-components + property that ``src`` REQUIRES. We deliberately don't watch: - The top-level/src ``CMakeLists.txt`` -- ESPHome owns those, and @@ -291,6 +296,9 @@ def has_outdated_files(): f"sdkconfig.{CORE.name}.esphomeinternal" ) idf_component_yml_path = CORE.relative_build_path("src/idf_component.yml") + exclude_components_path = CORE.relative_build_path( + "exclude_components.esphomeinternal" + ) dependency_lock_path = CORE.relative_build_path("dependencies.lock") build_ninja_path = CORE.relative_build_path("build/build.ninja") @@ -309,7 +317,11 @@ def has_outdated_files(): cmakecache_txt_mtime = cmakecache_txt_path.stat().st_mtime return any( f.stat().st_mtime > cmakecache_txt_mtime - for f in [sdkconfig_internal_path, idf_component_yml_path] + for f in [ + sdkconfig_internal_path, + idf_component_yml_path, + exclude_components_path, + ] if f.exists() ) @@ -386,6 +398,16 @@ def run_compile(config, verbose: bool) -> int: return rc _LOGGER.info("Regenerating CMakeLists.txt with discovered components...") write_project(minimal=False) + # Restamp the reference file has_outdated_files() compares against. + # A reconfigure that only changes properties or plain variables + # (sdkconfig options, the exclusion set) does not rewrite + # CMakeCache.txt, so without this the watched inputs stay newer + # forever and every subsequent build repeats the discovery pass. + # Done after the full write so an interrupt cannot leave a minimal + # CMakeLists behind that is already marked fresh. + cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + if cmakecache.is_file(): + os.utime(cmakecache) if CORE.testing_mode: # Reconfigure again so cmake is up to date with the full # component list before the build's idf.py invocation runs -- diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index f21549b48c..ec01000920 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -11,6 +11,7 @@ import pytest from esphome.components.esp32 import ( KEY_COMPONENTS, KEY_ESP32, + KEY_EXCLUDE_COMPONENTS, KEY_IDF_VERSION, KEY_PATH, KEY_REF, @@ -28,6 +29,7 @@ def _reset_core(tmp_path: Path) -> None: CORE.data.setdefault(KEY_CORE, {}) CORE.data[KEY_ESP32] = { KEY_COMPONENTS: {}, + KEY_EXCLUDE_COMPONENTS: set(), KEY_IDF_VERSION: cv.Version(5, 5, 4), } @@ -47,6 +49,17 @@ def _write_project_description(tmp_path: Path, components: dict[str, str]) -> No ) +def _render(minimal: bool = False) -> str: + """Render the top-level CMakeLists with the standard variant/name patches.""" + with ( + patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), + patch.object(CORE, "name", "test"), + ): + from esphome.build_gen.espidf import get_project_cmakelists + + return get_project_cmakelists(minimal=minimal) + + def test_get_available_components_returns_none_without_build_path() -> None: """No build_path set yet: must not raise on Path(None).""" CORE.build_path = None @@ -88,13 +101,7 @@ def test_get_project_cmakelists_minimal_omits_builtin_components_property( first write before the discovery pass refreshes it).""" _write_project_description(tmp_path, {"esp_lcd": "/idf/components/esp_lcd"}) - with ( - patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), - patch.object(CORE, "name", "test"), - ): - from esphome.build_gen.espidf import get_project_cmakelists - - content = get_project_cmakelists(minimal=True) + content = _render(minimal=True) assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS" not in content @@ -115,13 +122,7 @@ def test_get_project_cmakelists_full_emits_builtin_components_property( }, ) - with ( - patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), - patch.object(CORE, "name", "test"), - ): - from esphome.build_gen.espidf import get_project_cmakelists - - content = get_project_cmakelists(minimal=False) + content = _render() assert ( "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd APPEND)" @@ -136,6 +137,92 @@ def test_get_project_cmakelists_full_emits_builtin_components_property( assert "JPEGDEC APPEND" not in content +def test_get_project_cmakelists_emits_exclude_components(tmp_path: Path) -> None: + """Excluded components are passed to IDF via EXCLUDE_COMPONENTS and are + dropped from ESPHOME_PROJECT_BUILTIN_COMPONENTS even when a stale + project_description.json still lists them (requiring an excluded + component would pull it back into the build).""" + _write_project_description( + tmp_path, + { + "esp_lcd": "/idf/components/esp_lcd", + "freertos": "/idf/components/freertos", + "unity": "/idf/components/unity", + }, + ) + CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"} + + content = _render() + + assert 'set(EXCLUDE_COMPONENTS "esp_lcd;unity")' in content + # Must be set before project() so project.cmake sees it. + assert content.index("set(EXCLUDE_COMPONENTS") < content.index("project(test)") + assert ( + "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS freertos APPEND)" + in content + ) + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS unity" not in content + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd" not in content + + +def test_get_project_cmakelists_minimal_emits_exclude_components() -> None: + """The discovery (minimal) write also excludes components so they never + register in project_description.json.""" + CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity"} + + content = _render(minimal=True) + + assert 'set(EXCLUDE_COMPONENTS "unity")' in content + + +def test_get_project_cmakelists_no_exclude_components_line_when_empty() -> None: + """No EXCLUDE_COMPONENTS line at all when nothing is excluded.""" + content = _render() + + assert "EXCLUDE_COMPONENTS" not in content + + +def test_include_builtin_idf_component_removes_exclusion() -> None: + """include_builtin_idf_component() drops a name from the exclusion set so + a component a config actually uses is not passed to EXCLUDE_COMPONENTS.""" + from esphome.components.esp32 import ( + exclude_builtin_idf_component, + get_excluded_builtin_components, + include_builtin_idf_component, + ) + + exclude_builtin_idf_component("esp_eth") + exclude_builtin_idf_component("unity") + include_builtin_idf_component("esp_eth") + + assert get_excluded_builtin_components() == ["unity"] + + content = _render() + + assert 'set(EXCLUDE_COMPONENTS "unity")' in content + assert "esp_eth" not in content + + +def test_write_project_writes_exclude_components_stamp(tmp_path: Path) -> None: + """write_project() snapshots the exclusion set; the toolchain watches the + stamp to trigger a discovery reconfigure when the set changes (excluded + components never register in project_description.json).""" + CORE.build_flags = set() + CORE.build_path = tmp_path + CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] = {"unity", "esp_lcd"} + + with ( + patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), + patch.object(CORE, "name", "test"), + ): + from esphome.build_gen.espidf import write_project + + write_project() + + stamp = tmp_path / "exclude_components.esphomeinternal" + assert stamp.read_text() == "esp_lcd;unity" + + def test_get_component_cmakelists_no_link_flags() -> None: """With no -Wl, flags the target_link_options block is emitted with an empty body.""" CORE.build_flags = set() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 26d812af8b..2556397aef 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -100,6 +100,33 @@ def _setup_build(setup_core: Path) -> tuple[Path, Path]: return compile_commands, cache +def test_has_outdated_files_detects_exclusion_change(setup_core: Path) -> None: + """A newer exclude_components.esphomeinternal stamp forces a reconfigure + so components that leave the exclusion set get rediscovered.""" + CORE.build_path = setup_core + build = setup_core / "build" + (build / "config").mkdir(parents=True) + (build / "config" / "sdkconfig.h").write_text("") + cmakecache = build / "CMakeCache.txt" + cmakecache.write_text("") + (build / "build.ninja").write_text("") + + with patch.object(CORE, "name", "test"): + assert not toolchain.has_outdated_files() + + stamp = setup_core / "exclude_components.esphomeinternal" + stamp.write_text("unity") + os.utime(stamp, (cmakecache.stat().st_mtime + 10,) * 2) + + assert toolchain.has_outdated_files() + + # The flag must clear once the reference file is restamped (as + # run_compile does after a successful discovery reconfigure); + # otherwise every later build would repeat the discovery pass. + os.utime(cmakecache, (stamp.stat().st_mtime + 10,) * 2) + assert not toolchain.has_outdated_files() + + def test_get_idedata_returns_none_without_compile_commands(setup_core: Path) -> None: """No compile DB yet -> None (rather than an error).""" _setup_build(setup_core) @@ -373,6 +400,48 @@ def test_run_idf_py_jobs_sets_build_jobs_env(setup_core: Path) -> None: assert "IDF_PY_BUILD_JOBS" not in env +def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> None: + """After a successful discovery reconfigure the reference CMakeCache.txt + is restamped; cmake does not rewrite it when only properties or plain + variables change, so the staleness flag would otherwise never clear.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + cmakecache.parent.mkdir(parents=True, exist_ok=True) + cmakecache.write_text("") + old = cmakecache.stat().st_mtime - 100 + os.utime(cmakecache, (old, old)) + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project"), + patch.object(toolchain, "run_reconfigure", return_value=0), + patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + assert cmakecache.stat().st_mtime > old + + +def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: + """A discovery pass that produced no CMakeCache.txt (nothing to restamp) + still completes normally.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project"), + patch.object(toolchain, "run_reconfigure", return_value=0), + patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + assert not CORE.relative_build_path("build/CMakeCache.txt").exists() + + def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: """compile_process_limit is forwarded to run_idf_py as the job limit.""" _setup_build(setup_core)