[espidf] Regenerate bundled CMakeLists; auto-REQUIRE via IDF build properties (#16406)

esphome/build_gen/espidf.py

@@ -10,11 +10,14 @@ from esphome.writer import update_storage_json
 
 
 def get_available_components() -> list[str] | None:
-    """Get list of available ESP-IDF components from project_description.json.
+    """Get list of built-in ESP-IDF components from project_description.json.
 
-    Returns only internal ESP-IDF components, excluding external/managed
-    components (from idf_component.yml).
+    Excludes ``src``, IDF-managed components (``managed_components/``), and
+    converted PIO libs (``pio_components/``). Returns ``None`` if the build
+    dir or ``project_description.json`` isn't ready yet.
     """
+    if CORE.build_path is None:
+        return None
     project_desc = Path(CORE.build_path) / "build" / "project_description.json"
     if not project_desc.exists():
         return None
@@ -31,9 +34,9 @@ def get_available_components() -> list[str] | None:
             if name == "src":
                 continue
 
-            # Exclude managed/external components
+            # Exclude IDF-managed and converted-PIO components (external).
             comp_dir = info.get("dir", "")
-            if "managed_components" in comp_dir:
+            if "managed_components" in comp_dir or "pio_components" in comp_dir:
                 continue
 
             result.append(name)
@@ -48,8 +51,12 @@ def has_discovered_components() -> bool:
     return get_available_components() is not None
 
 
-def get_project_cmakelists() -> str:
-    """Generate the top-level CMakeLists.txt for ESP-IDF project."""
+def get_project_cmakelists(minimal: bool = False) -> str:
+    """Generate the top-level CMakeLists.txt for ESP-IDF project.
+
+    When ``minimal`` is true, omit ``ESPHOME_PROJECT_BUILTIN_COMPONENTS``
+    since ``project_description.json`` may be stale on the first write.
+    """
     # Get IDF target from ESP32 variant (e.g., ESP32S3 -> esp32s3)
     variant = get_esp32_variant()
     idf_target = variant.lower().replace("-", "")
@@ -72,6 +79,37 @@ def get_project_cmakelists() -> str:
         for flag in project_compile_opts
     )
 
+    # Per-project list exposed as a CMake variable so converted PIO libs
+    # can reference ${ESPHOME_PROJECT_MANAGED_COMPONENTS} without baking
+    # project-specific names into their cached CMakeLists.
+    #
+    # Emit via idf_build_set_property (not plain set()) so the value is
+    # serialised into build_properties.temp.cmake and visible to IDF's
+    # early requirements-expansion pass (component_get_requirements.cmake
+    # runs as a separate CMake script invocation that doesn't load the
+    # project's top-level CMakeLists; without this, ${ESPHOME_PROJECT_
+    # MANAGED_COMPONENTS} in a converted-lib REQUIRES expands to empty).
+    from esphome.components.esp32 import get_managed_component_require_names
+
+    managed_components_property = "\n".join(
+        f"idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS {name} APPEND)"
+        for name in get_managed_component_require_names()
+    )
+
+    # Built-in IDF components exposed via our own property (not IDF's
+    # __COMPONENT_REQUIRES_COMMON, which would append them to every
+    # component's REQUIRES including real IDF components). Referenced by
+    # src/CMakeLists and by each converted PIO lib's CMakeLists. Skipped
+    # on minimal writes because project_description.json may be stale.
+    builtin_components_property = (
+        ""
+        if minimal
+        else "\n".join(
+            f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)"
+            for name in sorted(get_available_components() or [])
+        )
+    )
+
     return f"""\
 # Auto-generated by ESPHome
 cmake_minimum_required(VERSION 3.16)
@@ -99,6 +137,10 @@ include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
 {extra_compile_options}
 
+{managed_components_property}
+
+{builtin_components_property}
+
 project({CORE.name})
 
 # Emit raw JSON size data for ESPHome to read post-build.
@@ -113,11 +155,12 @@ add_custom_command(
 """
 
 
-def get_component_cmakelists(minimal: bool = False) -> str:
-    """Generate the main component CMakeLists.txt."""
-    idf_requires = [] if minimal else (get_available_components() or [])
-    requires_str = " ".join(idf_requires)
+def get_component_cmakelists() -> str:
+    """Generate the main component CMakeLists.txt.
 
+    REQUIRES pulls in the discovered built-in IDF components via the
+    project-level variables set in the top-level CMakeLists.
+    """
     # Extract linker options (-Wl, flags). Compile flags (-D, -W) are
     # emitted project-wide via idf_build_set_property in
     # get_project_cmakelists so they reach every component, not just src/.
@@ -126,17 +169,30 @@ def get_component_cmakelists(minimal: bool = False) -> str:
 
     return f"""\
 # Auto-generated by ESPHome
-file(GLOB_RECURSE app_sources
-    "${{CMAKE_CURRENT_SOURCE_DIR}}/*.cpp"
-    "${{CMAKE_CURRENT_SOURCE_DIR}}/*.c"
-    "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.cpp"
-    "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.c"
-)
+# CONFIGURE_DEPENDS asks CMake to re-check the glob each build so test
+# runs that reuse the build dir don't compile stale source paths. It's
+# invalid in script mode (cmake -P), which is how IDF's
+# component_get_requirements.cmake includes us, so skip it there.
+if(CMAKE_SCRIPT_MODE_FILE)
+    file(GLOB_RECURSE app_sources
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/*.cpp"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/*.c"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.cpp"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.c"
+    )
+else()
+    file(GLOB_RECURSE app_sources CONFIGURE_DEPENDS
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/*.cpp"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/*.c"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.cpp"
+        "${{CMAKE_CURRENT_SOURCE_DIR}}/esphome/*.c"
+    )
+endif()
 
 idf_component_register(
     SRCS ${{app_sources}}
     INCLUDE_DIRS "." "esphome"
-    REQUIRES {requires_str}
+    REQUIRES ${{ESPHOME_PROJECT_BUILTIN_COMPONENTS}}
 )
 
 # Apply C++ standard
@@ -162,11 +218,11 @@ def write_project(minimal: bool = False) -> None:
     # Write top-level CMakeLists.txt
     write_file_if_changed(
         CORE.relative_build_path("CMakeLists.txt"),
-        get_project_cmakelists(),
+        get_project_cmakelists(minimal=minimal),
     )
 
     # Write component CMakeLists.txt in src/
     write_file_if_changed(
         CORE.relative_src_path("CMakeLists.txt"),
-        get_component_cmakelists(minimal=minimal),
+        get_component_cmakelists(),
     )

esphome/components/esp32/__init__.py

@@ -588,6 +588,18 @@ def add_idf_component(
     }
 
 
+def get_managed_component_require_names() -> list[str]:
+    """Return sorted IDF require names for components added via
+    ``add_idf_component`` (``owner/name`` -> ``owner__name``).
+
+    The build_gen layer (``build_gen.espidf.get_project_cmakelists``)
+    feeds this list into ``ESPHOME_PROJECT_MANAGED_COMPONENTS`` so
+    converted PIO libraries can REQUIRE them by name at configure time.
+    """
+    components_registry = CORE.data.get(KEY_ESP32, {}).get(KEY_COMPONENTS, {})
+    return sorted(name.replace("/", "__") for name in components_registry)
+
+
 def exclude_builtin_idf_component(name: str) -> None:
     """Exclude an ESP-IDF component from the build.
 

esphome/espidf/component.py

@@ -12,7 +12,6 @@ from typing import TypeVar
 from urllib.parse import urlparse, urlsplit, urlunsplit
 
 from esphome import git, yaml_util
-from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, Library
 from esphome.espidf.framework import archive_extract_all, download_from_mirrors, rmdir
 from esphome.helpers import write_file_if_changed
@@ -50,28 +49,6 @@ SRC_FILE_EXTENSIONS = [
 ESP32_PLATFORM = "espressif32"
 DOMAIN = "pio_components"
 
-#
-# Constants for workarounds
-#
-
-REQUIRES_DETECT_PATTERNS = {
-    "mbedtls": [re.compile(r'^\s*#\s*include\s*[<"]mbedtls[^">]*[">]', re.MULTILINE)],
-    "esp_netif": [
-        re.compile(r'^\s*#\s*include\s*[<"]esp_netif[^">]*[">]', re.MULTILINE)
-    ],
-    "esp_driver_gpio": [
-        re.compile(r'^\s*#\s*include\s*[<"]driver/gpio\.h[^">]*[">]', re.MULTILINE)
-    ],
-    "esp_timer": [
-        re.compile(r'^\s*#\s*include\s*[<"]esp_timer\.h[^">]*[">]', re.MULTILINE)
-    ],
-    "esp_wifi": [
-        re.compile(
-            r'^\s*#\s*include\s*[<"]WiFi\.h[^">]*[">]', re.MULTILINE
-        )  # Arduino WiFi
-    ],
-}
-
 ESPHOME_DATA_KEY = "ESPHOME"
 ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE"
 
@@ -86,10 +63,7 @@ class URLSource(Source):
         self.url = url
 
     def download(self, dir_suffix: str, force: bool = False) -> Path:
-        # Partition by framework: generated idf_component.yml content
-        # depends on CORE.using_arduino, so caches can't be shared.
-        framework = "arduino" if CORE.using_arduino else "idf"
-        base_dir = Path(CORE.data_dir) / DOMAIN / framework
+        base_dir = Path(CORE.data_dir) / DOMAIN
         h = hashlib.new("sha256")
         h.update(self.url.encode())
         path = base_dir / h.hexdigest()[:8] / dir_suffix
@@ -124,12 +98,11 @@ class GitSource(Source):
         self.ref = ref
 
     def download(self, dir_suffix: str, force: bool = False) -> Path:
-        framework = "arduino" if CORE.using_arduino else "idf"
         path, _ = git.clone_or_update(
             url=self.url,
             ref=self.ref,
             refresh=git.NEVER_REFRESH if not force else None,
-            domain=f"{DOMAIN}/{framework}",
+            domain=DOMAIN,
             submodules=[],
             subpath=Path(dir_suffix),
         )
@@ -282,46 +255,6 @@ def _get_package_from_pio_registry(
     return owner, name, version["name"], pkgfile["download_url"]
 
 
-def _patch_component(component: IDFComponent, first_pass: bool):
-    """
-    Apply patches/workarounds to specific components that have known issues.
-
-    This function modifies component data to fix compatibility issues or missing
-    dependencies for certain libraries. It applies different patches based on
-    whether it's the first or second pass of processing.
-
-    Args:
-        component: The IDFComponent object to potentially patch
-        first_pass: Boolean indicating if this is the first pass of processing
-    """
-
-    # Patch only on the second step
-    if not first_pass and CORE.using_arduino:
-        # Add the missing dependency to Arduino framework. Source is None so
-        # the IDF component manager resolves it from the registry instead of
-        # cloning the 2 GB arduino-esp32 git history.
-        component.dependencies.append(
-            IDFComponent(
-                "espressif/arduino-esp32",
-                str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]),
-                None,
-            )
-        )
-
-    #
-    # fastled/FastLED
-    #
-
-    # Patch only on the first step
-    if (
-        first_pass
-        and component.name == _owner_pkgname_to_name("fastled", "FastLED")
-        and not (component.path / "idf_component.yml").is_file()
-    ):
-        # Force fake idf_component: This project already support ESP-IDF
-        (component.path / "idf_component.yml").write_text("")
-
-
 def _apply_extra_script(component: IDFComponent) -> None:
     """Run a PIO ``extraScript`` and fold its captured env vars into
     ``component.data["build"]["flags"]`` so the existing -L/-l/-D
@@ -506,43 +439,6 @@ def _convert_library_to_component(library: Library) -> IDFComponent:
     return IDFComponent(name, version, source)
 
 
-def _detect_requires(build_src_files: list[str]) -> set[str]:
-    """
-    Detect required components from source files.
-
-    Args:
-        build_src_files: List of source file paths to analyze
-
-    Returns:
-        Set of detected required components
-    """
-    detected = set()
-
-    # 1. Process each source file
-    for file in build_src_files:
-        path = Path(file)
-
-        if not path.is_file():
-            continue
-
-        try:
-            content = path.read_text(encoding="utf-8", errors="ignore")
-        except Exception:  # pylint: disable=broad-exception-caught
-            continue
-
-        # 2. Add required component if one of these patterns matches
-        for require_name, patterns in REQUIRES_DETECT_PATTERNS.items():
-            if require_name in detected:
-                continue  # already found
-
-            for pattern in patterns:
-                if pattern.search(content):
-                    detected.add(require_name)
-                    break
-
-    return detected
-
-
 def _split_list_by_condition(
     items: list[str], match_fn: Callable[[str], str | None]
 ) -> tuple[list[str], list[str]]:
@@ -609,13 +505,14 @@ def generate_cmakelists_txt(component: IDFComponent) -> str:
         component.path / Path(build_src_dir), build_src_filter
     )
 
-    # Detect in the files which requirements to add
-    # By default in platformio, all the components are added: we need to detect them when using ESP-IDF
-    requires = _detect_requires(build_src_files)
-
-    # Dependencies are required
-    for dependency in component.dependencies:
-        requires.add(dependency.get_require_name())
+    # Only bake library.json-declared deps here. Project-managed and
+    # built-in components come in via ${ESPHOME_PROJECT_MANAGED_COMPONENTS}
+    # / ${ESPHOME_PROJECT_BUILTIN_COMPONENTS} set in the top-level
+    # CMakeLists, so this file stays project-agnostic when shared from
+    # the pio_components cache.
+    requires: set[str] = {
+        dependency.get_require_name() for dependency in component.dependencies
+    }
 
     # Only keep sources
     build_src_files = [os.path.relpath(p, component.path) for p in build_src_files]
@@ -654,9 +551,19 @@ def generate_cmakelists_txt(component: IDFComponent) -> str:
     if build_include_dirs:
         str_include_dirs = " ".join([escape_entry(p) for p in build_include_dirs])
         content += f"  INCLUDE_DIRS {str_include_dirs}\n"
-    if requires:
-        str_requires = " ".join(sorted(requires))
-        content += f"  REQUIRES {str_requires}\n"
+    # Project-managed and built-in component lists are set per-project
+    # via idf_build_set_property in the top-level CMakeLists; expanded
+    # here at configure time. Keeping them out of the per-lib REQUIRES
+    # means this CMakeLists is project-agnostic and reusable from the
+    # pio_components cache across builds.
+    str_requires = " ".join(
+        [
+            *sorted(requires),
+            "${ESPHOME_PROJECT_MANAGED_COMPONENTS}",
+            "${ESPHOME_PROJECT_BUILTIN_COMPONENTS}",
+        ]
+    )
+    content += f"  REQUIRES {str_requires}\n"
     content += ")\n"
 
     # Add public and private build flags
@@ -732,13 +639,10 @@ def generate_idf_component_yml(component: IDFComponent) -> str:
         try:
             dep["override_path"] = str(dependency.path)
         except RuntimeError as e:
-            # No local path; let the IDF component manager resolve.
-            # GitSource gives an explicit URL; arduino-esp32 is resolved by
-            # version from the registry. Anything else is a bug.
-            if isinstance(dependency.source, GitSource):
-                dep["git"] = dependency.source.url
-            elif dependency.name != "espressif/arduino-esp32":
+            # No local path: only a GitSource can substitute its URL.
+            if not isinstance(dependency.source, GitSource):
                 raise e
+            dep["git"] = dependency.source.url
 
         data["dependencies"][dependency.get_sanitized_name()] = dep
 
@@ -903,12 +807,9 @@ def _generate_idf_component(library: Library, force: bool = False) -> IDFCompone
     cmakelists_txt_path = component.path / "CMakeLists.txt"
     idf_component_yml_path = component.path / "idf_component.yml"
 
-    # Apply patches to the library metadata
-    _patch_component(component, True)
-
-    if cmakelists_txt_path.is_file() and idf_component_yml_path.is_file():
-        # Already an ESP-IDF component
-        return component
+    # Bundled CMakeLists.txt / idf_component.yml are ignored -- library
+    # authors' IDF support is frequently broken (bogus REQUIRES, hard-coded
+    # arduino-esp32, etc.). We always regenerate.
 
     if library_json_path.is_file():
         component.data = _parse_library_json(library_json_path)
@@ -919,9 +820,6 @@ def _generate_idf_component(library: Library, force: bool = False) -> IDFCompone
             "Invalid PIO library: missing library.json and/or library.properties"
         )
 
-    # Apply additional patches to the library metadata
-    _patch_component(component, False)
-
     # Check if the component is usable with ESP-IDF before executing any
     # third-party Python from the library (``_apply_extra_script`` below).
     _check_library_data(component.data)
@@ -936,7 +834,6 @@ def _generate_idf_component(library: Library, force: bool = False) -> IDFCompone
     # Handle the dependencies (convert PlatformIO library to ESP-IDF component if needed)
     _process_dependencies(component)
 
-    # Generate files
     _LOGGER.debug("Generating CMakeLists.txt for %s@%s  ...", name, version)
     write_file_if_changed(
         cmakelists_txt_path,

esphome/espidf/toolchain.py

@@ -302,10 +302,21 @@ def run_compile(config, verbose: bool) -> int:
             return rc
         _LOGGER.info("Regenerating CMakeLists.txt with discovered components...")
         write_project(minimal=False)
+        # The post-discovery rewrite leaves CMakeLists newer than
+        # CMakeCache.txt. CMake won't re-touch CMakeCache.txt on a
+        # configure that only changes idf_build_set_property values
+        # (those aren't cache variables), so has_outdated_files() would
+        # return True on every subsequent build, perpetually retriggering
+        # the two-pass. Touch CMakeCache.txt now so its mtime stays past
+        # the rewritten CMakeLists.
+        cmakecache = CORE.relative_build_path("build/CMakeCache.txt")
+        if cmakecache.is_file():
+            os.utime(cmakecache)
         if CORE.testing_mode:
-            # Reconfigure again so cmake is up to date with the full component
-            # list. This ensures idf.py build won't re-run cmake, which would
-            # regenerate memory.ld and wipe the DRAM/IRAM patches applied below.
+            # Reconfigure again so cmake is up to date with the full
+            # component list before the build's idf.py invocation runs --
+            # idf.py build would otherwise re-run cmake and regenerate
+            # memory.ld, wiping the DRAM/IRAM patches applied below.
             rc = run_reconfigure()
             if rc != 0:
                 _LOGGER.error("Reconfigure with discovered components failed")

tests/unit_tests/build_gen/test_espidf.py

@@ -0,0 +1,159 @@
+"""Tests for esphome.build_gen.espidf module."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from esphome.components.esp32 import (
+    KEY_COMPONENTS,
+    KEY_ESP32,
+    KEY_PATH,
+    KEY_REF,
+    KEY_REPO,
+)
+from esphome.const import KEY_CORE
+from esphome.core import CORE
+
+
+@pytest.fixture(autouse=True)
+def _reset_core(tmp_path: Path) -> None:
+    """Give each test its own CORE.build_path and a clean esp32 data slot."""
+    CORE.build_path = str(tmp_path)
+    CORE.data.setdefault(KEY_CORE, {})
+    CORE.data[KEY_ESP32] = {KEY_COMPONENTS: {}}
+
+
+def _write_project_description(tmp_path: Path, components: dict[str, str]) -> None:
+    """Stub a project_description.json with the given component_name -> dir map."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir(exist_ok=True)
+    (build_dir / "project_description.json").write_text(
+        json.dumps(
+            {
+                "build_component_info": {
+                    name: {"dir": dir_} for name, dir_ in components.items()
+                }
+            }
+        )
+    )
+
+
+def test_get_available_components_returns_none_without_build_path() -> None:
+    """No build_path set yet: must not raise on Path(None)."""
+    CORE.build_path = None
+    from esphome.build_gen.espidf import get_available_components
+
+    assert get_available_components() is None
+
+
+def test_get_available_components_returns_none_without_project_description(
+    tmp_path: Path,
+) -> None:
+    from esphome.build_gen.espidf import get_available_components
+
+    assert get_available_components() is None
+
+
+def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> None:
+    """Built-ins are returned; src/, managed_components/, pio_components/ skipped."""
+    _write_project_description(
+        tmp_path,
+        {
+            "src": f"{tmp_path}/src",
+            "esp_lcd": "/idf/components/esp_lcd",
+            "espressif__arduino-esp32": f"{tmp_path}/managed_components/arduino",
+            "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC",
+            "freertos": "/idf/components/freertos",
+        },
+    )
+    from esphome.build_gen.espidf import get_available_components
+
+    assert sorted(get_available_components()) == ["esp_lcd", "freertos"]
+
+
+def test_get_project_cmakelists_minimal_omits_builtin_components_property(
+    tmp_path: Path,
+) -> None:
+    """Minimal write must not emit ESPHOME_PROJECT_BUILTIN_COMPONENTS even
+    when project_description.json exists (the data may be stale on the
+    first write before the discovery pass refreshes it)."""
+    _write_project_description(tmp_path, {"esp_lcd": "/idf/components/esp_lcd"})
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        content = get_project_cmakelists(minimal=True)
+
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS" not in content
+
+
+def test_get_project_cmakelists_full_emits_builtin_components_property(
+    tmp_path: Path,
+) -> None:
+    """Non-minimal write emits one idf_build_set_property line per built-in,
+    sorted, and excludes src/managed/pio components."""
+    _write_project_description(
+        tmp_path,
+        {
+            "src": f"{tmp_path}/src",
+            "esp_lcd": "/idf/components/esp_lcd",
+            "freertos": "/idf/components/freertos",
+            "espressif__esp-dsp": f"{tmp_path}/managed_components/esp-dsp",
+            "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC",
+        },
+    )
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        content = get_project_cmakelists(minimal=False)
+
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_lcd APPEND)"
+        in content
+    )
+    assert (
+        "idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS freertos APPEND)"
+        in content
+    )
+    # Excluded by get_available_components filtering.
+    assert "espressif__esp-dsp APPEND" not in content
+    assert "JPEGDEC APPEND" not in content
+
+
+def test_get_project_cmakelists_emits_managed_components_property(
+    tmp_path: Path,
+) -> None:
+    """ESPHOME_PROJECT_MANAGED_COMPONENTS is always emitted (both modes)
+    from the esp32 add_idf_component registry."""
+    CORE.data[KEY_ESP32][KEY_COMPONENTS] = {
+        "espressif/esp-dsp": {KEY_REPO: None, KEY_REF: "1.7.1", KEY_PATH: None},
+        "espressif/arduino-esp32": {KEY_REPO: None, KEY_REF: "3.3.8", KEY_PATH: None},
+    }
+
+    with (
+        patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
+        patch.object(CORE, "name", "test"),
+    ):
+        from esphome.build_gen.espidf import get_project_cmakelists
+
+        for minimal in (True, False):
+            content = get_project_cmakelists(minimal=minimal)
+            assert (
+                "idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS"
+                " espressif__arduino-esp32 APPEND)"
+            ) in content
+            assert (
+                "idf_build_set_property(ESPHOME_PROJECT_MANAGED_COMPONENTS"
+                " espressif__esp-dsp APPEND)"
+            ) in content

tests/unit_tests/test_espidf_component.py

@@ -1,5 +1,6 @@
 import json
 import os
+from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
@@ -21,7 +22,6 @@ from esphome.espidf.component import (
     _check_library_data,
     _collect_filtered_files,
     _convert_library_to_component,
-    _detect_requires,
     _parse_library_json,
     _parse_library_properties,
     _process_dependencies,
@@ -83,19 +83,6 @@ def test_collect_filtered_files_exclude(tmp_path):
     assert str(f2) not in result
 
 
-def test_detect_requires(tmp_path):
-    f = tmp_path / "main.c"
-    f.write_text('#include "mbedtls/foo.h"')
-
-    result = _detect_requires([str(f)])
-    assert "mbedtls" in result
-
-
-def test_detect_requires_ignores_invalid_file(tmp_path):
-    result = _detect_requires([str(tmp_path / "missing.c")])
-    assert result == set()
-
-
 def test_split_list_by_condition():
     items = ["-Iinclude", "-Llib", "-Wall"]
 
@@ -142,7 +129,7 @@ def test_generate_cmakelists_txt_with_flags(tmp_component, tmp_path):
         == f"""idf_component_register(
   SRCS "src{sep}main.c"
   INCLUDE_DIRS "src"
-  REQUIRES dep
+  REQUIRES dep ${{ESPHOME_PROJECT_MANAGED_COMPONENTS}} ${{ESPHOME_PROJECT_BUILTIN_COMPONENTS}}
 )
 target_compile_options(${{COMPONENT_LIB}} PUBLIC
   "-DTEST"
@@ -160,6 +147,58 @@ target_link_libraries(${{COMPONENT_LIB}} INTERFACE
     )
 
 
+def test_generate_cmakelists_txt_references_project_managed_components_variable(
+    tmp_component: IDFComponent,
+) -> None:
+    # The CMakeLists is cached under pio_components/<hash>/ and shared
+    # across projects, so the project-managed REQUIRES list is exposed via
+    # a CMake variable expanded at configure time rather than baked here.
+    src_dir = tmp_component.path / "src"
+    src_dir.mkdir()
+    (src_dir / "main.c").write_text("int main() {}")
+    tmp_component.data = {}
+
+    content = generate_cmakelists_txt(tmp_component)
+    assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content
+
+
+def test_generate_idf_component_overwrites_bundled_files(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    esp32_idf_core: None,
+) -> None:
+    # A library that ships its own CMakeLists.txt + idf_component.yml must
+    # have both replaced by ESPHome's generated content. Library authors'
+    # bundled IDF metadata is frequently broken (bogus REQUIRES, hard-coded
+    # frameworks), so we always regenerate from library.json.
+    from esphome.espidf.component import _generate_idf_component
+
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "main.cpp").write_text("// dummy\n")
+    (tmp_path / "library.json").write_text(json.dumps({"name": "tripwire-lib"}))
+    (tmp_path / "CMakeLists.txt").write_text("# TRIPWIRE_BUNDLED_CMAKELISTS\n")
+    (tmp_path / "idf_component.yml").write_text("# TRIPWIRE_BUNDLED_MANIFEST\n")
+
+    fake_component = IDFComponent(
+        "owner/tripwire-lib", "1.0.0", source=URLSource("http://dummy")
+    )
+    fake_component.path = tmp_path
+    monkeypatch.setattr(
+        esphome.espidf.component,
+        "_convert_library_to_component",
+        lambda _lib: fake_component,
+    )
+    monkeypatch.setattr(fake_component, "download", lambda force=False: None)
+
+    _generate_idf_component(Library("owner/tripwire-lib", "1.0.0", None))
+
+    cml = (tmp_path / "CMakeLists.txt").read_text()
+    manifest = (tmp_path / "idf_component.yml").read_text()
+    assert "TRIPWIRE_BUNDLED_CMAKELISTS" not in cml
+    assert "TRIPWIRE_BUNDLED_MANIFEST" not in manifest
+    assert "idf_component_register" in cml
+
+
 def test_generate_idf_component_yml_basic(tmp_component):
     tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
     result = generate_idf_component_yml(tmp_component)
@@ -187,27 +226,6 @@ dependencies:
     )
 
 
-def test_generate_idf_component_yml_arduino_registry_dep(tmp_component):
-    # Synthetic arduino-esp32 dep with no source / no path: should emit a
-    # version-only entry so the IDF component manager resolves it from the
-    # registry instead of via git.
-    dep = IDFComponent("espressif/arduino-esp32", "3.3.8", source=None)
-
-    tmp_component.dependencies = [dep]
-    tmp_component.data = {}
-
-    result = generate_idf_component_yml(tmp_component)
-
-    assert (
-        result
-        == """version: 1.0.0
-dependencies:
-  espressif/arduino-esp32:
-    version: 3.3.8
-"""
-    )
-
-
 def test_generate_idf_component_yml_missing_path_reraises(tmp_component):
     # A dep without a path and without a recognised source should re-raise
     # the underlying RuntimeError instead of silently producing a bad manifest.