[esp32][api][noise] Resolve arduino's libsodium requirement to the converted esphome/libsodium (#20185)

esphome/components/esp32/__init__.py

@@ -57,7 +57,7 @@ from esphome.const import (
 from esphome.core import CORE, EsphomeError, HexInt
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.coroutine import CoroPriority, coroutine_with_priority
-from esphome.espidf.component import generate_idf_components
+from esphome.espidf.component import IDFComponent, generate_idf_components
 import esphome.final_validate as fv
 from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed
 from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor
@@ -3539,6 +3539,30 @@ def _write_idf_component_yml():
     yml_path = CORE.relative_build_path("src/idf_component.yml")
     dependencies: dict[str, dict] = {}
 
+    converted: list[IDFComponent] = []
+    if CORE.using_toolchain_esp_idf:
+        # Convert the PlatformIO libraries to ESP-IDF components as a batch so
+        # PlatformIO resolves the whole dependency tree at once -- deduplicating
+        # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c
+        # and esp_wireguard) to a single version instead of clashing
+        # override_path entries.
+        libraries = [
+            library
+            for name, library in CORE.platformio_libraries.items()
+            # Don't process arduino libraries
+            if name not in ARDUINO_DISABLED_LIBRARIES
+        ]
+        # A library also declared as a managed component is not converted too, or
+        # IDF sees the same requirement twice; converted components reach it through
+        # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}.
+        managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {}))
+        converted = generate_idf_components(libraries, managed=managed)
+    # IDF names a component after its directory and a later registration of the
+    # same name replaces the earlier one, so a stub beside a converted library of
+    # the same name (espressif/libsodium vs esphome/libsodium) would win or lose
+    # on path order. Such a stub points at the converted library instead.
+    converted_by_name = {component.path.name: component for component in converted}
+
     # For Arduino builds, override unused managed components from the Arduino framework
     # by pointing them to empty stub directories using override_path
     # This prevents the IDF component manager from downloading the real components
@@ -3562,8 +3586,17 @@ def _write_idf_component_yml():
         # always writes, and ninja keeps triggering CMake re-runs on
         # otherwise-cached rebuilds.
         for component_name in sorted(components_to_stub):
+            stub_name = _idf_component_stub_name(component_name)
+            stub_path = stubs_dir / stub_name
+            if (component := converted_by_name.get(stub_name)) is not None:
+                if stub_path.exists():
+                    rmtree(stub_path)
+                dependencies[_idf_component_dep_name(component_name)] = {
+                    "version": "*",
+                    "override_path": str(component.path),
+                }
+                continue
             # Create stub directory with minimal CMakeLists.txt
-            stub_path = stubs_dir / _idf_component_stub_name(component_name)
             stub_path.mkdir(exist_ok=True)
             stub_cmake = stub_path / "CMakeLists.txt"
             if not stub_cmake.exists():
@@ -3605,26 +3638,10 @@ def _write_idf_component_yml():
                 ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]),
             )
 
-    if CORE.using_toolchain_esp_idf:
-        # Convert the PlatformIO libraries to ESP-IDF components as a batch so
-        # PlatformIO resolves the whole dependency tree at once -- deduplicating
-        # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c
-        # and esp_wireguard) to a single version instead of clashing
-        # override_path entries.
-        libraries = [
-            library
-            for name, library in CORE.platformio_libraries.items()
-            # Don't process arduino libraries
-            if name not in ARDUINO_DISABLED_LIBRARIES
-        ]
-        # A library also declared as a managed component is not converted too, or
-        # IDF sees the same requirement twice; converted components reach it through
-        # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}.
-        managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {}))
-        for component in generate_idf_components(libraries, managed=managed):
-            dependencies[component.get_sanitized_name()] = {
-                "override_path": str(component.path)
-            }
+    for component in converted:
+        dependencies[component.get_sanitized_name()] = {
+            "override_path": str(component.path)
+        }
 
     if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
         components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]

esphome/espidf/clang_tidy.py

@@ -278,18 +278,25 @@ def _convert_pio_libs(
     return deps
 
 
-def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]:
+def _arduino_excluded_stubs(
+    work_dir: Path, converted: dict[str, dict[str, str]] | None = None
+) -> dict[str, dict]:
     """Stub the arduino-bundled IDF components ESPHome doesn't use.
 
     arduino-esp32 declares deps (libsodium, RainMaker, modbus, ...) that ESPHome
     replaces with its own library (noise-c) or doesn't use; point each at an
     empty override_path component so the IDF manager doesn't resolve/download
-    them -- notably so ``espressif/libsodium`` doesn't clash with the converted
-    noise-c's ``libsodium``. Mirrors esp32's ``_write_idf_component_yml``.
+    them. Mirrors esp32's ``_write_idf_component_yml``.
 
     Components ESPHome's own idf_component.yml provides (e.g. lan867x for
     ethernet) are NOT stubbed -- those are real deps we need, and arduino-esp32
     resolves to the same component rather than conflicting.
+
+    ``converted`` is the manifest block from ``_convert_pio_libs``. IDF names a
+    component after its directory and a later registration of the same name
+    replaces the earlier one, so a stub beside a converted library of the same
+    name (``espressif/libsodium`` vs ``esphome/libsodium``) would win or lose on
+    path order; such a stub points at the converted library instead.
     """
     from esphome.components.esp32 import (
         _idf_component_dep_name,
@@ -298,6 +305,10 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]:
     )
 
     esphome_deps = _esphome_manifest_deps()
+    converted_paths = {
+        Path(dep["override_path"]).name: dep["override_path"]
+        for dep in (converted or {}).values()
+    }
 
     stubs_dir = work_dir / "component_stubs"
     stubs_dir.mkdir(parents=True, exist_ok=True)
@@ -305,7 +316,14 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]:
     for component in sorted(arduino_excluded_idf_components()):
         if _idf_component_dep_name(component) in esphome_deps:
             continue  # ESPHome needs this one for real (don't stub it away)
-        stub_path = stubs_dir / _idf_component_stub_name(component)
+        stub_name = _idf_component_stub_name(component)
+        if (path := converted_paths.get(stub_name)) is not None:
+            deps[_idf_component_dep_name(component)] = {
+                "version": "*",
+                "override_path": path,
+            }
+            continue
+        stub_path = stubs_dir / stub_name
         stub_path.mkdir(exist_ok=True)
         (stub_path / "CMakeLists.txt").write_text(
             "idf_component_register()\n", encoding="utf-8"
@@ -401,11 +419,12 @@ def _generate_compile_commands(
     # Framework deps (e.g. arduino-esp32) + PlatformIO libs converted to local
     # IDF components, all added to the manifest as deps.
     extra_deps = dict(settings.framework_deps)
-    extra_deps.update(_convert_pio_libs(platformio_ini, settings.target_framework))
+    converted = _convert_pio_libs(platformio_ini, settings.target_framework)
+    extra_deps.update(converted)
     if settings.target_framework == "arduino":
         # Stub the arduino-bundled components ESPHome doesn't use (avoids the
         # libsodium clash with noise-c and ~26 unused heavy downloads).
-        extra_deps.update(_arduino_excluded_stubs(work_dir))
+        extra_deps.update(_arduino_excluded_stubs(work_dir, converted))
 
     # Phase 1: discover the components available for this target.
     _write_tidy_project(work_dir, [], extra_deps, settings)

tests/components/api/test.esp32-ard.yaml

@@ -0,0 +1,6 @@
+packages:
+  api: !include common.yaml
+
+wifi:
+  ssid: MySSID
+  password: password1

tests/components/noise/test.esp32-ard.yaml

@@ -0,0 +1,2 @@
+packages:
+  noise: !include common.yaml

tests/unit_tests/components/test_esp32_write_idf_component_yml.py

@@ -10,11 +10,13 @@ from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
+import yaml
 
 from esphome.components import esp32
 import esphome.config_validation as cv
 from esphome.const import (
     KEY_CORE,
+    KEY_FRAMEWORK_VERSION,
     KEY_TARGET_FRAMEWORK,
     KEY_TARGET_PLATFORM,
     Framework,
@@ -173,3 +175,47 @@ def test_write_idf_component_yml_arduino_stubs_follow_idf_version(
         / esp32._idf_component_stub_name("espressif__libsodium")
     )
     assert stub_dir.is_dir() is libsodium_stubbed
+
+
+def test_write_idf_component_yml_stub_yields_to_converted_library_of_same_name(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """IDF names a component after its directory and the later registration wins,
+    so the espressif/libsodium stub and the converted esphome/libsodium raced on
+    path order (#20102). The stub entry points at the converted library instead."""
+    _setup_core(tmp_path)
+    CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(Framework.ARDUINO)
+    CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version(3, 3, 11)
+    CORE.data[esp32.KEY_ESP32] = {
+        esp32.KEY_COMPONENTS: {},
+        esp32.KEY_IDF_VERSION: cv.Version(5, 5, 5),
+        esp32.KEY_ARDUINO_LIBRARIES: set(),
+    }
+    # A stale stub from an earlier build must not linger beside the real library
+    stale_stub = tmp_path / "component_stubs" / "libsodium"
+    stale_stub.mkdir(parents=True)
+
+    converted = MagicMock()
+    converted.get_sanitized_name.return_value = "esphome/libsodium"
+    converted.path = tmp_path / "pio_components" / "esphome" / "libsodium"
+    monkeypatch.setattr(
+        esp32, "generate_idf_components", lambda libraries, managed=None: [converted]
+    )
+
+    esp32._write_idf_component_yml()
+
+    manifest = yaml.safe_load(
+        (tmp_path / "src" / "idf_component.yml").read_text(encoding="utf-8")
+    )["dependencies"]
+    assert manifest["espressif/libsodium"] == {
+        "version": "*",
+        "override_path": str(converted.path),
+    }
+    assert manifest["esphome/libsodium"] == {"override_path": str(converted.path)}
+    assert not stale_stub.exists()
+    # Stubs without a converted namesake are unaffected
+    assert (tmp_path / "component_stubs" / "cbor").is_dir()
+    assert manifest["espressif/cbor"]["override_path"] == str(
+        tmp_path / "component_stubs" / "cbor"
+    )

tests/unit_tests/test_espidf_clang_tidy.py

@@ -153,6 +153,33 @@ def test_generate_compile_commands_configures_twice(
     assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"]
 
 
+def test_generate_compile_commands_hands_converted_libs_to_arduino_stubs(
+    tmp_path: Path,
+) -> None:
+    """On Arduino the stub generator sees the converted libraries, so a stub
+    sharing a converted library's directory name can point at it instead."""
+    converted = {"esphome/libsodium": {"override_path": str(tmp_path / "libsodium")}}
+    stubs = {"espressif/libsodium": {"version": "*", "override_path": "x"}}
+    with (
+        patch.object(clang_tidy, "_setup_core"),
+        patch.object(clang_tidy, "_convert_pio_libs", return_value=converted),
+        patch.object(
+            clang_tidy, "_arduino_excluded_stubs", return_value=stubs
+        ) as mock_stubs,
+        patch.object(clang_tidy, "_write_tidy_project") as mock_write,
+        patch("esphome.espidf.toolchain.run_reconfigure", return_value=0),
+        patch("esphome.build_gen.espidf.get_available_components", return_value=[]),
+    ):
+        clang_tidy._generate_compile_commands(
+            tmp_path, _settings(target_framework="arduino"), tmp_path / "platformio.ini"
+        )
+
+    mock_stubs.assert_called_once_with(tmp_path, converted)
+    extra_deps = mock_write.call_args_list[0].args[2]
+    assert extra_deps["esphome/libsodium"] == converted["esphome/libsodium"]
+    assert extra_deps["espressif/libsodium"] == stubs["espressif/libsodium"]
+
+
 def test_esphome_manifest_deps_reads_repo_manifest() -> None:
     """Top-level dependency names, independent of the per-dependency rules."""
     manifest = yaml.safe_load(
@@ -284,6 +311,27 @@ def test_arduino_excluded_stubs_skips_components_esphome_manifest_provides(
     assert (stub_path / "CMakeLists.txt").is_file()
 
 
+def test_arduino_excluded_stubs_points_libsodium_at_converted_library(
+    tmp_path: Path,
+) -> None:
+    """Below IDF 6.0 the converted esphome/libsodium shares its directory name
+    with the espressif/libsodium stub; IDF would keep whichever registers last,
+    so the stub entry points at the converted library instead (#20102)."""
+    _set_idf_version(cv.Version(5, 5, 4))
+    converted_path = str(tmp_path / "pio" / "esphome" / "libsodium")
+    converted = {"esphome/libsodium": {"override_path": converted_path}}
+
+    deps = _arduino_excluded_stubs(tmp_path, converted)
+
+    assert deps["espressif/libsodium"] == {
+        "version": "*",
+        "override_path": converted_path,
+    }
+    assert not (tmp_path / "component_stubs" / "libsodium").exists()
+    # Stubs without a converted namesake are unaffected.
+    assert (tmp_path / "component_stubs" / "cbor" / "CMakeLists.txt").is_file()
+
+
 def test_arduino_excluded_stubs_skips_libsodium_from_idf_6(tmp_path: Path) -> None:
     """From IDF 6.0 arduino-esp32 drops espressif/libsodium; a stub would clash
     with esphome/libsodium."""