diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 1166afe4ec..be6b2c3266 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -57,7 +57,7 @@ from esphome.const import ( from esphome.core import CORE, EsphomeError, HexInt from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority -from esphome.espidf.component import generate_idf_components +from esphome.espidf.component import IDFComponent, generate_idf_components import esphome.final_validate as fv from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -3539,6 +3539,30 @@ def _write_idf_component_yml(): yml_path = CORE.relative_build_path("src/idf_component.yml") dependencies: dict[str, dict] = {} + converted: list[IDFComponent] = [] + if CORE.using_toolchain_esp_idf: + # Convert the PlatformIO libraries to ESP-IDF components as a batch so + # PlatformIO resolves the whole dependency tree at once -- deduplicating + # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c + # and esp_wireguard) to a single version instead of clashing + # override_path entries. + libraries = [ + library + for name, library in CORE.platformio_libraries.items() + # Don't process arduino libraries + if name not in ARDUINO_DISABLED_LIBRARIES + ] + # A library also declared as a managed component is not converted too, or + # IDF sees the same requirement twice; converted components reach it through + # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. + managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) + converted = generate_idf_components(libraries, managed=managed) + # IDF names a component after its directory and a later registration of the + # same name replaces the earlier one, so a stub beside a converted library of + # the same name (espressif/libsodium vs esphome/libsodium) would win or lose + # on path order. Such a stub points at the converted library instead. + converted_by_name = {component.path.name: component for component in converted} + # For Arduino builds, override unused managed components from the Arduino framework # by pointing them to empty stub directories using override_path # This prevents the IDF component manager from downloading the real components @@ -3562,8 +3586,17 @@ def _write_idf_component_yml(): # always writes, and ninja keeps triggering CMake re-runs on # otherwise-cached rebuilds. for component_name in sorted(components_to_stub): + stub_name = _idf_component_stub_name(component_name) + stub_path = stubs_dir / stub_name + if (component := converted_by_name.get(stub_name)) is not None: + if stub_path.exists(): + rmtree(stub_path) + dependencies[_idf_component_dep_name(component_name)] = { + "version": "*", + "override_path": str(component.path), + } + continue # Create stub directory with minimal CMakeLists.txt - stub_path = stubs_dir / _idf_component_stub_name(component_name) stub_path.mkdir(exist_ok=True) stub_cmake = stub_path / "CMakeLists.txt" if not stub_cmake.exists(): @@ -3605,26 +3638,10 @@ def _write_idf_component_yml(): ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), ) - if CORE.using_toolchain_esp_idf: - # Convert the PlatformIO libraries to ESP-IDF components as a batch so - # PlatformIO resolves the whole dependency tree at once -- deduplicating - # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c - # and esp_wireguard) to a single version instead of clashing - # override_path entries. - libraries = [ - library - for name, library in CORE.platformio_libraries.items() - # Don't process arduino libraries - if name not in ARDUINO_DISABLED_LIBRARIES - ] - # A library also declared as a managed component is not converted too, or - # IDF sees the same requirement twice; converted components reach it through - # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. - managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) - for component in generate_idf_components(libraries, managed=managed): - dependencies[component.get_sanitized_name()] = { - "override_path": str(component.path) - } + for component in converted: + dependencies[component.get_sanitized_name()] = { + "override_path": str(component.path) + } if CORE.data[KEY_ESP32][KEY_COMPONENTS]: components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS] diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 623b352e97..e109363d21 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -278,18 +278,25 @@ def _convert_pio_libs( return deps -def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: +def _arduino_excluded_stubs( + work_dir: Path, converted: dict[str, dict[str, str]] | None = None +) -> dict[str, dict]: """Stub the arduino-bundled IDF components ESPHome doesn't use. arduino-esp32 declares deps (libsodium, RainMaker, modbus, ...) that ESPHome replaces with its own library (noise-c) or doesn't use; point each at an empty override_path component so the IDF manager doesn't resolve/download - them -- notably so ``espressif/libsodium`` doesn't clash with the converted - noise-c's ``libsodium``. Mirrors esp32's ``_write_idf_component_yml``. + them. Mirrors esp32's ``_write_idf_component_yml``. Components ESPHome's own idf_component.yml provides (e.g. lan867x for ethernet) are NOT stubbed -- those are real deps we need, and arduino-esp32 resolves to the same component rather than conflicting. + + ``converted`` is the manifest block from ``_convert_pio_libs``. IDF names a + component after its directory and a later registration of the same name + replaces the earlier one, so a stub beside a converted library of the same + name (``espressif/libsodium`` vs ``esphome/libsodium``) would win or lose on + path order; such a stub points at the converted library instead. """ from esphome.components.esp32 import ( _idf_component_dep_name, @@ -298,6 +305,10 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: ) esphome_deps = _esphome_manifest_deps() + converted_paths = { + Path(dep["override_path"]).name: dep["override_path"] + for dep in (converted or {}).values() + } stubs_dir = work_dir / "component_stubs" stubs_dir.mkdir(parents=True, exist_ok=True) @@ -305,7 +316,14 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: for component in sorted(arduino_excluded_idf_components()): if _idf_component_dep_name(component) in esphome_deps: continue # ESPHome needs this one for real (don't stub it away) - stub_path = stubs_dir / _idf_component_stub_name(component) + stub_name = _idf_component_stub_name(component) + if (path := converted_paths.get(stub_name)) is not None: + deps[_idf_component_dep_name(component)] = { + "version": "*", + "override_path": path, + } + continue + stub_path = stubs_dir / stub_name stub_path.mkdir(exist_ok=True) (stub_path / "CMakeLists.txt").write_text( "idf_component_register()\n", encoding="utf-8" @@ -401,11 +419,12 @@ def _generate_compile_commands( # Framework deps (e.g. arduino-esp32) + PlatformIO libs converted to local # IDF components, all added to the manifest as deps. extra_deps = dict(settings.framework_deps) - extra_deps.update(_convert_pio_libs(platformio_ini, settings.target_framework)) + converted = _convert_pio_libs(platformio_ini, settings.target_framework) + extra_deps.update(converted) if settings.target_framework == "arduino": # Stub the arduino-bundled components ESPHome doesn't use (avoids the # libsodium clash with noise-c and ~26 unused heavy downloads). - extra_deps.update(_arduino_excluded_stubs(work_dir)) + extra_deps.update(_arduino_excluded_stubs(work_dir, converted)) # Phase 1: discover the components available for this target. _write_tidy_project(work_dir, [], extra_deps, settings) diff --git a/tests/components/api/test.esp32-ard.yaml b/tests/components/api/test.esp32-ard.yaml new file mode 100644 index 0000000000..bd0915f0ef --- /dev/null +++ b/tests/components/api/test.esp32-ard.yaml @@ -0,0 +1,6 @@ +packages: + api: !include common.yaml + +wifi: + ssid: MySSID + password: password1 diff --git a/tests/components/noise/test.esp32-ard.yaml b/tests/components/noise/test.esp32-ard.yaml new file mode 100644 index 0000000000..550ffd1f88 --- /dev/null +++ b/tests/components/noise/test.esp32-ard.yaml @@ -0,0 +1,2 @@ +packages: + noise: !include common.yaml diff --git a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py index 931de627a1..878780cba8 100644 --- a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py +++ b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py @@ -10,11 +10,13 @@ from pathlib import Path from unittest.mock import MagicMock import pytest +import yaml from esphome.components import esp32 import esphome.config_validation as cv from esphome.const import ( KEY_CORE, + KEY_FRAMEWORK_VERSION, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, Framework, @@ -173,3 +175,47 @@ def test_write_idf_component_yml_arduino_stubs_follow_idf_version( / esp32._idf_component_stub_name("espressif__libsodium") ) assert stub_dir.is_dir() is libsodium_stubbed + + +def test_write_idf_component_yml_stub_yields_to_converted_library_of_same_name( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """IDF names a component after its directory and the later registration wins, + so the espressif/libsodium stub and the converted esphome/libsodium raced on + path order (#20102). The stub entry points at the converted library instead.""" + _setup_core(tmp_path) + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(Framework.ARDUINO) + CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version(3, 3, 11) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: {}, + esp32.KEY_IDF_VERSION: cv.Version(5, 5, 5), + esp32.KEY_ARDUINO_LIBRARIES: set(), + } + # A stale stub from an earlier build must not linger beside the real library + stale_stub = tmp_path / "component_stubs" / "libsodium" + stale_stub.mkdir(parents=True) + + converted = MagicMock() + converted.get_sanitized_name.return_value = "esphome/libsodium" + converted.path = tmp_path / "pio_components" / "esphome" / "libsodium" + monkeypatch.setattr( + esp32, "generate_idf_components", lambda libraries, managed=None: [converted] + ) + + esp32._write_idf_component_yml() + + manifest = yaml.safe_load( + (tmp_path / "src" / "idf_component.yml").read_text(encoding="utf-8") + )["dependencies"] + assert manifest["espressif/libsodium"] == { + "version": "*", + "override_path": str(converted.path), + } + assert manifest["esphome/libsodium"] == {"override_path": str(converted.path)} + assert not stale_stub.exists() + # Stubs without a converted namesake are unaffected + assert (tmp_path / "component_stubs" / "cbor").is_dir() + assert manifest["espressif/cbor"]["override_path"] == str( + tmp_path / "component_stubs" / "cbor" + ) diff --git a/tests/unit_tests/test_espidf_clang_tidy.py b/tests/unit_tests/test_espidf_clang_tidy.py index b0d016b1b4..2516a9012f 100644 --- a/tests/unit_tests/test_espidf_clang_tidy.py +++ b/tests/unit_tests/test_espidf_clang_tidy.py @@ -153,6 +153,33 @@ def test_generate_compile_commands_configures_twice( assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"] +def test_generate_compile_commands_hands_converted_libs_to_arduino_stubs( + tmp_path: Path, +) -> None: + """On Arduino the stub generator sees the converted libraries, so a stub + sharing a converted library's directory name can point at it instead.""" + converted = {"esphome/libsodium": {"override_path": str(tmp_path / "libsodium")}} + stubs = {"espressif/libsodium": {"version": "*", "override_path": "x"}} + with ( + patch.object(clang_tidy, "_setup_core"), + patch.object(clang_tidy, "_convert_pio_libs", return_value=converted), + patch.object( + clang_tidy, "_arduino_excluded_stubs", return_value=stubs + ) as mock_stubs, + patch.object(clang_tidy, "_write_tidy_project") as mock_write, + patch("esphome.espidf.toolchain.run_reconfigure", return_value=0), + patch("esphome.build_gen.espidf.get_available_components", return_value=[]), + ): + clang_tidy._generate_compile_commands( + tmp_path, _settings(target_framework="arduino"), tmp_path / "platformio.ini" + ) + + mock_stubs.assert_called_once_with(tmp_path, converted) + extra_deps = mock_write.call_args_list[0].args[2] + assert extra_deps["esphome/libsodium"] == converted["esphome/libsodium"] + assert extra_deps["espressif/libsodium"] == stubs["espressif/libsodium"] + + def test_esphome_manifest_deps_reads_repo_manifest() -> None: """Top-level dependency names, independent of the per-dependency rules.""" manifest = yaml.safe_load( @@ -284,6 +311,27 @@ def test_arduino_excluded_stubs_skips_components_esphome_manifest_provides( assert (stub_path / "CMakeLists.txt").is_file() +def test_arduino_excluded_stubs_points_libsodium_at_converted_library( + tmp_path: Path, +) -> None: + """Below IDF 6.0 the converted esphome/libsodium shares its directory name + with the espressif/libsodium stub; IDF would keep whichever registers last, + so the stub entry points at the converted library instead (#20102).""" + _set_idf_version(cv.Version(5, 5, 4)) + converted_path = str(tmp_path / "pio" / "esphome" / "libsodium") + converted = {"esphome/libsodium": {"override_path": converted_path}} + + deps = _arduino_excluded_stubs(tmp_path, converted) + + assert deps["espressif/libsodium"] == { + "version": "*", + "override_path": converted_path, + } + assert not (tmp_path / "component_stubs" / "libsodium").exists() + # Stubs without a converted namesake are unaffected. + assert (tmp_path / "component_stubs" / "cbor" / "CMakeLists.txt").is_file() + + def test_arduino_excluded_stubs_skips_libsodium_from_idf_6(tmp_path: Path) -> None: """From IDF 6.0 arduino-esp32 drops espressif/libsodium; a stub would clash with esphome/libsodium."""