Match a listed library before the bundled copy and fold by-name duplicates

A library in the graph wins over a same-named bundled one, as on the
Arduino backend. A node fetched by name alone that resolves to a package
another manifest later pins folds into that node. Pending entries are
kept per requester so a re-walk replaces them. ESP32 Arduino warns when
a manifest depends on a bundled library that is not enabled.

esphome/components/esp32/__init__.py

@@ -3813,6 +3813,20 @@ def _write_sdkconfig():
             clean_build(clear_pio_cache=False)
 
 
+def _arduino_provides(name: str) -> bool:
+    """Whether arduino-esp32 bundles the library a manifest depends on, so it is
+    never fetched by name; it still has to be enabled to build."""
+    if name not in ARDUINO_DISABLED_LIBRARIES:
+        return False
+    if name not in CORE.data[KEY_ESP32][KEY_ARDUINO_LIBRARIES]:
+        _LOGGER.warning(
+            "A library depends on the Arduino library %s, which is not enabled; "
+            "add it under esphome: libraries:",
+            name,
+        )
+    return True
+
+
 def _write_idf_component_yml():
     yml_path = CORE.relative_build_path("src/idf_component.yml")
     dependencies: dict[str, dict] = {}
@@ -3837,10 +3851,7 @@ def _write_idf_component_yml():
         converted = generate_idf_components(
             libraries,
             managed=managed,
-            # The arduino-esp32 component bundles these; never fetch them by name
-            provides=ARDUINO_DISABLED_LIBRARIES.__contains__
-            if CORE.using_arduino
-            else None,
+            provides=_arduino_provides if CORE.using_arduino else None,
         )
     # IDF names a component after its directory and a later registration of the
     # same name replaces the earlier one, so a stub beside a converted library of

esphome/platformio/library.py

@@ -615,7 +615,7 @@ def parse_library_properties(library_properties_path: PathType):
 
 
 def parse_properties_depends(depends: str) -> list[dict[str, str]]:
-    """``depends=Name, Other (>=1.0)`` as dependency entries, as PlatformIO reads it."""
+    """The Arduino ``depends=Name, Other (>=1.0)`` line as dependency entries."""
     entries: list[dict[str, str]] = []
     for item in depends.split(","):
         name, _, version = item.strip().partition("(")
@@ -1203,37 +1203,47 @@ def convert_libraries(
     components: dict[str, ConvertedLibrary] = {}
     resolved_requirements: dict[str, frozenset[str]] = {}
     top_level_keys = set(top_level)
-    # (requester key, dependency) settled once the graph is known, so a library
-    # already in it satisfies the name before the registry is asked
-    pending_versionless: list[tuple[str, dict]] = []
+    # Version-less dependencies per requester key, settled once the graph is
+    # known, so a library already in it satisfies the name before the registry
+    # is asked; a re-walked requester replaces its entries
+    pending_versionless: dict[str, list[dict]] = {}
 
     def settle_versionless() -> list[str]:
         """Wire each deferred dependency to the library that satisfies it, or
         fetch it by name; return the new keys to walk."""
         new_keys: list[str] = []
-        for requester, dependency in pending_versionless:
-            name, owner = dependency["name"], dependency.get("owner")
-            if not owner and backend.provides is not None and backend.provides(name):
-                _LOGGER.debug(
-                    "Skip backend-provided dependency %s of %s", name, requester
-                )
-                backend.provided_requests.add(name)
-                continue
-            if (match := _satisfying_key(name, owner, nodes, components)) is not None:
-                _LOGGER.debug(
-                    "Dependency %s of %s satisfied by %s", name, requester, match
-                )
-                nodes[requester].edges.add(match)
-                continue
-            try:
-                dep_key = add_spec(_owner_pkgname_to_name(owner, name), None, None)
-            except RuntimeError as e:
-                _LOGGER.warning("Skipping dependency %s of %s: %s", name, requester, e)
-                continue
-            if dep_key not in resolved_requirements:
+        for requester, dependencies in pending_versionless.items():
+            for dependency in dependencies:
+                name, owner = dependency["name"], dependency.get("owner")
+                # A library in the graph beats a bundled copy, as on the Arduino backend
+                if (
+                    match := _satisfying_key(name, owner, nodes, components)
+                ) is not None:
+                    _LOGGER.debug(
+                        "Dependency %s of %s satisfied by %s", name, requester, match
+                    )
+                    nodes[requester].edges.add(match)
+                    continue
+                if (
+                    not owner
+                    and backend.provides is not None
+                    and backend.provides(name)
+                ):
+                    _LOGGER.debug(
+                        "Skip backend-provided dependency %s of %s", name, requester
+                    )
+                    backend.provided_requests.add(name)
+                    continue
+                try:
+                    dep_key = add_spec(_owner_pkgname_to_name(owner, name), None, None)
+                except RuntimeError as e:
+                    _LOGGER.warning(
+                        "Skipping dependency %s of %s: %s", name, requester, e
+                    )
+                    continue
                 nodes[dep_key].optional = True
-            nodes[requester].edges.add(dep_key)
-            new_keys.append(dep_key)
+                nodes[requester].edges.add(dep_key)
+                new_keys.append(dep_key)
         pending_versionless.clear()
         return new_keys
 
@@ -1360,6 +1370,7 @@ def convert_libraries(
             # Requirements changed (we got past the short-circuit above), so
             # (re)walk this component's dependencies.
             node.edges = set()
+            pending = pending_versionless[key] = []
             for dependency in normalize_dependencies(
                 component.data.get("dependencies"), component.name
             ):
@@ -1370,7 +1381,7 @@ def convert_libraries(
                         dependency, backend.platform, backend.framework, component.name
                     ):
                         # Filtered or ignored deps are deliberately absent
-                        pending_versionless.append((key, dependency))
+                        pending.append(dependency)
                     continue
                 if not dependency_is_usable(
                     dependency, backend.platform, backend.framework, component.name
@@ -1437,11 +1448,23 @@ def convert_libraries(
 
     # Two graph nodes that resolve to the same component name (e.g. a package
     # referenced both bare and as ``owner/name``) are not deduplicated and can
-    # produce conflicting component definitions. Warn so it's not silent.
+    # produce conflicting component definitions. A node fetched by name alone
+    # folds into the other one; otherwise warn so it's not silent.
     canonical_keys: dict[str, str] = {}
-    for node_key, component in components.items():
+    for node_key, component in list(components.items()):
         canonical = component.get_sanitized_name()
         if canonical_keys.setdefault(canonical, node_key) != node_key:
+            first = canonical_keys[canonical]
+            drop = next((k for k in (node_key, first) if nodes[k].optional), None)
+            if drop is not None:
+                keep = first if drop == node_key else node_key
+                canonical_keys[canonical] = keep
+                del components[drop]
+                for node in nodes.values():
+                    if drop in node.edges:
+                        node.edges.remove(drop)
+                        node.edges.add(keep)
+                continue
             _LOGGER.warning(
                 "Library %s is referenced under multiple names (%s and %s); these "
                 "are not deduplicated. Reference it consistently as %s.",

tests/unit_tests/components/test_esp32_write_idf_component_yml.py

@@ -119,6 +119,7 @@ def test_write_idf_component_yml_marks_arduino_bundled_libraries_provided(
     CORE.data[esp32.KEY_ESP32] = {
         esp32.KEY_COMPONENTS: {},
         esp32.KEY_IDF_VERSION: cv.Version(5, 5, 4),
+        esp32.KEY_ARDUINO_LIBRARIES: {"Wire", "SPI"},
     }
 
     captured: dict[str, object] = {}
@@ -136,6 +137,20 @@ def test_write_idf_component_yml_marks_arduino_bundled_libraries_provided(
     assert not provides("Adafruit BusIO")
 
 
+def test_arduino_provides_warns_for_a_bundled_library_not_enabled(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A manifest dependency on a bundled Arduino library is never fetched, but
+    the library still has to be enabled, so an unenabled one is called out."""
+    CORE.reset()
+    CORE.data[esp32.KEY_ESP32] = {esp32.KEY_ARDUINO_LIBRARIES: {"Wire"}}
+    assert esp32._arduino_provides("Wire")
+    assert "Wire" not in caplog.text
+    assert esp32._arduino_provides("SPI")
+    assert "Arduino library SPI, which is not enabled" in caplog.text
+    assert not esp32._arduino_provides("Adafruit BusIO")
+
+
 @pytest.mark.parametrize(
     ("version", "libsodium_stubbed"),
     [

tests/unit_tests/test_arduino_library.py

@@ -852,7 +852,6 @@ def test_bundled_library_properties_depends_warns(
     wire = framework / "libraries" / "Wire"
     (wire / "library.properties").write_text("name=Wire\nversion=1.0\ndepends=SPI\n")
     _add_library("Wire", None)
-    caplog.set_level("INFO")
     _resolve(framework)
     assert "Bundled library Wire declares dependencies" in caplog.text
 

tests/unit_tests/test_espidf_component.py

@@ -15,7 +15,6 @@ from esphome.const import (
     Platform,
 )
 from esphome.core import CORE, Library
-import esphome.espidf.component as espidf_component
 from esphome.espidf.component import (
     _emit_idf_component,
     generate_cmakelists_txt,
@@ -417,24 +416,6 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component):
         generate_idf_component_yml(tmp_component)
 
 
-def test_generate_idf_components_passes_provides_to_the_backend(
-    esp32_idf_core, monkeypatch
-) -> None:
-    """The bundled-name predicate reaches the walk, so such names are never fetched."""
-    backends = []
-    monkeypatch.setattr(
-        espidf_component,
-        "convert_libraries",
-        lambda libraries, backend, provided=None: backends.append(backend) or [],
-    )
-
-    def provides(name: str) -> bool:
-        return name == "Wire"
-
-    assert generate_idf_components([], provides=provides) == []
-    assert backends[0].provides is provides
-
-
 def test_parse_library_json(tmp_path):
     f = tmp_path / "library.json"
     f.write_text(json.dumps({"name": "test"}))

tests/unit_tests/test_platformio_library.py

@@ -261,23 +261,11 @@ def test_make_registry_client_creates_http_cache_dir(
     lib._make_registry_client()
 
 
-def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Stub the registry lookup so tests never touch the network."""
-    monkeypatch.setattr(
-        lib,
-        "_resolve_registry_version",
-        lambda owner, pkgname, requirements: (
-            owner,
-            pkgname,
-            "1.0.0",
-            f"http://x/{pkgname}.tar.gz",
-            None,
-        ),
-    )
-
-
-def _record_registry(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str | None, str]]:
-    """Stub the registry like _patch_registry_resolve, recording each lookup."""
+def _patch_registry_resolve(
+    monkeypatch: pytest.MonkeyPatch,
+) -> list[tuple[str | None, str]]:
+    """Stub the registry lookup so tests never touch the network; returns the
+    (owner, name) pairs looked up."""
     asked: list[tuple[str | None, str]] = []
 
     def resolve(owner, pkgname, requirements):
@@ -289,7 +277,8 @@ def _record_registry(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str | None,
 
 
 def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properties=()):
-    """Fake ConvertedLibrary.download to materialize canned manifests on disk."""
+    """Fake ConvertedLibrary.download to materialize canned manifests on disk;
+    returns the registry lookups made."""
 
     def fake_download(self, force=False, salt="", namespace=""):
         self.path = tmp_path / self.get_require_name()
@@ -300,7 +289,7 @@ def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properti
             (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
 
     monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
-    _patch_registry_resolve(monkeypatch)
+    return _patch_registry_resolve(monkeypatch)
 
 
 def test_wave_requirement_growth_defers_the_superseded_download(tmp_path, monkeypatch):
@@ -357,7 +346,7 @@ def test_convert_libraries_parses_library_properties(tmp_path, monkeypatch):
 
 def test_convert_libraries_fetches_dependency_without_version(tmp_path, monkeypatch):
     # A dependency entry lacking a version resolves by name alone, as PlatformIO did.
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -365,7 +354,6 @@ def test_convert_libraries_fetches_dependency_without_version(tmp_path, monkeypa
             "C": {"name": "C"},
         },
     )
-    asked = _record_registry(monkeypatch)
 
     # No version on the top-level spec exercises the "no requirement" path too.
     top = convert_libraries([Library("esphome/A", None, None)], _backend())
@@ -1033,7 +1021,7 @@ def test_convert_libraries_malformed_transitive_dep_skips(
 def test_walk_resolves_properties_only_depends(tmp_path, monkeypatch) -> None:
     """A library.properties depends= line is a dependency list: a bundled name
     goes to the backend, the rest resolve by name."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1042,7 +1030,6 @@ def test_walk_resolves_properties_only_depends(tmp_path, monkeypatch) -> None:
         },
         properties=("esphome/A",),
     )
-    asked = _record_registry(monkeypatch)
     backend = _backend(provides=lambda name: name == "Wire")
     top = convert_libraries([Library("esphome/A", "1.0.0", None)], backend)
     assert [d.name for d in top[0].dependencies] == ["SPI"]
@@ -1119,7 +1106,7 @@ def test_versionless_platform_filtered_dependency_is_not_fetched(
 ) -> None:
     """A version-less dependency the platform filter excludes is deliberately
     absent, not fetched by name."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1129,7 +1116,6 @@ def test_versionless_platform_filtered_dependency_is_not_fetched(
             }
         },
     )
-    asked = _record_registry(monkeypatch)
     top = convert_libraries([Library("esphome/A", None, None)], _backend())
     assert top[0].dependencies == []
     assert (None, "Hash") not in asked
@@ -1138,12 +1124,11 @@ def test_versionless_platform_filtered_dependency_is_not_fetched(
 def test_versionless_ignored_dependency_is_not_fetched(tmp_path, monkeypatch) -> None:
     """A lib_ignore'd version-less dependency is deliberately excluded, not
     fetched by name."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {"esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]}},
     )
-    asked = _record_registry(monkeypatch)
     CORE.platformio_options = {"lib_ignore": ["Hash"]}
     top = convert_libraries([Library("esphome/A", None, None)], _backend())
     assert top[0].dependencies == []
@@ -1216,7 +1201,7 @@ def test_versionless_owner_qualified_dependency_is_fetched_despite_provides(
 ) -> None:
     """An owner-qualified version-less dependency names a registry package,
     so provides() does not stand in for it."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1227,7 +1212,6 @@ def test_versionless_owner_qualified_dependency_is_fetched_despite_provides(
             "Foo/Wire": {"name": "Wire"},
         },
     )
-    asked = _record_registry(monkeypatch)
     backend = _backend(provides=lambda name: name == "Wire")
     top = convert_libraries([Library("esphome/A", None, None)], backend)
     assert [d.name for d in top[0].dependencies] == ["Foo/Wire"]
@@ -1240,12 +1224,11 @@ def test_versionless_provided_dependency_is_left_to_the_backend(
 ) -> None:
     """An owner-less version-less dependency the backend provides is recorded
     for it and never fetched."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {"esphome/A": {"name": "A", "dependencies": [{"name": "Wire"}]}},
     )
-    asked = _record_registry(monkeypatch)
     backend = _backend(provides=lambda name: name == "Wire")
     top = convert_libraries([Library("esphome/A", None, None)], backend)
     assert top[0].dependencies == []
@@ -1258,7 +1241,7 @@ def test_versionless_dependency_requested_top_level_is_wired(
 ) -> None:
     """A version-less dependency the config also requests top-level is wired
     to that library, not fetched a second time."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1266,7 +1249,6 @@ def test_versionless_dependency_requested_top_level_is_wired(
             "Hash": {"name": "Hash"},
         },
     )
-    asked = _record_registry(monkeypatch)
     top = convert_libraries(
         [Library("esphome/A", None, None), Library("Hash", None, None)],
         _backend(),
@@ -1295,7 +1277,7 @@ def test_versionless_dependency_matching_resolved_manifest_name_is_wired(
 ) -> None:
     """A bare name satisfied by an owner-qualified component's manifest name
     is wired to it."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1303,7 +1285,6 @@ def test_versionless_dependency_matching_resolved_manifest_name_is_wired(
             "esphome/B": {"name": "B"},
         },
     )
-    asked = _record_registry(monkeypatch)
     top = convert_libraries(
         [Library("esphome/A", None, None), Library("esphome/B", None, None)],
         _backend(),
@@ -1318,7 +1299,7 @@ def test_versionless_dependency_settles_after_the_graph(
     """The issue 21308 shape: a library.properties depends= naming a library the
     config lists owner-qualified is wired to that one package, case-insensitively,
     without a second node for the bare name."""
-    _patch_download_with_manifests(
+    asked = _patch_download_with_manifests(
         monkeypatch,
         tmp_path,
         {
@@ -1329,7 +1310,6 @@ def test_versionless_dependency_settles_after_the_graph(
         },
         properties=("adafruit/Adafruit GFX Library",),
     )
-    asked = _record_registry(monkeypatch)
     top = convert_libraries(
         [
             Library("adafruit/Adafruit GFX Library", "^1.13.0", None),
@@ -1342,6 +1322,62 @@ def test_versionless_dependency_settles_after_the_graph(
     assert "multiple names" not in caplog.text
 
 
+def test_versionless_dependency_prefers_a_listed_library_to_the_bundled_one(
+    tmp_path, monkeypatch
+) -> None:
+    """A library the config lists wins over a same-named bundled one, as on the
+    Arduino backend, so the requester is wired to it."""
+    asked = _patch_download_with_manifests(
+        monkeypatch,
+        tmp_path,
+        {
+            "esphome/A": {"name": "A", "dependencies": [{"name": "Ethernet"}]},
+            "arduino-libraries/Ethernet": {"name": "Ethernet"},
+        },
+    )
+    backend = _backend(provides=lambda name: name == "Ethernet")
+    top = convert_libraries(
+        [
+            Library("esphome/A", None, None),
+            Library("arduino-libraries/Ethernet", "2.0.0", None),
+        ],
+        backend,
+    )
+    assert [d.name for d in top[0].dependencies] == ["arduino-libraries/Ethernet"]
+    assert not backend.provided_requests
+    assert len(asked) == 2
+
+
+def test_by_name_node_folds_into_a_later_pin_of_the_same_package(
+    tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A library fetched by name alone may pull in a manifest that pins the same
+    package owner-qualified; the by-name node folds into that one instead of
+    converting the package twice."""
+    manifests = {
+        "esphome/A": {"name": "A", "dependencies": [{"name": "B"}]},
+        # Fetched by name; its manifest pins C, whose manifest pins owner/B
+        "B": {"name": "B", "dependencies": {"esphome/C": "1.0.0"}},
+        "esphome/C": {"name": "C", "dependencies": {"esphome/B": "1.0.0"}},
+        "esphome/B": {"name": "B"},
+    }
+    asked = _patch_download_with_manifests(monkeypatch, tmp_path, manifests)
+
+    def resolve(owner, pkgname, requirements):
+        asked.append((owner, pkgname))
+        return "esphome", pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None
+
+    monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
+    emitted: list[str] = []
+    top = convert_libraries(
+        [Library("esphome/A", None, None)],
+        _backend(emit=lambda c: emitted.append(c.name)),
+    )
+    assert [d.name for d in top[0].dependencies] == ["esphome/B"]
+    assert emitted.count("esphome/B") == 1
+    assert "multiple names" not in caplog.text
+
+
 def test_parse_properties_depends_reads_names_and_constraints() -> None:
     assert lib.parse_properties_depends("Adafruit BusIO, Foo (>=1.2), ,Bar( <2 )") == [
         {"name": "Adafruit BusIO"},