github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-22 13:56:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Wire version-less requests in one pass once the graph is final

Browse Source 
The settle step now only decides what to fetch by name; every request is
kept for the whole walk and wired at the end to the explicit library of
that name if there is one, else to the node fetched by name, else
reported for the libraries that are built. Nodes nothing reaches are
dropped. That replaces the separate yield, fold, re-match and late
report passes and the miss bookkeeping. The depends= parser also serves
the string form of a library.json dependencies entry.
This commit is contained in:
J. Nick Koston
2026-10-21 16:20:14 -10:00
parent fa823d1a30
commit afd5d324ed
2 changed files with 112 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/platformio/library.py
+78 -120
View File
@@ -801,16 +801,13 @@ def normalize_dependencies(
so callers see a uniform list. ``manifest_name`` names the manifest in the
warning for entries that cannot be normalized.
Bare-name spellings carry no version; the dependency walk later drops
version-less entries at DEBUG, since they are usually names of bundled
Bare-name spellings carry no version; the walk settles them once the graph is known.
framework libraries (Wire, SPI) that need no registry install.
"""
if dependencies is None:
return []
if isinstance(dependencies, str):
# A plain string is one or more comma-separated names; iterating it
# as a list would shred it into one-character "libraries"
return [{"name": n.strip()} for n in dependencies.split(",") if n.strip()]
return parse_properties_depends(dependencies)
if isinstance(dependencies, dict):
normalized = []
for raw_name, spec in dependencies.items():
@@ -988,10 +985,10 @@ def _satisfying_key(
requester: str,
nodes: dict[str, _LibNode],
components: dict[str, ConvertedLibrary],
) -> str | None:
"""The key of a resolved library a version-less dependency names: by registry
package name (with the owner, when the dependency gives one) first, then by
manifest name; never the requester itself."""
) -> tuple[str, bool] | None:
"""The key of a resolved library a version-less dependency names, and whether
only its manifest name matched: the registry package name (with the owner,
when the dependency gives one) is checked first; never the requester itself."""
wanted = name.lower()
def owner_of(key: str, component: ConvertedLibrary) -> str:
@@ -1007,20 +1004,13 @@ def _satisfying_key(
]
for key, _ in candidates:
if (nodes[key].pkgname or key).lower() == wanted:
return key
return key, False
if owner:
return None
for key, component in candidates:
manifest_name = component.data.get("name")
if isinstance(manifest_name, str) and manifest_name.lower() == wanted:
# Visible, so a chance name collision can be pinned to the right package
_LOGGER.info(
"Dependency %s of %s satisfied by %s through its manifest name",
name,
requester,
key,
)
return key
return key, True
return None
@@ -1190,8 +1180,6 @@ def convert_libraries(
)
resolved_requirements: dict[str, frozenset[str]] = {}
# Keys fetched by name that the registry did not know
missed: set[str] = set()
def add_spec(name: str | None, version: str | None, repository: str | None) -> str:
key, kind, locator = _node_key(name, version, repository)
@@ -1201,7 +1189,6 @@ def convert_libraries(
# A registry node (perhaps one the registry did not know) becomes a
# git or local source: resolve it again from that source
resolved_requirements.pop(key, None)
missed.discard(key)
# The same key requested from two different kinds of source (or two
# different local paths) is a config mistake: one silently wins. Warn so
# it isn't a surprise. (git-vs-registry is reported separately below.)
@@ -1257,11 +1244,10 @@ def convert_libraries(
def fetched_by_name(key: str) -> bool:
"""A registry node only version-less manifest dependencies asked for: a
miss skips it, and it folds into a node that names the same package."""
miss skips it, and an explicit library of the same name replaces it."""
node = nodes[key]
return (
key not in top_level_keys
and node.is_registry
and not (node.is_git or node.is_local)
and not node.requirements
)
@@ -1276,18 +1262,19 @@ def convert_libraries(
return True
return False
# Version-less dependencies per requester key, settled once the graph is
# known, so a library already in it satisfies the name before the registry
# is asked; a re-walked requester replaces its entries
# Version-less dependencies per requester key, kept for the whole walk and
# wired once the graph is final; a re-walked requester replaces its entries
pending_versionless: dict[str, list[dict]] = {}
# Names fetched by name so far, whatever their case: each is asked for once
fetched: dict[tuple[str, str], str] = {}
def settle_versionless() -> list[str]:
"""Wire each deferred dependency to the library that satisfies it, or
fetch it by name; return the new keys to walk."""
"""Fetch by name the deferred dependencies nothing in the graph or the
framework satisfies yet; return the new keys to walk. Entries that need
no wiring later are dropped."""
new_keys: list[str] = []
# One node per name this round, however the requesters spell it
added: dict[tuple[str, str], str] = {}
for requester, dependencies in pending_versionless.items():
kept: list[dict] = []
for dependency in dependencies:
name, owner = dependency["name"], dependency.get("owner")
if _owner_pkgname_to_name(owner, name).lower() in _own_names(
@@ -1296,34 +1283,27 @@ def convert_libraries(
_LOGGER.debug("Dependency %s of %s names itself", name, requester)
continue
# A library in the graph beats a bundled copy, as on the Arduino backend
if (
match := _satisfying_key(name, owner, requester, nodes, components)
) is not None:
_LOGGER.debug(
"Dependency %s of %s satisfied by %s", name, requester, match
)
nodes[requester].edges.add(match)
continue
if backend_provides(name, owner):
_LOGGER.debug(
"Skip backend-provided dependency %s of %s", name, requester
)
continue
spelling = (name.lower(), (owner or "").lower())
if (dep_key := added.get(spelling)) is None:
try:
dep_key = add_spec(
_owner_pkgname_to_name(owner, name), None, None
)
except RuntimeError as e:
_LOGGER.warning(
"Skipping dependency %s of %s: %s", name, requester, e
if _satisfying_key(name, owner, requester, nodes, components) is None:
if backend_provides(name, owner):
_LOGGER.debug(
"Skip backend-provided dependency %s of %s", name, requester
)
continue
added[spelling] = dep_key
new_keys.append(dep_key)
nodes[requester].edges.add(dep_key)
pending_versionless.clear()
spelling = (name.lower(), (owner or "").lower())
if spelling not in fetched:
try:
dep_key = add_spec(
_owner_pkgname_to_name(owner, name), None, None
)
except RuntimeError as e:
_LOGGER.warning(
"Skipping dependency %s of %s: %s", name, requester, e
)
continue
fetched[spelling] = dep_key
new_keys.append(dep_key)
kept.append(dependency)
dependencies[:] = kept
return new_keys
worklist = deque(dict.fromkeys(top_level))
@@ -1357,7 +1337,6 @@ def convert_libraries(
raise
# Reported per requester once the graph is final
_LOGGER.debug("Library %s is not in the registry: %s", key, e)
missed.add(key)
continue
component = ConvertedLibrary(
_owner_pkgname_to_name(owner, name), version, URLSource(url, size)
@@ -1517,74 +1496,53 @@ def convert_libraries(
source,
)
# A library asked for explicitly beats one fetched by name that goes by the
# same name, whichever round brought it in, and one package fetched by name
# under two spellings keeps a single node; the settle step applied the first
# rule to the names it could match up front
# Every version-less request resolves to the explicit library that goes by
# its name if the final graph has one, else to the node fetched by name for
# that package, else to nothing
explicit = {k: c for k, c in components.items() if not fetched_by_name(k)}
yielded = False
for key in [k for k in components if fetched_by_name(k)]:
node = nodes[key]
match = _satisfying_key(node.pkgname, node.owner, key, nodes, explicit)
if match is None:
canonical = components[key].get_sanitized_name()
match = next(
(
k
for k, c in components.items()
if k != key and c.get_sanitized_name() == canonical
),
None,
)
if match is None:
continue
_LOGGER.debug("Library %s fetched by name yields to %s", key, match)
del components[key]
yielded = True
for other in nodes.values():
if key in other.edges:
other.edges.remove(key)
other.edges.add(match)
# A name the registry did not know may be satisfied by a library a later
# round pulled in; match those against the finished graph, with no refetch
late: list[tuple[str, str, str | None]] = []
for key in components:
node = nodes[key]
for missing in node.edges & missed:
match = _satisfying_key(
nodes[missing].pkgname, nodes[missing].owner, key, nodes, components
)
if match is not None:
node.edges.add(match)
late.append((key, missing, match))
if yielded:
# What only the yielded library needed is not built either
reachable: set[str] = set()
stack = [key for key in top_level if key in components]
while stack:
if (key := stack.pop()) in reachable:
by_name = {k: c for k, c in components.items() if fetched_by_name(k)}
unresolved: dict[tuple[str, str], None] = {}
for requester, dependencies in pending_versionless.items():
for dependency in dependencies:
name, owner = dependency["name"], dependency.get("owner")
found = _satisfying_key(
name, owner, requester, nodes, explicit
) or _satisfying_key(name, owner, requester, nodes, by_name)
if found is None:
unresolved[(requester, name)] = None
continue
reachable.add(key)
stack.extend(edge for edge in nodes[key].edges if edge in components)
for key in [key for key in components if key not in reachable]:
del components[key]
key, by_manifest_name = found
if by_manifest_name:
# Visible, so a chance name collision can be pinned to the right package
_LOGGER.info(
"Dependency %s of %s satisfied by %s through its manifest name",
name,
requester,
key,
)
else:
_LOGGER.debug(
"Dependency %s of %s satisfied by %s", name, requester, key
)
nodes[requester].edges.add(key)
# Reported once the graph is final, for every library that asked and is built
for key, missing, match in late:
if key not in components:
# What no listed library needs any more is not built: a node fetched by name
# that an explicit library replaced, and whatever only it pulled in
reachable: set[str] = set()
stack = [key for key in top_level if key in components]
while stack:
if (key := stack.pop()) in reachable:
continue
if match is None:
reachable.add(key)
stack.extend(edge for edge in nodes[key].edges if edge in components)
for key in [key for key in components if key not in reachable]:
_LOGGER.debug("Library %s is not needed; dropping it", key)
del components[key]
for requester, name in unresolved:
if requester in components:
_LOGGER.warning(
"Dependency %s of %s is not in the registry; skipping", missing, key
)
else:
_LOGGER.info(
"Dependency %s of %s is not in the registry; satisfied by %s",
missing,
key,
match,
"Dependency %s of %s was not resolved; skipping", name, requester
)
# Two graph nodes that resolve to the same component name (e.g. a package
tests/unit_tests/test_platformio_library.py
+34 -127
View File
@@ -4,11 +4,13 @@ Covers the shared download/parse/resolve/dependency-walk paths in
``esphome.platformio.library`` directly (the ESP-IDF and Zephyr backends are
exercised in their own test modules)."""
from collections.abc import Collection
import json
import logging
from pathlib import Path
from unittest.mock import Mock
from platformio.package.exception import UnknownPackageError
import pytest
from esphome.core import CORE, EsphomeError, Library
@@ -263,14 +265,20 @@ def test_make_registry_client_creates_http_cache_dir(
def _patch_registry_resolve(
monkeypatch: pytest.MonkeyPatch,
*,
unknown: Collection[str] = (),
owner: str | None = None,
) -> list[tuple[str | None, str]]:
"""Stub the registry lookup so tests never touch the network; returns the
(owner, name) pairs looked up."""
(owner, name) pairs looked up. ``unknown`` names miss; ``owner`` is what the
registry answers for an owner-less name."""
asked: list[tuple[str | None, str]] = []
def resolve(owner, pkgname, requirements):
asked.append((owner, pkgname))
return _resolution(owner, pkgname)
def resolve(requested_owner, pkgname, requirements):
asked.append((requested_owner, pkgname))
if pkgname in unknown:
raise UnknownPackageError(pkgname)
return _resolution(requested_owner or owner, pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
return asked
@@ -281,9 +289,11 @@ def _resolution(owner, pkgname):
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None
def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properties=()):
def _patch_download_with_manifests(
monkeypatch, tmp_path, manifests, *, properties=(), unknown=(), owner=None
):
"""Fake ConvertedLibrary.download to materialize canned manifests on disk;
returns the registry lookups made."""
returns the registry lookups made (see _patch_registry_resolve)."""
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_require_name()
@@ -294,7 +304,7 @@ def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properti
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
return _patch_registry_resolve(monkeypatch)
return _patch_registry_resolve(monkeypatch, unknown=unknown, owner=owner)
def test_wave_requirement_growth_defers_the_superseded_download(tmp_path, monkeypatch):
@@ -1038,6 +1048,7 @@ def test_walk_resolves_properties_only_depends(tmp_path, monkeypatch) -> None:
backend = _backend(provides=lambda name: name == "Wire")
top = convert_libraries([Library("esphome/A", "1.0.0", None)], backend)
assert [d.name for d in top[0].dependencies] == ["SPI"]
assert "depends" not in top[0].data
assert backend.provided_requests == {"Wire"}
assert (None, "SPI") in asked
@@ -1146,8 +1157,6 @@ def test_versionless_dependency_registry_miss_is_skipped(
"""A name-only dependency the registry does not know is skipped with one
warning naming the requester, and the requester still converts; a listed
library the registry does not know still fails the build."""
from platformio.package.exception import UnknownPackageError
_patch_download_with_manifests(
monkeypatch,
tmp_path,
@@ -1158,20 +1167,13 @@ def test_versionless_dependency_registry_miss_is_skipped(
"dependencies": [{"name": "Hash"}, {"name": "Hash"}],
}
},
unknown={"Hash"},
)
def resolve(owner, pkgname, requirements):
if pkgname == "Hash":
raise UnknownPackageError("Hash")
return _resolution(owner, pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
top = convert_libraries([Library("esphome/A", None, None)], _backend())
assert top[0].dependencies == []
assert (
caplog.text.count(
"Dependency Hash of esphome/A is not in the registry; skipping"
)
caplog.text.count("Dependency Hash of esphome/A was not resolved; skipping")
== 1
)
with pytest.raises(UnknownPackageError):
@@ -1229,27 +1231,6 @@ def test_versionless_owner_qualified_dependency_is_fetched_despite_provides(
assert not backend.provided_requests
def test_versionless_dependency_requested_top_level_is_wired(
tmp_path, monkeypatch
) -> None:
"""A version-less dependency the config also requests top-level is wired
to that library, not fetched a second time."""
asked = _patch_download_with_manifests(
monkeypatch,
tmp_path,
{
"esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]},
"Hash": {"name": "Hash"},
},
)
top = convert_libraries(
[Library("esphome/A", None, None), Library("Hash", None, None)],
_backend(),
)
assert [d.name for d in top[0].dependencies] == ["Hash"]
assert asked == [("esphome", "A"), (None, "Hash")]
def test_versionless_url_ish_dependency_name_warns_cleanly(
tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
) -> None:
@@ -1322,7 +1303,8 @@ def test_versionless_dependency_settles_after_the_graph(
) -> None:
"""The issue 21308 shape: a library.properties depends= naming a library the
config lists owner-qualified is wired to that one package, case-insensitively
and whatever constraint it carries, without a second node for the bare name."""
and whatever constraint it carries, without a second node for the bare name
and ahead of a same-named bundled copy, as on the Arduino backend."""
asked = _patch_download_with_manifests(
monkeypatch,
tmp_path,
@@ -1334,42 +1316,18 @@ def test_versionless_dependency_settles_after_the_graph(
},
properties=("adafruit/Adafruit GFX Library",),
)
backend = _backend(provides=lambda name: name == "adafruit busio")
top = convert_libraries(
[
Library("adafruit/Adafruit GFX Library", "^1.13.0", None),
Library("adafruit/Adafruit BusIO", "^1.17.4", None),
],
_backend(),
backend,
)
assert [d.name for d in top[0].dependencies] == ["adafruit/Adafruit BusIO"]
assert len(asked) == 2
assert "multiple names" not in caplog.text
def test_versionless_dependency_prefers_a_listed_library_to_the_bundled_one(
tmp_path, monkeypatch
) -> None:
"""A library the config lists wins over a same-named bundled one, as on the
Arduino backend, so the requester is wired to it."""
asked = _patch_download_with_manifests(
monkeypatch,
tmp_path,
{
"esphome/A": {"name": "A", "dependencies": [{"name": "Ethernet"}]},
"arduino-libraries/Ethernet": {"name": "Ethernet"},
},
)
backend = _backend(provides=lambda name: name == "Ethernet")
top = convert_libraries(
[
Library("esphome/A", None, None),
Library("arduino-libraries/Ethernet", "2.0.0", None),
],
backend,
)
assert [d.name for d in top[0].dependencies] == ["arduino-libraries/Ethernet"]
assert not backend.provided_requests
assert len(asked) == 2
assert "multiple names" not in caplog.text
def test_by_name_node_folds_into_a_later_pin_of_the_same_package(
@@ -1385,13 +1343,8 @@ def test_by_name_node_folds_into_a_later_pin_of_the_same_package(
"esphome/B": {"name": "B", "dependencies": {"esphome/C": "1.0.0"}},
"esphome/C": {"name": "C", "dependencies": {"esphome/B": "1.0.0"}},
}
asked = _patch_download_with_manifests(monkeypatch, tmp_path, manifests)
_patch_download_with_manifests(monkeypatch, tmp_path, manifests, owner="esphome")
def resolve(owner, pkgname, requirements):
asked.append((owner, pkgname))
return _resolution("esphome", pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
emitted: list[str] = []
top = convert_libraries(
[Library("esphome/A", None, None)],
@@ -1485,8 +1438,6 @@ def test_by_name_node_pinned_later_reports_a_requirement_conflict(
def test_by_name_fetch_yields_to_a_fork_pulled_in_later(
tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture
) -> None:
from platformio.package.exception import UnknownPackageError
"""A name fetched from the registry yields to a library another round
pins explicitly under the same manifest name, so only the fork is built."""
_patch_download_with_manifests(
@@ -1502,14 +1453,9 @@ def test_by_name_fetch_yields_to_a_fork_pulled_in_later(
"C": {"name": "C", "dependencies": {"custom/B-fork": "1.0.0"}},
"custom/B-fork": {"name": "B"},
},
unknown={"Zed"},
)
def resolve(owner, pkgname, requirements):
if pkgname == "Zed":
raise UnknownPackageError("Zed")
return _resolution(owner, pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
emitted: list[str] = []
top = convert_libraries(
[Library("esphome/A", None, None), Library("esphome/X", None, None)],
@@ -1547,7 +1493,7 @@ def test_git_source_replacing_a_registry_key_in_the_same_wave(
def test_owner_qualified_name_matches_the_resolved_owner(tmp_path, monkeypatch) -> None:
"""A version-less ``o/B`` is satisfied by a bare ``B`` pin the registry
resolved to owner ``o``, so the package is not fetched a second time."""
_patch_download_with_manifests(
asked = _patch_download_with_manifests(
monkeypatch,
tmp_path,
{
@@ -1560,14 +1506,8 @@ def test_owner_qualified_name_matches_the_resolved_owner(tmp_path, monkeypatch)
},
"o/B": {"name": "B"},
},
owner="o",
)
asked: list[tuple[str | None, str]] = []
def resolve(owner, pkgname, requirements):
asked.append((owner, pkgname))
return _resolution(owner or "o", pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
top = convert_libraries([Library("esphome/A", None, None)], _backend())
assert [d.name for d in top[0].dependencies] == ["o/B"]
assert asked.count((None, "B")) == 1
@@ -1587,11 +1527,7 @@ def test_two_spellings_fetched_by_name_fold_into_one_node(
"esphome/X": {"name": "X", "dependencies": [{"name": "B", "owner": "o"}]},
"o/B": {"name": "B"},
},
)
monkeypatch.setattr(
lib,
"_resolve_registry_version",
lambda owner, pkgname, requirements: _resolution(owner or "o", pkgname),
owner="o",
)
emitted: list[str] = []
top = convert_libraries(
@@ -1607,8 +1543,6 @@ def test_two_spellings_fetched_by_name_fold_into_one_node(
def test_registry_miss_resolves_again_as_a_git_source(tmp_path, monkeypatch) -> None:
"""A key the registry missed is resolved again when a later manifest names
the same key as a git source."""
from platformio.package.exception import UnknownPackageError
_patch_download_with_manifests(
monkeypatch,
tmp_path,
@@ -1618,14 +1552,9 @@ def test_registry_miss_resolves_again_as_a_git_source(tmp_path, monkeypatch) ->
"C": {"name": "C", "dependencies": {"B": "https://github.com/o/B.git"}},
"o/B": {"name": "B"},
},
unknown={"B"},
)
def resolve(owner, pkgname, requirements):
if pkgname == "B":
raise UnknownPackageError("B")
return _resolution(owner, pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
emitted: dict[str, ConvertedLibrary] = {}
top = convert_libraries(
[Library("esphome/A", None, None), Library("esphome/X", None, None)],
@@ -1641,9 +1570,7 @@ def test_missed_name_is_satisfied_by_a_library_pulled_in_later(
) -> None:
"""A bare name the registry does not know is wired to a library a later
round pulls in under another package name but the same manifest name."""
from platformio.package.exception import UnknownPackageError
_patch_download_with_manifests(
asked = _patch_download_with_manifests(
monkeypatch,
tmp_path,
{
@@ -1653,16 +1580,8 @@ def test_missed_name_is_satisfied_by_a_library_pulled_in_later(
"C": {"name": "C", "dependencies": {"owner/B-fork": "1.0.0"}},
"owner/B-fork": {"name": "B"},
},
unknown={"B"},
)
asked: list[tuple[str | None, str]] = []
def resolve(owner, pkgname, requirements):
asked.append((owner, pkgname))
if pkgname == "B":
raise UnknownPackageError("B")
return _resolution(owner, pkgname)
monkeypatch.setattr(lib, "_resolve_registry_version", resolve)
with caplog.at_level(logging.INFO):
top = convert_libraries(
[Library("esphome/A", None, None), Library("esphome/X", None, None)],
@@ -1671,7 +1590,7 @@ def test_missed_name_is_satisfied_by_a_library_pulled_in_later(
assert [d.name for d in top[0].dependencies] == ["owner/B-fork"]
assert asked.count((None, "B")) == 1
assert (
"Dependency B of esphome/A is not in the registry; satisfied by owner/B-fork"
"Dependency B of esphome/A satisfied by owner/B-fork through its manifest name"
in caplog.text
)
assert "skipping" not in caplog.text
@@ -1730,18 +1649,6 @@ def test_provided_dependency_asks_the_backend_once_per_name(
assert (None, "Wire") not in asked
def test_parse_library_properties_turns_depends_into_dependencies(
tmp_path: Path,
) -> None:
manifest = tmp_path / "library.properties"
manifest.write_text("name=A\ndepends=Wire (>=1.0)\n")
data = lib.parse_library_properties(manifest)
assert data["dependencies"] == [{"name": "Wire"}]
assert "depends" not in data
manifest.write_text("name=A\ndepends=\n")
assert "dependencies" not in lib.parse_library_properties(manifest)
def test_convert_libraries_symlink_url_resolves_as_local(setup_core: Path) -> None:
"""symlink:// is PlatformIO's other spelling for a local library folder."""
src = setup_core / "lib_dev"