github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-06-24 14:55:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

[esp32] Deduplicate PlatformIO library conversion by resolving the batch together (#16756)

Browse Source
This commit is contained in:
Jonathan Swoboda
2026-06-04 07:39:17 -04:00
committed by GitHub
parent ffaa31febc
commit 891ec33c94
4 changed files with 702 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

626
tests/unit_tests/test_espidf_component.py
View File

@@ -21,13 +21,15 @@ from esphome.espidf.component import (
URLSource,
_check_library_data,
_collect_filtered_files,
_convert_library_to_component,
_node_key,
_normalize_dependencies,
_parse_library_json,
_parse_library_properties,
_process_dependencies,
_resolve_registry_version,
_split_list_by_condition,
generate_cmakelists_txt,
generate_idf_component_yml,
generate_idf_components,
)
@@ -162,43 +164,6 @@ def test_generate_cmakelists_txt_references_project_managed_components_variable(
assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content
def test_generate_idf_component_overwrites_bundled_files(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# A library that ships its own CMakeLists.txt + idf_component.yml must
# have both replaced by ESPHome's generated content. Library authors'
# bundled IDF metadata is frequently broken (bogus REQUIRES, hard-coded
# frameworks), so we always regenerate from library.json.
from esphome.espidf.component import _generate_idf_component
(tmp_path / "src").mkdir()
(tmp_path / "src" / "main.cpp").write_text("// dummy\n")
(tmp_path / "library.json").write_text(json.dumps({"name": "tripwire-lib"}))
(tmp_path / "CMakeLists.txt").write_text("# TRIPWIRE_BUNDLED_CMAKELISTS\n")
(tmp_path / "idf_component.yml").write_text("# TRIPWIRE_BUNDLED_MANIFEST\n")
fake_component = IDFComponent(
"owner/tripwire-lib", "1.0.0", source=URLSource("http://dummy")
)
fake_component.path = tmp_path
monkeypatch.setattr(
esphome.espidf.component,
"_convert_library_to_component",
lambda _lib: fake_component,
)
monkeypatch.setattr(fake_component, "download", lambda force=False: None)
_generate_idf_component(Library("owner/tripwire-lib", "1.0.0", None))
cml = (tmp_path / "CMakeLists.txt").read_text()
manifest = (tmp_path / "idf_component.yml").read_text()
assert "TRIPWIRE_BUNDLED_CMAKELISTS" not in cml
assert "TRIPWIRE_BUNDLED_MANIFEST" not in manifest
assert "idf_component_register" in cml
def test_generate_idf_component_yml_basic(tmp_component):
tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
result = generate_idf_component_yml(tmp_component)
@@ -419,200 +384,58 @@ empty=
assert "empty" not in result
def test_convert_library_with_repository():
lib = Library("name", None, "https://github.com/foo/bar.git#v1.2.3")
result = _convert_library_to_component(lib)
assert result.name == "foo/bar"
assert result.version == "*"
assert isinstance(result.source, GitSource)
assert result.source.ref == "v1.2.3"
def test_convert_library_with_branch_ref():
lib = Library("name", None, "https://github.com/foo/bar.git#some-branch")
result = _convert_library_to_component(lib)
assert result.name == "foo/bar"
assert result.version == "*"
assert isinstance(result.source, GitSource)
assert result.source.ref == "some-branch"
def test_convert_library_missing_ref_uses_default_branch():
"""A bare URL with no #ref clones the remote's default branch.
Matches PIO's lib_deps behavior and external_components handling --
git.clone_or_update with ref=None leaves the depth-1 clone on
whatever branch the remote HEAD points at.
"""
lib = Library("name", None, "https://github.com/foo/bar.git")
result = _convert_library_to_component(lib)
assert result.name == "foo/bar"
assert result.version == "*"
assert isinstance(result.source, GitSource)
assert result.source.ref is None
def test_convert_library_registry(monkeypatch):
lib = Library("foo/bar", "^1.0.0", None)
monkeypatch.setattr(
esphome.espidf.component,
"_get_package_from_pio_registry",
lambda o, n, r: ("foo", "bar", "1.2.3", "http://example.com/pkg.zip"),
def test_node_key_git_with_ref():
key, is_git, locator = _node_key(
"name", None, "https://github.com/foo/bar.git#v1.2.3"
)
result = _convert_library_to_component(lib)
assert result.name == "foo/bar"
assert result.version == "1.2.3"
assert isinstance(result.source, URLSource)
assert key == "foo/bar"
assert is_git is True
assert locator == ("https://github.com/foo/bar.git", "v1.2.3")
def test_process_dependencies_adds_valid_dependency(tmp_component, monkeypatch):
tmp_component.data = {
"dependencies": [
{
"name": "foo",
"version": "1.0",
}
]
}
monkeypatch.setattr(
esphome.espidf.component,
"_generate_idf_component",
lambda lib: esphome.espidf.component.IDFComponent(
lib.name, lib.version, source=URLSource("http://dummy.com")
),
def test_node_key_git_branch_ref():
key, is_git, locator = _node_key(
"name", None, "https://github.com/foo/bar.git#some-branch"
)
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
_process_dependencies(tmp_component)
assert len(tmp_component.dependencies) == 1
assert (key, is_git, locator[1]) == ("foo/bar", True, "some-branch")
def test_process_dependencies_skips_invalid(tmp_component):
tmp_component.data = {
"dependencies": [
{"name": "foo", "version": "1.0", "platforms": ["arduino"]},
{"invalid": "entry"},
]
}
_process_dependencies(tmp_component)
assert tmp_component.dependencies == []
def test_node_key_git_no_ref():
_key, is_git, locator = _node_key("name", None, "https://github.com/foo/bar.git")
assert is_git is True
assert locator == ("https://github.com/foo/bar.git", None)
def test_process_dependencies_dict_form(tmp_component, monkeypatch):
"""PIO library.json shorthand ``{"owner/Name": "version"}`` is honored.
def test_node_key_registry_owner_name():
key, is_git, locator = _node_key("foo/bar", "^1.0.0", None)
assert (key, is_git, locator) == ("foo/bar", False, ("foo", "bar"))
Iterating a dict gives string keys, which would silently fail the
``"name" in dependency`` substring check. Normalize to list-of-dicts
first so the dict form (used by e.g. tesla-ble for its nanopb dep)
is treated the same as the verbose list form.
"""
captured: list[Library] = []
def fake_generate(library):
captured.append(library)
return IDFComponent(
library.name, library.version, source=URLSource("http://dummy.com")
)
def test_node_key_registry_bare_name():
key, is_git, locator = _node_key("bar", "1.0", None)
assert (key, is_git, locator) == ("bar", False, (None, "bar"))
tmp_component.data = {
"dependencies": {
"nanopb/Nanopb": "^0.4.91",
"BareName": "1.2.3",
}
}
monkeypatch.setattr(
esphome.espidf.component, "_generate_idf_component", fake_generate
def test_normalize_dependencies_none():
assert _normalize_dependencies(None) == []
def test_normalize_dependencies_list_form():
deps = [{"name": "foo", "version": "1.0"}]
assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
def test_normalize_dependencies_dict_form():
out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out
assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out
def test_normalize_dependencies_dict_form_nested_spec():
out = _normalize_dependencies(
{"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}}
)
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
_process_dependencies(tmp_component)
assert len(tmp_component.dependencies) == 2
names = sorted(lib.name for lib in captured)
versions = sorted(lib.version for lib in captured)
assert names == ["BareName", "nanopb/Nanopb"]
assert versions == ["1.2.3", "^0.4.91"]
def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch):
"""A dict-value that's a URL gets routed to ``repository`` like the list form."""
captured: list[Library] = []
def fake_generate(library):
captured.append(library)
return IDFComponent(library.name, "*", source=URLSource("http://dummy.com"))
tmp_component.data = {
"dependencies": {
"foo/Bar": "https://github.com/foo/bar.git#main",
}
}
monkeypatch.setattr(
esphome.espidf.component, "_generate_idf_component", fake_generate
)
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
_process_dependencies(tmp_component)
assert len(captured) == 1
assert captured[0].name == "foo/Bar"
assert captured[0].version is None
assert captured[0].repository == "https://github.com/foo/bar.git#main"
def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch):
"""A dict-value that's itself a dict is merged into the entry.
PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}``
for entries that need fields beyond just a version (platforms,
frameworks, etc.). The extra fields flow into _check_library_data
via the entry merge.
"""
captured: list[Library] = []
checked: list[dict] = []
def fake_generate(library):
captured.append(library)
return IDFComponent(
library.name, library.version, source=URLSource("http://dummy.com")
)
tmp_component.data = {
"dependencies": {
"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"},
}
}
monkeypatch.setattr(
esphome.espidf.component, "_generate_idf_component", fake_generate
)
monkeypatch.setattr(
esphome.espidf.component,
"_check_library_data",
checked.append,
)
_process_dependencies(tmp_component)
assert len(captured) == 1
assert captured[0].name == "nanopb/Nanopb"
assert captured[0].version == "^0.4.91"
# Extra spec fields reach _check_library_data so platform/framework
# gating still applies.
assert checked == [
assert out == [
{
"name": "Nanopb",
"owner": "nanopb",
@@ -620,3 +443,364 @@ def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypa
"platforms": "espidf",
}
]
def _patch_registry(monkeypatch, versions):
"""Patch the registry client to serve a canned version list (no network).
Only ``fetch_registry_package`` is faked; the real
``get_compatible_registry_versions`` / ``pick_best_registry_version`` run on
the canned data so the intersection logic is exercised for real.
"""
registry = esphome.espidf.component._make_registry_client()
monkeypatch.setattr(
registry,
"fetch_registry_package",
lambda spec: {
"owner": {"username": spec.owner or "owner"},
"name": spec.name,
"versions": [
{"name": v, "files": [{"download_url": f"http://x/{v}.tar.gz"}]}
for v in versions
],
},
)
monkeypatch.setattr(
esphome.espidf.component, "_make_registry_client", lambda: registry
)
def test_resolve_registry_version_intersects_constraints(monkeypatch):
_patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"])
owner, name, version, url = _resolve_registry_version(
"esphome", "libsodium", {"==1.10021.0", "^1.10018.1"}
)
assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0")
assert url == "http://x/1.10021.0.tar.gz"
def test_resolve_registry_version_picks_highest_satisfying(monkeypatch):
_patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"])
_owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"})
assert version == "1.5.0"
def test_resolve_registry_version_conflict_raises(monkeypatch):
_patch_registry(monkeypatch, ["1.0.0", "2.0.0"])
with pytest.raises(RuntimeError, match="satisfies all requirements"):
_resolve_registry_version("o", "p", {"==1.0.0", "==2.0.0"})
def test_generate_idf_components_dedupes_shared_dependency(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# A and B both depend on shared C under different version specs. The batch
# must resolve C once with BOTH requirements collected, wire a single C
# instance into both, and regenerate (overwrite) each library's build files.
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [
{"owner": "esphome", "name": "C", "version": "==1.10021.0"}
],
},
"esphome/B": {
"name": "B",
"dependencies": [
{"owner": "esphome", "name": "C", "version": "^1.10018.1"}
],
},
"esphome/C": {"name": "C"},
}
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
(self.path / "CMakeLists.txt").write_text("# TRIPWIRE\n")
monkeypatch.setattr(IDFComponent, "download", fake_download)
captured: dict[str, set[str]] = {}
resolve_calls: list[str] = []
def fake_resolve(owner, pkgname, requirements):
resolve_calls.append(pkgname)
captured[f"{owner}/{pkgname}"] = set(requirements)
version = "1.10021.0" if pkgname == "C" else "1.0.0"
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz"
monkeypatch.setattr(
esphome.espidf.component, "_resolve_registry_version", fake_resolve
)
top = generate_idf_components(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
)
# C resolved once (not once per consumer) with BOTH requirements gathered.
assert captured["esphome/C"] == {"==1.10021.0", "^1.10018.1"}
assert resolve_calls.count("C") == 1
# Top-level components returned in request order.
assert [c.name for c in top] == ["esphome/A", "esphome/B"]
# A and B reference the SAME single C instance (deduped).
a_dep = top[0].dependencies[0]
b_dep = top[1].dependencies[0]
assert a_dep.name == "esphome/C"
assert a_dep is b_dep
# The bundled CMakeLists was overwritten with generated content.
generated = (a_dep.path / "CMakeLists.txt").read_text()
assert "TRIPWIRE" not in generated
assert "idf_component_register" in generated
def test_generate_idf_components_handles_dependency_cycle(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# A -> B -> A. Must terminate (not recurse forever) and wire the cycle with
# a single instance per component.
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}],
},
"esphome/B": {
"name": "B",
"dependencies": [{"owner": "esphome", "name": "A", "version": "1.0.0"}],
},
}
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(IDFComponent, "download", fake_download)
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner,
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
top = generate_idf_components([Library("esphome/A", "1.0.0", None)])
assert [c.name for c in top] == ["esphome/A"]
component_a = top[0]
component_b = component_a.dependencies[0]
assert component_b.name == "esphome/B"
# The cycle is wired back to the same A instance, not a duplicate.
assert component_b.dependencies[0] is component_a
def test_generate_idf_components_git_overrides_registry_warns(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
caplog: pytest.LogCaptureFixture,
) -> None:
# A pulls shared as a registry pin; B pulls the same component from a git
# source. The git source wins, but the dropped registry pin must be warned
# about (not silently discarded).
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [
{"owner": "esphome", "name": "shared", "version": "==1.0.0"}
],
},
"esphome/B": {
"name": "B",
"dependencies": [
{
"owner": "esphome",
"name": "shared",
"version": "https://github.com/esphome/shared.git#main",
}
],
},
"esphome/shared": {"name": "shared"},
}
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(IDFComponent, "download", fake_download)
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner,
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
top = generate_idf_components(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
)
# shared resolved from the git source (version "*"), not the registry pin.
shared = top[0].dependencies[0]
assert shared.name == "esphome/shared"
assert isinstance(shared.source, GitSource)
assert "using the git source" in caplog.text
assert "==1.0.0" in caplog.text
def test_generate_idf_components_missing_manifest_raises(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# A library with neither library.json nor library.properties is invalid;
# fail loudly rather than silently generating build files for it.
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
# no library.json / library.properties written
monkeypatch.setattr(IDFComponent, "download", fake_download)
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner,
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
with pytest.raises(RuntimeError, match="missing library.json"):
generate_idf_components([Library("esphome/A", "1.0.0", None)])
def test_generate_idf_components_warns_on_noncanonical_duplicate(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
caplog: pytest.LogCaptureFixture,
) -> None:
# A references "shared" (bare) and B references "owner/shared"; both resolve
# to the same canonical name but as distinct graph nodes, so they aren't
# deduplicated -- warn about it.
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [{"name": "shared", "version": "1.0.0"}],
},
"esphome/B": {
"name": "B",
"dependencies": [{"owner": "owner", "name": "shared", "version": "1.0.0"}],
},
"owner/shared": {"name": "shared"},
}
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(IDFComponent, "download", fake_download)
# Bare "shared" and "owner/shared" both resolve to canonical owner/shared.
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner or "owner",
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
generate_idf_components(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
)
assert "referenced under multiple names" in caplog.text
def test_generate_idf_components_incompatible_top_level_raises(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
# not be silently dropped.
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(
json.dumps({"name": "A", "platforms": ["espressif8266"]})
)
monkeypatch.setattr(IDFComponent, "download", fake_download)
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner,
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
with pytest.raises(RuntimeError, match="not compatible with ESP-IDF"):
generate_idf_components([Library("esphome/A", "1.0.0", None)])
def test_generate_idf_components_incompatible_dependency_skipped(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# An incompatible *transitive* dependency is skipped (not fatal): A is fine,
# its esp8266-only dep B is dropped and not wired.
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}],
},
"esphome/B": {"name": "B", "platforms": ["espressif8266"]},
}
def fake_download(self, force=False):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(IDFComponent, "download", fake_download)
monkeypatch.setattr(
esphome.espidf.component,
"_resolve_registry_version",
lambda owner, pkgname, requirements: (
owner,
pkgname,
"1.0.0",
f"http://x/{pkgname}.tar.gz",
),
)
top = generate_idf_components([Library("esphome/A", "1.0.0", None)])
assert [c.name for c in top] == ["esphome/A"]
# The incompatible dependency was dropped, not wired in.
assert top[0].dependencies == []