[platformio] Extract toolchain-agnostic PlatformIO library converter (#17243)

tests/unit_tests/test_espidf_component.py

@@ -14,23 +14,23 @@ from esphome.const import (
     Platform,
 )
 from esphome.core import CORE, Library
-import esphome.espidf.component
 from esphome.espidf.component import (
+    generate_cmakelists_txt,
+    generate_idf_component_yml,
+    generate_idf_components,
+)
+import esphome.platformio.library
+from esphome.platformio.library import (
+    ConvertedLibrary as IDFComponent,
     GitSource,
-    IDFComponent,
-    InvalidIDFComponent,
     URLSource,
-    _check_library_data,
-    _collect_filtered_files,
     _node_key,
     _normalize_dependencies,
     _parse_library_json,
     _parse_library_properties,
     _resolve_registry_version,
-    _split_list_by_condition,
-    generate_cmakelists_txt,
-    generate_idf_component_yml,
-    generate_idf_components,
+    collect_filtered_files,
+    split_list_by_condition,
 )
 
 
@@ -70,7 +70,7 @@ def test_collect_filtered_files_basic(tmp_path):
     f2.parent.mkdir(parents=True)
     f2.write_text("int b;")
 
-    result = _collect_filtered_files(tmp_path, ["+<*>"])
+    result = collect_filtered_files(tmp_path, ["+<*>"])
     assert str(f1) in result
     assert str(f2) in result
 
@@ -81,7 +81,7 @@ def test_collect_filtered_files_exclude(tmp_path):
     f1.write_text("int a;")
     f2.write_text("int b;")
 
-    result = _collect_filtered_files(tmp_path, ["+<*> -<*.cpp>"])
+    result = collect_filtered_files(tmp_path, ["+<*> -<*.cpp>"])
     assert str(f1) in result
     assert str(f2) not in result
 
@@ -89,7 +89,7 @@ def test_collect_filtered_files_exclude(tmp_path):
 def test_split_list_by_condition():
     items = ["-Iinclude", "-Llib", "-Wall"]
 
-    matched, rest = _split_list_by_condition(
+    matched, rest = split_list_by_condition(
         items, lambda x: x[2:] if x.startswith("-I") else None
     )
 
@@ -202,41 +202,6 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component):
         generate_idf_component_yml(tmp_component)
 
 
-def test_check_library_data_valid(esp32_idf_core):
-    _check_library_data({"platforms": "*", "frameworks": "*"})
-
-
-def test_check_library_data_valid2(esp32_idf_core):
-    _check_library_data({"platforms": "*"})
-
-
-def test_check_library_data_valid3(esp32_idf_core):
-    _check_library_data({})
-
-
-def test_check_library_data_valid4(esp32_idf_core):
-    _check_library_data({"platforms": "espressif32", "frameworks": "*"})
-
-
-def test_check_library_data_valid5(esp32_idf_core):
-    _check_library_data({"platforms": "*", "frameworks": "espidf"})
-
-
-def test_check_library_data_invalid_platform(esp32_idf_core):
-    with pytest.raises(InvalidIDFComponent):
-        _check_library_data({"platforms": ["other"], "frameworks": "*"})
-
-
-def test_check_library_data_invalid_framework(
-    esp32_idf_core: None, caplog: pytest.LogCaptureFixture
-) -> None:
-    # Framework mismatch is a warning, not a hard skip: the library is still
-    # included so that PIO manifests that only list "arduino" (but actually
-    # compile under IDF) can be used without forking them.
-    _check_library_data({"name": "lib", "platforms": "*", "frameworks": ["other"]})
-    assert "do not include 'espidf'" in caplog.text
-
-
 def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
     from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
 
@@ -453,7 +418,7 @@ def _patch_registry(monkeypatch, versions):
     ``get_compatible_registry_versions`` / ``pick_best_registry_version`` run on
     the canned data so the intersection logic is exercised for real.
     """
-    registry = esphome.espidf.component._make_registry_client()
+    registry = esphome.platformio.library._make_registry_client()
     monkeypatch.setattr(
         registry,
         "fetch_registry_package",
@@ -467,7 +432,7 @@ def _patch_registry(monkeypatch, versions):
         },
     )
     monkeypatch.setattr(
-        esphome.espidf.component, "_make_registry_client", lambda: registry
+        esphome.platformio.library, "_make_registry_client", lambda: registry
     )
 
 
@@ -535,7 +500,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
         return owner, pkgname, version, f"http://x/{pkgname}.tar.gz"
 
     monkeypatch.setattr(
-        esphome.espidf.component, "_resolve_registry_version", fake_resolve
+        esphome.platformio.library, "_resolve_registry_version", fake_resolve
     )
 
     top = generate_idf_components(
@@ -594,7 +559,7 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
         return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
 
     monkeypatch.setattr(
-        esphome.espidf.component, "_resolve_registry_version", fake_resolve
+        esphome.platformio.library, "_resolve_registry_version", fake_resolve
     )
     # lib_ignore is read from CORE.platformio_options (stored there by
     # _add_platformio_options); matched by lowercase short name.
@@ -640,7 +605,7 @@ def test_generate_idf_components_handles_dependency_cycle(
 
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner,
@@ -697,7 +662,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
 
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner,
@@ -733,7 +698,7 @@ def test_generate_idf_components_missing_manifest_raises(
 
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner,
@@ -777,7 +742,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     # Bare "shared" and "owner/shared" both resolve to canonical owner/shared.
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner or "owner",
@@ -810,7 +775,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
 
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner,
@@ -820,7 +785,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
         ),
     )
 
-    with pytest.raises(RuntimeError, match="not compatible with ESP-IDF"):
+    with pytest.raises(RuntimeError, match="not compatible with espidf"):
         generate_idf_components([Library("esphome/A", "1.0.0", None)])
 
 
@@ -846,7 +811,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
 
     monkeypatch.setattr(IDFComponent, "download", fake_download)
     monkeypatch.setattr(
-        esphome.espidf.component,
+        esphome.platformio.library,
         "_resolve_registry_version",
         lambda owner, pkgname, requirements: (
             owner,
@@ -892,7 +857,7 @@ def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
         return Path("/cloned"), None
 
     monkeypatch.setattr(
-        esphome.espidf.component.git, "clone_or_update", fake_clone_or_update
+        esphome.platformio.library.git, "clone_or_update", fake_clone_or_update
     )
 
     source = GitSource("https://github.com/esphome/noise-c.git", "v1.0")

tests/unit_tests/test_platformio_library.py

@@ -0,0 +1,229 @@
+"""Tests for the toolchain-agnostic PlatformIO library converter.
+
+Covers the shared download/parse/resolve/dependency-walk paths in
+``esphome.platformio.library`` directly (the ESP-IDF and Zephyr backends are
+exercised in their own test modules)."""
+
+import json
+import logging
+from pathlib import Path
+
+import pytest
+
+from esphome.core import Library
+import esphome.platformio.library as lib
+from esphome.platformio.library import (
+    ConvertedLibrary,
+    GitSource,
+    InvalidLibrary,
+    LibraryBackend,
+    Source,
+    URLSource,
+    _resolve_registry_version,
+    check_library_data,
+    convert_libraries,
+)
+
+
+def _backend(emit=lambda component: None) -> LibraryBackend:
+    return LibraryBackend(platform="espressif32", framework="espidf", emit=emit)
+
+
+def test_check_library_data_accepts_wildcards():
+    check_library_data({"platforms": "*", "frameworks": "*"}, "espressif32", "espidf")
+
+
+def test_check_library_data_accepts_missing_frameworks():
+    check_library_data({"platforms": "*"}, "espressif32", "espidf")
+
+
+def test_check_library_data_accepts_empty_manifest():
+    check_library_data({}, "espressif32", "espidf")
+
+
+def test_check_library_data_accepts_matching_platform():
+    check_library_data(
+        {"platforms": "espressif32", "frameworks": "*"}, "espressif32", "espidf"
+    )
+
+
+def test_check_library_data_accepts_matching_framework():
+    check_library_data(
+        {"platforms": "*", "frameworks": "espidf"}, "espressif32", "espidf"
+    )
+
+
+def test_check_library_data_rejects_unsupported_platform():
+    with pytest.raises(InvalidLibrary):
+        check_library_data(
+            {"platforms": ["other"], "frameworks": "*"}, "espressif32", "espidf"
+        )
+
+
+def test_check_library_data_warns_on_framework_mismatch(
+    caplog: pytest.LogCaptureFixture,
+):
+    # Framework mismatch is a warning, not a hard skip: the library is still
+    # included so manifests that only list "arduino" (but compile fine under the
+    # target framework) can be used without forking them.
+    with caplog.at_level(logging.WARNING, logger="esphome.platformio.library"):
+        check_library_data(
+            {"name": "lib", "platforms": "*", "frameworks": ["other"]},
+            "espressif32",
+            "espidf",
+        )
+    assert "do not include 'espidf'" in caplog.text
+
+
+def test_source_download_not_implemented():
+    with pytest.raises(NotImplementedError):
+        Source().download("x")
+
+
+def test_gitsource_str_includes_ref_when_present():
+    assert str(GitSource("http://git/repo.git", "main")) == "http://git/repo.git#main"
+    assert str(GitSource("http://git/repo.git", None)) == "http://git/repo.git"
+
+
+def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch):
+    monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None)
+    dl_calls: list[list[str]] = []
+    monkeypatch.setattr(
+        lib, "download_from_mirrors", lambda urls, headers, f: dl_calls.append(urls)
+    )
+
+    def fake_extract(fileobj, path):
+        Path(path).mkdir(parents=True, exist_ok=True)
+
+    monkeypatch.setattr(lib, "archive_extract_all", fake_extract)
+
+    src = URLSource("http://example.test/lib.tar.gz")
+    out = src.download("mylib")
+
+    assert (out / ".esphome_extracted").is_file()
+    assert dl_calls == [["http://example.test/lib.tar.gz"]]
+
+    # The completion marker means a second download is skipped (cache hit).
+    out2 = src.download("mylib")
+    assert out2 == out
+    assert len(dl_calls) == 1
+
+
+def test_resolve_registry_version_raises_without_pkg_file(monkeypatch):
+    registry = lib._make_registry_client()
+    monkeypatch.setattr(
+        registry,
+        "fetch_registry_package",
+        lambda spec: {
+            "owner": {"username": spec.owner or "owner"},
+            "name": spec.name,
+            "versions": [{"name": "1.0.0", "files": [{}]}],
+        },
+    )
+    # A best version exists but none of its files is a compatible package.
+    monkeypatch.setattr(
+        registry, "pick_best_registry_version", lambda versions: versions[0]
+    )
+    monkeypatch.setattr(registry, "pick_compatible_pkg_file", lambda files: None)
+    monkeypatch.setattr(lib, "_make_registry_client", lambda: registry)
+
+    with pytest.raises(RuntimeError, match="No package file"):
+        _resolve_registry_version("owner", "pkg", set())
+
+
+def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properties=()):
+    """Fake ConvertedLibrary.download to materialize canned manifests on disk."""
+
+    def fake_download(self, force=False, salt=""):
+        self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
+        self.path.mkdir(parents=True, exist_ok=True)
+        if self.name in properties:
+            (self.path / "library.properties").write_text(manifests[self.name])
+        else:
+            (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
+
+    monkeypatch.setattr(ConvertedLibrary, "download", fake_download)
+    monkeypatch.setattr(
+        lib,
+        "_resolve_registry_version",
+        lambda owner, pkgname, requirements: (
+            owner,
+            pkgname,
+            "1.0.0",
+            f"http://x/{pkgname}.tar.gz",
+        ),
+    )
+
+
+def test_convert_libraries_parses_library_properties(tmp_path, monkeypatch):
+    # A manifest provided as library.properties (Arduino style) instead of
+    # library.json must still be parsed and converted.
+    _patch_download_with_manifests(
+        monkeypatch,
+        tmp_path,
+        {"esphome/A": "name=A\nversion=1.0\n"},
+        properties=("esphome/A",),
+    )
+
+    emitted: list[ConvertedLibrary] = []
+    top = convert_libraries(
+        [Library("esphome/A", "1.0.0", None)], _backend(emitted.append)
+    )
+
+    assert [c.name for c in top] == ["esphome/A"]
+    assert top[0].data["name"] == "A"
+    assert emitted[0].data["version"] == "1.0"
+
+
+def test_convert_libraries_skips_dependency_without_version(tmp_path, monkeypatch):
+    # A dependency entry lacking a version is malformed and silently skipped.
+    _patch_download_with_manifests(
+        monkeypatch,
+        tmp_path,
+        {"esphome/A": {"name": "A", "dependencies": [{"name": "C"}]}},
+    )
+
+    # No version on the top-level spec exercises the "no requirement" path too.
+    top = convert_libraries([Library("esphome/A", None, None)], _backend())
+
+    assert top[0].dependencies == []
+
+
+def test_convert_libraries_handles_unparsable_dependency_version(tmp_path, monkeypatch):
+    # If the git/archive URL probe (urlparse) raises on a malformed value, the
+    # dependency is still kept and treated as a plain version spec.
+    _patch_download_with_manifests(
+        monkeypatch,
+        tmp_path,
+        {
+            "esphome/A": {
+                "name": "A",
+                # An unterminated IPv6 URL makes urlparse raise ValueError.
+                "dependencies": [{"name": "C", "version": "http://[::1"}],
+            },
+            "C": {"name": "C"},
+        },
+    )
+
+    top = convert_libraries([Library("esphome/A", "1.0.0", None)], _backend())
+
+    assert [d.name for d in top[0].dependencies] == ["C"]
+
+
+def test_convert_libraries_skips_incompatible_dependency(tmp_path, monkeypatch):
+    # A dependency that declares an incompatible platform is skipped (the
+    # top-level library still builds).
+    _patch_download_with_manifests(
+        monkeypatch,
+        tmp_path,
+        {
+            "esphome/A": {
+                "name": "A",
+                "dependencies": [{"name": "C", "version": "1.0", "platforms": ["avr"]}],
+            }
+        },
+    )
+
+    top = convert_libraries([Library("esphome/A", "1.0.0", None)], _backend())
+
+    assert top[0].dependencies == []