diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index d2dc979966..160c06534e 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -46,10 +46,10 @@ from esphome.const import ( Toolchain, __version__, ) -from esphome.core import CORE, EsphomeError, HexInt, Library +from esphome.core import CORE, EsphomeError, HexInt from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority -from esphome.espidf.component import generate_idf_component +from esphome.espidf.component import generate_idf_components import esphome.final_validate as fv from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed from esphome.types import ConfigType @@ -2598,13 +2598,6 @@ def _write_sdkconfig(): clean_build(clear_pio_cache=False) -def _platformio_library_to_dependency(library: Library) -> tuple[str, dict[str, str]]: - dependency: dict[str, str] = {} - name, _version, path = generate_idf_component(library) - dependency["override_path"] = str(path) - return name, dependency - - def _write_idf_component_yml(): yml_path = CORE.relative_build_path("src/idf_component.yml") dependencies: dict[str, dict] = {} @@ -2678,13 +2671,21 @@ def _write_idf_component_yml(): ) if CORE.using_toolchain_esp_idf: - # Try to convert PlatformIO library to ESP-IDF components - for name, library in CORE.platformio_libraries.items(): + # Convert the PlatformIO libraries to ESP-IDF components as a batch so + # PlatformIO resolves the whole dependency tree at once -- deduplicating + # shared transitive deps (e.g. esphome/libsodium pulled by both noise-c + # and esp_wireguard) to a single version instead of clashing + # override_path entries. + libraries = [ + library + for name, library in CORE.platformio_libraries.items() # Don't process arduino libraries - if name in ARDUINO_DISABLED_LIBRARIES: - continue - dependency_name, dependency = _platformio_library_to_dependency(library) - dependencies[dependency_name] = dependency + if name not in ARDUINO_DISABLED_LIBRARIES + ] + for component in generate_idf_components(libraries): + dependencies[component.get_sanitized_name()] = { + "override_path": str(component.path) + } if CORE.data[KEY_ESP32][KEY_COMPONENTS]: components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS] diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 050002d9e2..7398a91c36 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -1,4 +1,6 @@ +from collections import deque from collections.abc import Callable +from dataclasses import dataclass, field import glob import hashlib import itertools @@ -8,7 +10,7 @@ import os from pathlib import Path import re import tempfile -from typing import TypeVar +from typing import Any, TypeVar from urllib.parse import urlparse, urlsplit, urlunsplit from esphome import git, yaml_util @@ -154,72 +156,6 @@ class IDFComponent: self.path = self.source.download(self.get_sanitized_name(), force=force) -def _get_package_from_pio_registry( - username: str | None, pkgname: str, requirements: str -) -> tuple[str, str, str | None, str | None]: - """ - Fetch package information from PlatformIO registry. - - This function queries the PlatformIO registry to find a library package - that matches the given criteria and returns its metadata including version - and download URL. - - Args: - username: The owner/username of the package (can be None) - pkgname: The name of the package - requirements: Version requirements (e.g., "^1.0.0") - - Returns: - tuple[str, str, str | None, str | None]: - A tuple containing (owner, name, version, download_url) - where version and download_url can be None if not found - """ - - from platformio.package.manager._registry import PackageManagerRegistryMixin - from platformio.package.meta import PackageSpec - - # Create a minimal PackageManagerRegistry class - class PackageManagerRegistry(PackageManagerRegistryMixin): - def __init__(self): - self._registry_client = None - self.pkg_type = "library" - - @staticmethod - def is_system_compatible(value, custom_system=None): - return True - - pio_registry = PackageManagerRegistry() - - # Fetch package metadata from registry - package = pio_registry.fetch_registry_package( - PackageSpec( - owner=username, - name=pkgname, - ) - ) - owner = package["owner"]["username"] - name = package["name"] - - # Find the best matching version based on requirements - version = pio_registry.pick_best_registry_version( - package.get("versions"), - PackageSpec(owner=username, name=pkgname, requirements=requirements), - ) - - # If no version found, return with None for version and URL - if not version: - return owner, name, None, None - - # Find the compatible package file for this version - pkgfile = pio_registry.pick_compatible_pkg_file(version["files"]) - - # If no package file found, return with None for URL but valid version - if not pkgfile: - return owner, name, version["name"], None - - return owner, name, version["name"], pkgfile["download_url"] - - def _apply_extra_script(component: IDFComponent) -> None: """Run a PIO ``extraScript`` and fold its captured env vars into ``component.data["build"]["flags"]`` so the existing -L/-l/-D @@ -339,77 +275,6 @@ def _collect_filtered_files(src_dir: PathType, src_filters: list[str]) -> list[s return [r for r in selected if Path(r).is_file()] -def _convert_library_to_component(library: Library) -> IDFComponent: - """ - Convert a Library object to an IDFComponent object by resolving its metadata. - - This function handles the conversion of library specifications to component - objects, resolving versions through PlatformIO registry when needed or - parsing direct repository URLs. - - Args: - library: The Library object containing name, version, and/or repository information - - Returns: - IDFComponent: The resolved component with name, version, and URL - - Raises: - RuntimeError: If no artifact can be found for the library - """ - name = None - version = None - source = None - - # Repository is provided directly - if library.repository: - # Parse repository URL: path becomes the component name, fragment - # (if any) becomes the git ref stored on GitSource. A missing - # fragment is fine -- clone_or_update leaves the depth-1 clone on - # the remote's default branch, matching PIO's lib_deps behavior - # and external_components handling. - split_result = urlsplit(library.repository) - - # Sanitize name - name = str(split_result.path).strip("/") - name = name.removesuffix(".git") - - # IDF Component Manager only accepts "*", a 40-char commit hash, or - # semver here. The actual git ref is preserved in GitSource.ref; - # override_path makes this field cosmetic at build time. - version = "*" - repository = urlunsplit(split_result._replace(fragment="")) - - ref = split_result.fragment.strip() or None - source = GitSource(str(repository), ref) - - # Version is provided - resolve using PlatformIO registry - elif library.version: - name = library.name - if "/" not in name: - owner, pkgname = None, name - else: - owner, pkgname = name.split("/", 1) - - owner, pkgname, version, url = _get_package_from_pio_registry( - owner, pkgname, library.version - ) - if url is None: - raise RuntimeError( - f"Can't find an pkg file from PlatformIO registry for library {library}" - ) - - name = _owner_pkgname_to_name(owner, pkgname) - source = URLSource(url) - - if source is None: - raise RuntimeError(f"Can't find an artifact associated to library {library}") - - assert name, "Missing library name" - assert version, "Missing library version" - - return IDFComponent(name, version, source) - - def _split_list_by_condition( items: list[str], match_fn: Callable[[str], str | None] ) -> tuple[list[str], list[str]]: @@ -599,8 +464,8 @@ def generate_idf_component_yml(component: IDFComponent) -> str: if "dependencies" not in data: data["dependencies"] = {} - # Every dependency goes through _generate_idf_component → - # component.download() before this runs, so .path is always set. + # Every dependency has been resolved and downloaded before this runs, + # so .path is always set. data["dependencies"][dependency.get_sanitized_name()] = { "override_path": str(dependency.path), } @@ -657,81 +522,6 @@ def _check_library_data(data: dict): ) -def _process_dependencies(component: IDFComponent): - """ - Process library dependencies and generate ESP-IDF components. - - Args: - component: IDFComponent object being processed - - Returns: - None - """ - - name, version = component.name, component.version - dependencies = component.data.get("dependencies") - if not dependencies: - return - - # PIO's library.json accepts both the list-of-dicts form and the - # shorthand dict form ``{"owner/Name": "version_spec"}``. Normalize - # the dict form so the loop below sees a uniform list. Iterating a - # dict gives string keys, which would silently fail the - # ``"name" in dependency`` substring check and skip every entry. - if isinstance(dependencies, dict): - normalized = [] - for raw_name, spec in dependencies.items(): - if "/" in raw_name: - owner, pkgname = raw_name.split("/", 1) - else: - owner, pkgname = None, raw_name - entry = {"name": pkgname, "owner": owner} - if isinstance(spec, dict): - entry.update(spec) - else: - entry["version"] = spec - normalized.append(entry) - dependencies = normalized - - _LOGGER.info("Processing %s@%s component dependencies...", name, version) - for dependency in dependencies: - # Validate dependency structure - if not all(k in dependency for k in ("name", "version")): - _LOGGER.debug("Ignore invalid library: %s", dependency) - continue - - try: - _check_library_data(dependency) - except InvalidIDFComponent as e: - _LOGGER.debug( - "Skip %s@%s: %s", dependency["name"], dependency["version"], str(e) - ) - continue - - # The version field may actually contain a URL - version = dependency["version"] - url = None - try: - result = urlparse(version) - if all([result.scheme, result.netloc]): - url, version = version, None - except (TypeError, ValueError): - pass - - # Generate ESP-IDF component from PlatformIO library - component.dependencies.append( - _generate_idf_component( - Library( - _owner_pkgname_to_name( - dependency.get("owner", None), dependency.get("name") - ), - version, - url, - ) - ) - ) - - def _parse_library_json(library_json_path: PathType): """ Load and parse a JSON file describing a library. @@ -772,92 +562,294 @@ def _parse_library_properties(library_properties_path: PathType): return data -def _generate_idf_component(library: Library, force: bool = False) -> IDFComponent: +def _make_registry_client() -> Any: + """Create a minimal PlatformIO registry client with no system filtering. + + ``is_system_compatible`` is forced True so version selection is driven purely + by the requested version requirements -- ESP-IDF/target compatibility is + handled elsewhere, not by the PlatformIO registry. """ - Generate an ESP-IDF component from a library specification. + from platformio.package.manager._registry import PackageManagerRegistryMixin - This function resolves the library, downloads it, processes metadata files, - and generates necessary ESP-IDF build files (CMakeLists.txt, idf_component.yml). + class _Registry(PackageManagerRegistryMixin): + def __init__(self) -> None: + self._registry_client = None + self.pkg_type = "library" - Args: - library: The library specification containing name, version, and repository URL - force: If True, forces re-download of the library even if it exists locally + @staticmethod + def is_system_compatible(value: Any, custom_system: Any = None) -> bool: + return True - Returns: - IDFComponent: The generated component object with resolved metadata + return _Registry() + + +def _resolve_registry_version( + owner: str | None, pkgname: str, requirements: set[str] +) -> tuple[str, str, str, str]: + """Resolve a registry package to the single highest version satisfying ALL + the given requirements; return ``(owner, name, version, download_url)``. + + Intersecting every requirement (rather than resolving each consumer in + isolation) makes the result independent of processing order and guarantees + no stated constraint is violated -- e.g. ``esphome/libsodium`` requested as + both ``==1.10021.0`` and ``^1.10018.1`` resolves to ``1.10021.0``. """ - _LOGGER.info("Generate IDF component for %s library ...", library) + from platformio.package.meta import PackageSpec - # Resolve component name, version and url - component = _convert_library_to_component(library) - name, version = component.name, component.version + registry = _make_registry_client() + package = registry.fetch_registry_package(PackageSpec(owner=owner, name=pkgname)) + owner = package["owner"]["username"] + name = package["name"] - # Download the library - component.download(force) - - # Paths to component metadata and build files - library_json_path = component.path / "library.json" - library_properties_path = component.path / "library.properties" - cmakelists_txt_path = component.path / "CMakeLists.txt" - idf_component_yml_path = component.path / "idf_component.yml" - - # Bundled CMakeLists.txt / idf_component.yml are ignored -- library - # authors' IDF support is frequently broken (bogus REQUIRES, hard-coded - # arduino-esp32, etc.). We always regenerate. - - if library_json_path.is_file(): - component.data = _parse_library_json(library_json_path) - elif library_properties_path.is_file(): - component.data = _parse_library_properties(library_properties_path) - else: + # Chaining the per-requirement filter intersects all constraints. + versions = package.get("versions") or [] + for requirement in sorted(requirements): + versions = registry.get_compatible_registry_versions( + versions, PackageSpec(owner=owner, name=name, requirements=requirement) + ) + if not versions: raise RuntimeError( - "Invalid PIO library: missing library.json and/or library.properties" + f"No version of {owner}/{name} satisfies all requirements " + f"{sorted(requirements)} requested across the library tree" ) - # Check if the component is usable with ESP-IDF before executing any - # third-party Python from the library (``_apply_extra_script`` below). - _check_library_data(component.data) - - # If the library declares a PIO ``extraScript``, run it against a - # fake SCons env so we can fold its captured LIBPATH/LIBS/etc into - # the build-flag pipeline ``generate_cmakelists_txt`` consumes - # below. Without this, libraries that wire per-MCU archive linking - # via extraScript fail to link under native ESP-IDF. - _apply_extra_script(component) - - # Handle the dependencies (convert PlatformIO library to ESP-IDF component if needed) - _process_dependencies(component) - - _LOGGER.debug("Generating CMakeLists.txt for %s@%s ...", name, version) - write_file_if_changed( - cmakelists_txt_path, - generate_cmakelists_txt(component), - ) - - _LOGGER.debug("Generating idf_component.yml for %s@%s ...", name, version) - write_file_if_changed( - idf_component_yml_path, - generate_idf_component_yml(component), - ) - - return component + best = registry.pick_best_registry_version(versions) + pkgfile = registry.pick_compatible_pkg_file(best["files"]) + if not pkgfile: + raise RuntimeError(f"No package file for {owner}/{name}@{best['name']}") + return owner, name, best["name"], pkgfile["download_url"] -def generate_idf_component( - library: Library, force: bool = False -) -> tuple[str, str, Path]: +def _normalize_dependencies(dependencies: Any) -> list[dict]: + """Normalize a library manifest's ``dependencies`` to a list of dicts. + + PIO's library.json accepts both the list-of-dicts form and the shorthand + dict form (``{"owner/Name": "version_spec"}``); normalize the latter so + callers see a uniform list. """ - Generate an ESP-IDF component and return its name, version, and path. + if not dependencies: + return [] + if isinstance(dependencies, dict): + normalized = [] + for raw_name, spec in dependencies.items(): + if "/" in raw_name: + owner, pkgname = raw_name.split("/", 1) + else: + owner, pkgname = None, raw_name + entry = {"name": pkgname, "owner": owner} + if isinstance(spec, dict): + entry.update(spec) + else: + entry["version"] = spec + normalized.append(entry) + return normalized + return [d for d in dependencies if isinstance(d, dict)] - This is a wrapper function that calls _generate_idf_component and returns - the standardized tuple format (name, version, path). - Args: - library: The library specification containing name, version, and repository URL - force: If True, forces re-download of the library even if it exists locally +@dataclass +class _LibNode: + """A node in the library dependency graph being resolved as a batch.""" - Returns: - tuple[str, str, Path]: A tuple containing (component_name, component_version, component_path) + key: str + is_git: bool + owner: str | None = None + pkgname: str | None = None + requirements: set[str] = field(default_factory=set) + url: str | None = None + ref: str | None = None + edges: set[str] = field(default_factory=set) + + +def _node_key( + name: str | None, version: str | None, repository: str | None +) -> tuple[str, bool, tuple[str | None, str | None]]: + """Return ``(key, is_git, locator)`` for a library or dependency spec. + + The key is derived from the *input* spec (the registry name as written, or + the git URL path), not the resolved canonical name. So a package referenced + inconsistently -- bare ``name`` vs ``owner/name``, or git vs registry -- maps + to distinct keys and isn't deduplicated; ``generate_idf_components`` warns + about that after resolution rather than merging the nodes. """ - component = _generate_idf_component(library, force) - return component.get_sanitized_name(), component.version, component.path + if repository: + split_result = urlsplit(repository) + key = str(split_result.path).strip("/").removesuffix(".git") + ref = split_result.fragment.strip() or None + url = urlunsplit(split_result._replace(fragment="")) + return key, True, (url, ref) + if name and "/" in name: + owner, pkgname = name.split("/", 1) + else: + owner, pkgname = None, name + return name, False, (owner, pkgname) + + +def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: + """Resolve and convert a batch of PlatformIO libraries to IDF components. + + Resolves the whole set together rather than each library independently: it + walks the dependency graph collecting every version *requirement* per + component name, then resolves each name once to a single version satisfying + all of them. So a transitive dependency shared under + different specs (e.g. ``esphome/libsodium``, pulled by both ``noise-c`` and + ``esp_wireguard``) becomes one component instead of two clashing + ``override_path`` entries -- order-independently, and without ever violating + a stated constraint. + + The returned list holds the top-level components (those directly requested); + transitive dependencies are converted too and wired into each component's + generated manifest. + """ + nodes: dict[str, _LibNode] = {} + + def add_spec(name: str | None, version: str | None, repository: str | None) -> str: + key, is_git, locator = _node_key(name, version, repository) + node = nodes.get(key) or _LibNode(key=key, is_git=is_git) + nodes[key] = node + if is_git: + node.is_git = True + node.url, node.ref = locator + else: + node.owner, node.pkgname = locator + if version: + node.requirements.add(version) + return key + + top_level = [ + add_spec(library.name, library.version, library.repository) + for library in libraries + ] + + # Collect + resolve to a fixpoint: a node is (re)resolved whenever its + # requirement set has grown since the last time, so every requirement in the + # graph is accounted for before conversion. + components: dict[str, IDFComponent] = {} + resolved_requirements: dict[str, frozenset[str]] = {} + top_level_keys = set(top_level) + worklist = deque(dict.fromkeys(top_level)) + while worklist: + key = worklist.popleft() + node = nodes[key] + + # A node is queued once per referring edge; skip the (uncached) registry + # lookup + download + dependency walk unless its requirement set grew + # since the last resolve. Requirements only ever grow, so this still + # converges the fixpoint and terminates dependency cycles. + requirements = frozenset(node.requirements) + if resolved_requirements.get(key) == requirements: + continue + resolved_requirements[key] = requirements + + if node.is_git: + component = IDFComponent(key, "*", GitSource(node.url, node.ref)) + else: + owner, name, version, url = _resolve_registry_version( + node.owner, node.pkgname, node.requirements + ) + component = IDFComponent( + _owner_pkgname_to_name(owner, name), version, URLSource(url) + ) + component.download() + + library_json_path = component.path / "library.json" + library_properties_path = component.path / "library.properties" + if library_json_path.is_file(): + component.data = _parse_library_json(library_json_path) + elif library_properties_path.is_file(): + component.data = _parse_library_properties(library_properties_path) + else: + raise RuntimeError( + f"Invalid PIO library {key}: missing library.json and " + "library.properties" + ) + + try: + _check_library_data(component.data) + except InvalidIDFComponent as e: + # Skip an incompatible transitive dependency, but fail fast if a + # top-level library the build explicitly requested is incompatible. + if key in top_level_keys: + raise RuntimeError( + f"Requested library {key} is not compatible with ESP-IDF: {e}" + ) from e + _LOGGER.debug("Skip incompatible dependency %s: %s", key, str(e)) + continue + components[key] = component + + # Requirements changed (we got past the short-circuit above), so + # (re)walk this component's dependencies. + node.edges = set() + for dependency in _normalize_dependencies(component.data.get("dependencies")): + if "name" not in dependency or "version" not in dependency: + continue + try: + _check_library_data(dependency) + except InvalidIDFComponent as e: + _LOGGER.debug("Skip dependency %s: %s", dependency.get("name"), str(e)) + continue + # The version field may actually be a URL (git/archive dependency). + dep_version = dependency["version"] + dep_url = None + try: + parsed = urlparse(dep_version) + if all([parsed.scheme, parsed.netloc]): + dep_url, dep_version = dep_version, None + except (TypeError, ValueError): + pass + dep_key = add_spec( + _owner_pkgname_to_name(dependency.get("owner"), dependency.get("name")), + dep_version, + dep_url, + ) + node.edges.add(dep_key) + worklist.append(dep_key) + + # A git source wins over any registry version requested for the same + # component. That's intentional, but warn so a dropped registry pin isn't a + # silent surprise. + for node in nodes.values(): + if node.is_git and node.requirements: + _LOGGER.warning( + "Library %s is requested both from a git source (%s) and as " + "registry version(s) %s; using the git source.", + node.key, + node.url, + sorted(node.requirements), + ) + + # Two graph nodes that resolve to the same component name (e.g. a package + # referenced both bare and as ``owner/name``) are not deduplicated and can + # produce conflicting component definitions. Warn so it's not silent. + canonical_keys: dict[str, str] = {} + for node_key, component in components.items(): + canonical = component.get_sanitized_name() + if canonical_keys.setdefault(canonical, node_key) != node_key: + _LOGGER.warning( + "Library %s is referenced under multiple names (%s and %s); these " + "are not deduplicated. Reference it consistently as %s.", + canonical, + canonical_keys[canonical], + node_key, + canonical, + ) + + # Wire each component's dependencies to the single resolved instances, then + # regenerate build files. + for key, component in components.items(): + component.dependencies = [ + components[dep_key] + for dep_key in sorted(nodes[key].edges) + if dep_key in components + ] + for component in components.values(): + _apply_extra_script(component) + write_file_if_changed( + component.path / "CMakeLists.txt", + generate_cmakelists_txt(component), + ) + write_file_if_changed( + component.path / "idf_component.yml", + generate_idf_component_yml(component), + ) + + return [components[key] for key in top_level if key in components] diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py index 5f59254aee..4d06fb842a 100644 --- a/esphome/espidf/extra_script.py +++ b/esphome/espidf/extra_script.py @@ -6,8 +6,8 @@ section instead of static fields. The script runs under SCons during PIO's build and mutates the active ``Environment`` (``env.Append``, ``env.Replace``, …) — chiefly to set ``LIBPATH``/``LIBS`` per chip MCU. -ESPHome's PIO→IDF converter (``_generate_idf_component``) doesn't run -SCons, so these scripts were previously ignored and any library +ESPHome's PIO→IDF converter doesn't run SCons, so these scripts were +previously ignored and any library relying on them failed to link under ``toolchain: esp-idf``. This module provides a small shim that ``exec``s an extra-script with a fake ``env`` object, captures the common ``env.Append(...)`` calls, diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 4f0a71053d..602ff03942 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -21,13 +21,15 @@ from esphome.espidf.component import ( URLSource, _check_library_data, _collect_filtered_files, - _convert_library_to_component, + _node_key, + _normalize_dependencies, _parse_library_json, _parse_library_properties, - _process_dependencies, + _resolve_registry_version, _split_list_by_condition, generate_cmakelists_txt, generate_idf_component_yml, + generate_idf_components, ) @@ -162,43 +164,6 @@ def test_generate_cmakelists_txt_references_project_managed_components_variable( assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content -def test_generate_idf_component_overwrites_bundled_files( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - esp32_idf_core: None, -) -> None: - # A library that ships its own CMakeLists.txt + idf_component.yml must - # have both replaced by ESPHome's generated content. Library authors' - # bundled IDF metadata is frequently broken (bogus REQUIRES, hard-coded - # frameworks), so we always regenerate from library.json. - from esphome.espidf.component import _generate_idf_component - - (tmp_path / "src").mkdir() - (tmp_path / "src" / "main.cpp").write_text("// dummy\n") - (tmp_path / "library.json").write_text(json.dumps({"name": "tripwire-lib"})) - (tmp_path / "CMakeLists.txt").write_text("# TRIPWIRE_BUNDLED_CMAKELISTS\n") - (tmp_path / "idf_component.yml").write_text("# TRIPWIRE_BUNDLED_MANIFEST\n") - - fake_component = IDFComponent( - "owner/tripwire-lib", "1.0.0", source=URLSource("http://dummy") - ) - fake_component.path = tmp_path - monkeypatch.setattr( - esphome.espidf.component, - "_convert_library_to_component", - lambda _lib: fake_component, - ) - monkeypatch.setattr(fake_component, "download", lambda force=False: None) - - _generate_idf_component(Library("owner/tripwire-lib", "1.0.0", None)) - - cml = (tmp_path / "CMakeLists.txt").read_text() - manifest = (tmp_path / "idf_component.yml").read_text() - assert "TRIPWIRE_BUNDLED_CMAKELISTS" not in cml - assert "TRIPWIRE_BUNDLED_MANIFEST" not in manifest - assert "idf_component_register" in cml - - def test_generate_idf_component_yml_basic(tmp_component): tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}} result = generate_idf_component_yml(tmp_component) @@ -419,200 +384,58 @@ empty= assert "empty" not in result -def test_convert_library_with_repository(): - lib = Library("name", None, "https://github.com/foo/bar.git#v1.2.3") - - result = _convert_library_to_component(lib) - - assert result.name == "foo/bar" - assert result.version == "*" - assert isinstance(result.source, GitSource) - assert result.source.ref == "v1.2.3" - - -def test_convert_library_with_branch_ref(): - lib = Library("name", None, "https://github.com/foo/bar.git#some-branch") - - result = _convert_library_to_component(lib) - - assert result.name == "foo/bar" - assert result.version == "*" - assert isinstance(result.source, GitSource) - assert result.source.ref == "some-branch" - - -def test_convert_library_missing_ref_uses_default_branch(): - """A bare URL with no #ref clones the remote's default branch. - - Matches PIO's lib_deps behavior and external_components handling -- - git.clone_or_update with ref=None leaves the depth-1 clone on - whatever branch the remote HEAD points at. - """ - lib = Library("name", None, "https://github.com/foo/bar.git") - - result = _convert_library_to_component(lib) - - assert result.name == "foo/bar" - assert result.version == "*" - assert isinstance(result.source, GitSource) - assert result.source.ref is None - - -def test_convert_library_registry(monkeypatch): - lib = Library("foo/bar", "^1.0.0", None) - - monkeypatch.setattr( - esphome.espidf.component, - "_get_package_from_pio_registry", - lambda o, n, r: ("foo", "bar", "1.2.3", "http://example.com/pkg.zip"), +def test_node_key_git_with_ref(): + key, is_git, locator = _node_key( + "name", None, "https://github.com/foo/bar.git#v1.2.3" ) - - result = _convert_library_to_component(lib) - - assert result.name == "foo/bar" - assert result.version == "1.2.3" - assert isinstance(result.source, URLSource) + assert key == "foo/bar" + assert is_git is True + assert locator == ("https://github.com/foo/bar.git", "v1.2.3") -def test_process_dependencies_adds_valid_dependency(tmp_component, monkeypatch): - tmp_component.data = { - "dependencies": [ - { - "name": "foo", - "version": "1.0", - } - ] - } - - monkeypatch.setattr( - esphome.espidf.component, - "_generate_idf_component", - lambda lib: esphome.espidf.component.IDFComponent( - lib.name, lib.version, source=URLSource("http://dummy.com") - ), +def test_node_key_git_branch_ref(): + key, is_git, locator = _node_key( + "name", None, "https://github.com/foo/bar.git#some-branch" ) - - monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) - - _process_dependencies(tmp_component) - - assert len(tmp_component.dependencies) == 1 + assert (key, is_git, locator[1]) == ("foo/bar", True, "some-branch") -def test_process_dependencies_skips_invalid(tmp_component): - tmp_component.data = { - "dependencies": [ - {"name": "foo", "version": "1.0", "platforms": ["arduino"]}, - {"invalid": "entry"}, - ] - } - - _process_dependencies(tmp_component) - - assert tmp_component.dependencies == [] +def test_node_key_git_no_ref(): + _key, is_git, locator = _node_key("name", None, "https://github.com/foo/bar.git") + assert is_git is True + assert locator == ("https://github.com/foo/bar.git", None) -def test_process_dependencies_dict_form(tmp_component, monkeypatch): - """PIO library.json shorthand ``{"owner/Name": "version"}`` is honored. +def test_node_key_registry_owner_name(): + key, is_git, locator = _node_key("foo/bar", "^1.0.0", None) + assert (key, is_git, locator) == ("foo/bar", False, ("foo", "bar")) - Iterating a dict gives string keys, which would silently fail the - ``"name" in dependency`` substring check. Normalize to list-of-dicts - first so the dict form (used by e.g. tesla-ble for its nanopb dep) - is treated the same as the verbose list form. - """ - captured: list[Library] = [] - def fake_generate(library): - captured.append(library) - return IDFComponent( - library.name, library.version, source=URLSource("http://dummy.com") - ) +def test_node_key_registry_bare_name(): + key, is_git, locator = _node_key("bar", "1.0", None) + assert (key, is_git, locator) == ("bar", False, (None, "bar")) - tmp_component.data = { - "dependencies": { - "nanopb/Nanopb": "^0.4.91", - "BareName": "1.2.3", - } - } - monkeypatch.setattr( - esphome.espidf.component, "_generate_idf_component", fake_generate + +def test_normalize_dependencies_none(): + assert _normalize_dependencies(None) == [] + + +def test_normalize_dependencies_list_form(): + deps = [{"name": "foo", "version": "1.0"}] + assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}] + + +def test_normalize_dependencies_dict_form(): + out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"}) + assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out + assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out + + +def test_normalize_dependencies_dict_form_nested_spec(): + out = _normalize_dependencies( + {"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}} ) - monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) - - _process_dependencies(tmp_component) - - assert len(tmp_component.dependencies) == 2 - names = sorted(lib.name for lib in captured) - versions = sorted(lib.version for lib in captured) - assert names == ["BareName", "nanopb/Nanopb"] - assert versions == ["1.2.3", "^0.4.91"] - - -def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch): - """A dict-value that's a URL gets routed to ``repository`` like the list form.""" - captured: list[Library] = [] - - def fake_generate(library): - captured.append(library) - return IDFComponent(library.name, "*", source=URLSource("http://dummy.com")) - - tmp_component.data = { - "dependencies": { - "foo/Bar": "https://github.com/foo/bar.git#main", - } - } - monkeypatch.setattr( - esphome.espidf.component, "_generate_idf_component", fake_generate - ) - monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) - - _process_dependencies(tmp_component) - - assert len(captured) == 1 - assert captured[0].name == "foo/Bar" - assert captured[0].version is None - assert captured[0].repository == "https://github.com/foo/bar.git#main" - - -def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch): - """A dict-value that's itself a dict is merged into the entry. - - PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}`` - for entries that need fields beyond just a version (platforms, - frameworks, etc.). The extra fields flow into _check_library_data - via the entry merge. - """ - captured: list[Library] = [] - checked: list[dict] = [] - - def fake_generate(library): - captured.append(library) - return IDFComponent( - library.name, library.version, source=URLSource("http://dummy.com") - ) - - tmp_component.data = { - "dependencies": { - "nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}, - } - } - monkeypatch.setattr( - esphome.espidf.component, "_generate_idf_component", fake_generate - ) - monkeypatch.setattr( - esphome.espidf.component, - "_check_library_data", - checked.append, - ) - - _process_dependencies(tmp_component) - - assert len(captured) == 1 - assert captured[0].name == "nanopb/Nanopb" - assert captured[0].version == "^0.4.91" - # Extra spec fields reach _check_library_data so platform/framework - # gating still applies. - assert checked == [ + assert out == [ { "name": "Nanopb", "owner": "nanopb", @@ -620,3 +443,364 @@ def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypa "platforms": "espidf", } ] + + +def _patch_registry(monkeypatch, versions): + """Patch the registry client to serve a canned version list (no network). + + Only ``fetch_registry_package`` is faked; the real + ``get_compatible_registry_versions`` / ``pick_best_registry_version`` run on + the canned data so the intersection logic is exercised for real. + """ + registry = esphome.espidf.component._make_registry_client() + monkeypatch.setattr( + registry, + "fetch_registry_package", + lambda spec: { + "owner": {"username": spec.owner or "owner"}, + "name": spec.name, + "versions": [ + {"name": v, "files": [{"download_url": f"http://x/{v}.tar.gz"}]} + for v in versions + ], + }, + ) + monkeypatch.setattr( + esphome.espidf.component, "_make_registry_client", lambda: registry + ) + + +def test_resolve_registry_version_intersects_constraints(monkeypatch): + _patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"]) + owner, name, version, url = _resolve_registry_version( + "esphome", "libsodium", {"==1.10021.0", "^1.10018.1"} + ) + assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0") + assert url == "http://x/1.10021.0.tar.gz" + + +def test_resolve_registry_version_picks_highest_satisfying(monkeypatch): + _patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"]) + _owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"}) + assert version == "1.5.0" + + +def test_resolve_registry_version_conflict_raises(monkeypatch): + _patch_registry(monkeypatch, ["1.0.0", "2.0.0"]) + with pytest.raises(RuntimeError, match="satisfies all requirements"): + _resolve_registry_version("o", "p", {"==1.0.0", "==2.0.0"}) + + +def test_generate_idf_components_dedupes_shared_dependency( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # A and B both depend on shared C under different version specs. The batch + # must resolve C once with BOTH requirements collected, wire a single C + # instance into both, and regenerate (overwrite) each library's build files. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [ + {"owner": "esphome", "name": "C", "version": "==1.10021.0"} + ], + }, + "esphome/B": { + "name": "B", + "dependencies": [ + {"owner": "esphome", "name": "C", "version": "^1.10018.1"} + ], + }, + "esphome/C": {"name": "C"}, + } + + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + (self.path / "CMakeLists.txt").write_text("# TRIPWIRE\n") + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + captured: dict[str, set[str]] = {} + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + captured[f"{owner}/{pkgname}"] = set(requirements) + version = "1.10021.0" if pkgname == "C" else "1.0.0" + return owner, pkgname, version, f"http://x/{pkgname}.tar.gz" + + monkeypatch.setattr( + esphome.espidf.component, "_resolve_registry_version", fake_resolve + ) + + top = generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)] + ) + + # C resolved once (not once per consumer) with BOTH requirements gathered. + assert captured["esphome/C"] == {"==1.10021.0", "^1.10018.1"} + assert resolve_calls.count("C") == 1 + # Top-level components returned in request order. + assert [c.name for c in top] == ["esphome/A", "esphome/B"] + # A and B reference the SAME single C instance (deduped). + a_dep = top[0].dependencies[0] + b_dep = top[1].dependencies[0] + assert a_dep.name == "esphome/C" + assert a_dep is b_dep + # The bundled CMakeLists was overwritten with generated content. + generated = (a_dep.path / "CMakeLists.txt").read_text() + assert "TRIPWIRE" not in generated + assert "idf_component_register" in generated + + +def test_generate_idf_components_handles_dependency_cycle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # A -> B -> A. Must terminate (not recurse forever) and wire the cycle with + # a single instance per component. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}], + }, + "esphome/B": { + "name": "B", + "dependencies": [{"owner": "esphome", "name": "A", "version": "1.0.0"}], + }, + } + + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner, + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + top = generate_idf_components([Library("esphome/A", "1.0.0", None)]) + + assert [c.name for c in top] == ["esphome/A"] + component_a = top[0] + component_b = component_a.dependencies[0] + assert component_b.name == "esphome/B" + # The cycle is wired back to the same A instance, not a duplicate. + assert component_b.dependencies[0] is component_a + + +def test_generate_idf_components_git_overrides_registry_warns( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, + caplog: pytest.LogCaptureFixture, +) -> None: + # A pulls shared as a registry pin; B pulls the same component from a git + # source. The git source wins, but the dropped registry pin must be warned + # about (not silently discarded). + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [ + {"owner": "esphome", "name": "shared", "version": "==1.0.0"} + ], + }, + "esphome/B": { + "name": "B", + "dependencies": [ + { + "owner": "esphome", + "name": "shared", + "version": "https://github.com/esphome/shared.git#main", + } + ], + }, + "esphome/shared": {"name": "shared"}, + } + + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner, + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + top = generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)] + ) + + # shared resolved from the git source (version "*"), not the registry pin. + shared = top[0].dependencies[0] + assert shared.name == "esphome/shared" + assert isinstance(shared.source, GitSource) + assert "using the git source" in caplog.text + assert "==1.0.0" in caplog.text + + +def test_generate_idf_components_missing_manifest_raises( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # A library with neither library.json nor library.properties is invalid; + # fail loudly rather than silently generating build files for it. + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + # no library.json / library.properties written + + monkeypatch.setattr(IDFComponent, "download", fake_download) + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner, + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + with pytest.raises(RuntimeError, match="missing library.json"): + generate_idf_components([Library("esphome/A", "1.0.0", None)]) + + +def test_generate_idf_components_warns_on_noncanonical_duplicate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, + caplog: pytest.LogCaptureFixture, +) -> None: + # A references "shared" (bare) and B references "owner/shared"; both resolve + # to the same canonical name but as distinct graph nodes, so they aren't + # deduplicated -- warn about it. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [{"name": "shared", "version": "1.0.0"}], + }, + "esphome/B": { + "name": "B", + "dependencies": [{"owner": "owner", "name": "shared", "version": "1.0.0"}], + }, + "owner/shared": {"name": "shared"}, + } + + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + # Bare "shared" and "owner/shared" both resolve to canonical owner/shared. + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner or "owner", + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)] + ) + + assert "referenced under multiple names" in caplog.text + + +def test_generate_idf_components_incompatible_top_level_raises( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # A top-level library that isn't ESP-IDF/esp32 compatible must fail fast, + # not be silently dropped. + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "library.json").write_text( + json.dumps({"name": "A", "platforms": ["espressif8266"]}) + ) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner, + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + with pytest.raises(RuntimeError, match="not compatible with ESP-IDF"): + generate_idf_components([Library("esphome/A", "1.0.0", None)]) + + +def test_generate_idf_components_incompatible_dependency_skipped( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # An incompatible *transitive* dependency is skipped (not fatal): A is fine, + # its esp8266-only dep B is dropped and not wired. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}], + }, + "esphome/B": {"name": "B", "platforms": ["espressif8266"]}, + } + + def fake_download(self, force=False): + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + monkeypatch.setattr( + esphome.espidf.component, + "_resolve_registry_version", + lambda owner, pkgname, requirements: ( + owner, + pkgname, + "1.0.0", + f"http://x/{pkgname}.tar.gz", + ), + ) + + top = generate_idf_components([Library("esphome/A", "1.0.0", None)]) + + assert [c.name for c in top] == ["esphome/A"] + # The incompatible dependency was dropped, not wired in. + assert top[0].dependencies == []