mirror of
https://github.com/esphome/esphome.git
synced 2026-10-05 18:41:32 +00:00
[esp32] Deduplicate PlatformIO library conversion by resolving the batch together (#16756)
This commit is contained in:
@@ -46,10 +46,10 @@ from esphome.const import (
|
|||||||
Toolchain,
|
Toolchain,
|
||||||
__version__,
|
__version__,
|
||||||
)
|
)
|
||||||
from esphome.core import CORE, EsphomeError, HexInt, Library
|
from esphome.core import CORE, EsphomeError, HexInt
|
||||||
from esphome.core.config import BOARD_MAX_LENGTH
|
from esphome.core.config import BOARD_MAX_LENGTH
|
||||||
from esphome.coroutine import CoroPriority, coroutine_with_priority
|
from esphome.coroutine import CoroPriority, coroutine_with_priority
|
||||||
from esphome.espidf.component import generate_idf_component
|
from esphome.espidf.component import generate_idf_components
|
||||||
import esphome.final_validate as fv
|
import esphome.final_validate as fv
|
||||||
from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed
|
from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed
|
||||||
from esphome.types import ConfigType
|
from esphome.types import ConfigType
|
||||||
@@ -2598,13 +2598,6 @@ def _write_sdkconfig():
|
|||||||
clean_build(clear_pio_cache=False)
|
clean_build(clear_pio_cache=False)
|
||||||
|
|
||||||
|
|
||||||
def _platformio_library_to_dependency(library: Library) -> tuple[str, dict[str, str]]:
|
|
||||||
dependency: dict[str, str] = {}
|
|
||||||
name, _version, path = generate_idf_component(library)
|
|
||||||
dependency["override_path"] = str(path)
|
|
||||||
return name, dependency
|
|
||||||
|
|
||||||
|
|
||||||
def _write_idf_component_yml():
|
def _write_idf_component_yml():
|
||||||
yml_path = CORE.relative_build_path("src/idf_component.yml")
|
yml_path = CORE.relative_build_path("src/idf_component.yml")
|
||||||
dependencies: dict[str, dict] = {}
|
dependencies: dict[str, dict] = {}
|
||||||
@@ -2678,13 +2671,21 @@ def _write_idf_component_yml():
|
|||||||
)
|
)
|
||||||
|
|
||||||
if CORE.using_toolchain_esp_idf:
|
if CORE.using_toolchain_esp_idf:
|
||||||
# Try to convert PlatformIO library to ESP-IDF components
|
# Convert the PlatformIO libraries to ESP-IDF components as a batch so
|
||||||
for name, library in CORE.platformio_libraries.items():
|
# PlatformIO resolves the whole dependency tree at once -- deduplicating
|
||||||
|
# shared transitive deps (e.g. esphome/libsodium pulled by both noise-c
|
||||||
|
# and esp_wireguard) to a single version instead of clashing
|
||||||
|
# override_path entries.
|
||||||
|
libraries = [
|
||||||
|
library
|
||||||
|
for name, library in CORE.platformio_libraries.items()
|
||||||
# Don't process arduino libraries
|
# Don't process arduino libraries
|
||||||
if name in ARDUINO_DISABLED_LIBRARIES:
|
if name not in ARDUINO_DISABLED_LIBRARIES
|
||||||
continue
|
]
|
||||||
dependency_name, dependency = _platformio_library_to_dependency(library)
|
for component in generate_idf_components(libraries):
|
||||||
dependencies[dependency_name] = dependency
|
dependencies[component.get_sanitized_name()] = {
|
||||||
|
"override_path": str(component.path)
|
||||||
|
}
|
||||||
|
|
||||||
if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
|
if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
|
||||||
components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]
|
components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]
|
||||||
|
|||||||
+279
-287
@@ -1,4 +1,6 @@
|
|||||||
|
from collections import deque
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass, field
|
||||||
import glob
|
import glob
|
||||||
import hashlib
|
import hashlib
|
||||||
import itertools
|
import itertools
|
||||||
@@ -8,7 +10,7 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
from typing import TypeVar
|
from typing import Any, TypeVar
|
||||||
from urllib.parse import urlparse, urlsplit, urlunsplit
|
from urllib.parse import urlparse, urlsplit, urlunsplit
|
||||||
|
|
||||||
from esphome import git, yaml_util
|
from esphome import git, yaml_util
|
||||||
@@ -154,72 +156,6 @@ class IDFComponent:
|
|||||||
self.path = self.source.download(self.get_sanitized_name(), force=force)
|
self.path = self.source.download(self.get_sanitized_name(), force=force)
|
||||||
|
|
||||||
|
|
||||||
def _get_package_from_pio_registry(
|
|
||||||
username: str | None, pkgname: str, requirements: str
|
|
||||||
) -> tuple[str, str, str | None, str | None]:
|
|
||||||
"""
|
|
||||||
Fetch package information from PlatformIO registry.
|
|
||||||
|
|
||||||
This function queries the PlatformIO registry to find a library package
|
|
||||||
that matches the given criteria and returns its metadata including version
|
|
||||||
and download URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
username: The owner/username of the package (can be None)
|
|
||||||
pkgname: The name of the package
|
|
||||||
requirements: Version requirements (e.g., "^1.0.0")
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple[str, str, str | None, str | None]:
|
|
||||||
A tuple containing (owner, name, version, download_url)
|
|
||||||
where version and download_url can be None if not found
|
|
||||||
"""
|
|
||||||
|
|
||||||
from platformio.package.manager._registry import PackageManagerRegistryMixin
|
|
||||||
from platformio.package.meta import PackageSpec
|
|
||||||
|
|
||||||
# Create a minimal PackageManagerRegistry class
|
|
||||||
class PackageManagerRegistry(PackageManagerRegistryMixin):
|
|
||||||
def __init__(self):
|
|
||||||
self._registry_client = None
|
|
||||||
self.pkg_type = "library"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_system_compatible(value, custom_system=None):
|
|
||||||
return True
|
|
||||||
|
|
||||||
pio_registry = PackageManagerRegistry()
|
|
||||||
|
|
||||||
# Fetch package metadata from registry
|
|
||||||
package = pio_registry.fetch_registry_package(
|
|
||||||
PackageSpec(
|
|
||||||
owner=username,
|
|
||||||
name=pkgname,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
owner = package["owner"]["username"]
|
|
||||||
name = package["name"]
|
|
||||||
|
|
||||||
# Find the best matching version based on requirements
|
|
||||||
version = pio_registry.pick_best_registry_version(
|
|
||||||
package.get("versions"),
|
|
||||||
PackageSpec(owner=username, name=pkgname, requirements=requirements),
|
|
||||||
)
|
|
||||||
|
|
||||||
# If no version found, return with None for version and URL
|
|
||||||
if not version:
|
|
||||||
return owner, name, None, None
|
|
||||||
|
|
||||||
# Find the compatible package file for this version
|
|
||||||
pkgfile = pio_registry.pick_compatible_pkg_file(version["files"])
|
|
||||||
|
|
||||||
# If no package file found, return with None for URL but valid version
|
|
||||||
if not pkgfile:
|
|
||||||
return owner, name, version["name"], None
|
|
||||||
|
|
||||||
return owner, name, version["name"], pkgfile["download_url"]
|
|
||||||
|
|
||||||
|
|
||||||
def _apply_extra_script(component: IDFComponent) -> None:
|
def _apply_extra_script(component: IDFComponent) -> None:
|
||||||
"""Run a PIO ``extraScript`` and fold its captured env vars into
|
"""Run a PIO ``extraScript`` and fold its captured env vars into
|
||||||
``component.data["build"]["flags"]`` so the existing -L/-l/-D
|
``component.data["build"]["flags"]`` so the existing -L/-l/-D
|
||||||
@@ -339,77 +275,6 @@ def _collect_filtered_files(src_dir: PathType, src_filters: list[str]) -> list[s
|
|||||||
return [r for r in selected if Path(r).is_file()]
|
return [r for r in selected if Path(r).is_file()]
|
||||||
|
|
||||||
|
|
||||||
def _convert_library_to_component(library: Library) -> IDFComponent:
|
|
||||||
"""
|
|
||||||
Convert a Library object to an IDFComponent object by resolving its metadata.
|
|
||||||
|
|
||||||
This function handles the conversion of library specifications to component
|
|
||||||
objects, resolving versions through PlatformIO registry when needed or
|
|
||||||
parsing direct repository URLs.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
library: The Library object containing name, version, and/or repository information
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
IDFComponent: The resolved component with name, version, and URL
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
RuntimeError: If no artifact can be found for the library
|
|
||||||
"""
|
|
||||||
name = None
|
|
||||||
version = None
|
|
||||||
source = None
|
|
||||||
|
|
||||||
# Repository is provided directly
|
|
||||||
if library.repository:
|
|
||||||
# Parse repository URL: path becomes the component name, fragment
|
|
||||||
# (if any) becomes the git ref stored on GitSource. A missing
|
|
||||||
# fragment is fine -- clone_or_update leaves the depth-1 clone on
|
|
||||||
# the remote's default branch, matching PIO's lib_deps behavior
|
|
||||||
# and external_components handling.
|
|
||||||
split_result = urlsplit(library.repository)
|
|
||||||
|
|
||||||
# Sanitize name
|
|
||||||
name = str(split_result.path).strip("/")
|
|
||||||
name = name.removesuffix(".git")
|
|
||||||
|
|
||||||
# IDF Component Manager only accepts "*", a 40-char commit hash, or
|
|
||||||
# semver here. The actual git ref is preserved in GitSource.ref;
|
|
||||||
# override_path makes this field cosmetic at build time.
|
|
||||||
version = "*"
|
|
||||||
repository = urlunsplit(split_result._replace(fragment=""))
|
|
||||||
|
|
||||||
ref = split_result.fragment.strip() or None
|
|
||||||
source = GitSource(str(repository), ref)
|
|
||||||
|
|
||||||
# Version is provided - resolve using PlatformIO registry
|
|
||||||
elif library.version:
|
|
||||||
name = library.name
|
|
||||||
if "/" not in name:
|
|
||||||
owner, pkgname = None, name
|
|
||||||
else:
|
|
||||||
owner, pkgname = name.split("/", 1)
|
|
||||||
|
|
||||||
owner, pkgname, version, url = _get_package_from_pio_registry(
|
|
||||||
owner, pkgname, library.version
|
|
||||||
)
|
|
||||||
if url is None:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Can't find an pkg file from PlatformIO registry for library {library}"
|
|
||||||
)
|
|
||||||
|
|
||||||
name = _owner_pkgname_to_name(owner, pkgname)
|
|
||||||
source = URLSource(url)
|
|
||||||
|
|
||||||
if source is None:
|
|
||||||
raise RuntimeError(f"Can't find an artifact associated to library {library}")
|
|
||||||
|
|
||||||
assert name, "Missing library name"
|
|
||||||
assert version, "Missing library version"
|
|
||||||
|
|
||||||
return IDFComponent(name, version, source)
|
|
||||||
|
|
||||||
|
|
||||||
def _split_list_by_condition(
|
def _split_list_by_condition(
|
||||||
items: list[str], match_fn: Callable[[str], str | None]
|
items: list[str], match_fn: Callable[[str], str | None]
|
||||||
) -> tuple[list[str], list[str]]:
|
) -> tuple[list[str], list[str]]:
|
||||||
@@ -599,8 +464,8 @@ def generate_idf_component_yml(component: IDFComponent) -> str:
|
|||||||
if "dependencies" not in data:
|
if "dependencies" not in data:
|
||||||
data["dependencies"] = {}
|
data["dependencies"] = {}
|
||||||
|
|
||||||
# Every dependency goes through _generate_idf_component →
|
# Every dependency has been resolved and downloaded before this runs,
|
||||||
# component.download() before this runs, so .path is always set.
|
# so .path is always set.
|
||||||
data["dependencies"][dependency.get_sanitized_name()] = {
|
data["dependencies"][dependency.get_sanitized_name()] = {
|
||||||
"override_path": str(dependency.path),
|
"override_path": str(dependency.path),
|
||||||
}
|
}
|
||||||
@@ -657,81 +522,6 @@ def _check_library_data(data: dict):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _process_dependencies(component: IDFComponent):
|
|
||||||
"""
|
|
||||||
Process library dependencies and generate ESP-IDF components.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
component: IDFComponent object being processed
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
"""
|
|
||||||
|
|
||||||
name, version = component.name, component.version
|
|
||||||
dependencies = component.data.get("dependencies")
|
|
||||||
if not dependencies:
|
|
||||||
return
|
|
||||||
|
|
||||||
# PIO's library.json accepts both the list-of-dicts form and the
|
|
||||||
# shorthand dict form ``{"owner/Name": "version_spec"}``. Normalize
|
|
||||||
# the dict form so the loop below sees a uniform list. Iterating a
|
|
||||||
# dict gives string keys, which would silently fail the
|
|
||||||
# ``"name" in dependency`` substring check and skip every entry.
|
|
||||||
if isinstance(dependencies, dict):
|
|
||||||
normalized = []
|
|
||||||
for raw_name, spec in dependencies.items():
|
|
||||||
if "/" in raw_name:
|
|
||||||
owner, pkgname = raw_name.split("/", 1)
|
|
||||||
else:
|
|
||||||
owner, pkgname = None, raw_name
|
|
||||||
entry = {"name": pkgname, "owner": owner}
|
|
||||||
if isinstance(spec, dict):
|
|
||||||
entry.update(spec)
|
|
||||||
else:
|
|
||||||
entry["version"] = spec
|
|
||||||
normalized.append(entry)
|
|
||||||
dependencies = normalized
|
|
||||||
|
|
||||||
_LOGGER.info("Processing %s@%s component dependencies...", name, version)
|
|
||||||
for dependency in dependencies:
|
|
||||||
# Validate dependency structure
|
|
||||||
if not all(k in dependency for k in ("name", "version")):
|
|
||||||
_LOGGER.debug("Ignore invalid library: %s", dependency)
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
_check_library_data(dependency)
|
|
||||||
except InvalidIDFComponent as e:
|
|
||||||
_LOGGER.debug(
|
|
||||||
"Skip %s@%s: %s", dependency["name"], dependency["version"], str(e)
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# The version field may actually contain a URL
|
|
||||||
version = dependency["version"]
|
|
||||||
url = None
|
|
||||||
try:
|
|
||||||
result = urlparse(version)
|
|
||||||
if all([result.scheme, result.netloc]):
|
|
||||||
url, version = version, None
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Generate ESP-IDF component from PlatformIO library
|
|
||||||
component.dependencies.append(
|
|
||||||
_generate_idf_component(
|
|
||||||
Library(
|
|
||||||
_owner_pkgname_to_name(
|
|
||||||
dependency.get("owner", None), dependency.get("name")
|
|
||||||
),
|
|
||||||
version,
|
|
||||||
url,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_library_json(library_json_path: PathType):
|
def _parse_library_json(library_json_path: PathType):
|
||||||
"""
|
"""
|
||||||
Load and parse a JSON file describing a library.
|
Load and parse a JSON file describing a library.
|
||||||
@@ -772,92 +562,294 @@ def _parse_library_properties(library_properties_path: PathType):
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def _generate_idf_component(library: Library, force: bool = False) -> IDFComponent:
|
def _make_registry_client() -> Any:
|
||||||
|
"""Create a minimal PlatformIO registry client with no system filtering.
|
||||||
|
|
||||||
|
``is_system_compatible`` is forced True so version selection is driven purely
|
||||||
|
by the requested version requirements -- ESP-IDF/target compatibility is
|
||||||
|
handled elsewhere, not by the PlatformIO registry.
|
||||||
"""
|
"""
|
||||||
Generate an ESP-IDF component from a library specification.
|
from platformio.package.manager._registry import PackageManagerRegistryMixin
|
||||||
|
|
||||||
This function resolves the library, downloads it, processes metadata files,
|
class _Registry(PackageManagerRegistryMixin):
|
||||||
and generates necessary ESP-IDF build files (CMakeLists.txt, idf_component.yml).
|
def __init__(self) -> None:
|
||||||
|
self._registry_client = None
|
||||||
|
self.pkg_type = "library"
|
||||||
|
|
||||||
Args:
|
@staticmethod
|
||||||
library: The library specification containing name, version, and repository URL
|
def is_system_compatible(value: Any, custom_system: Any = None) -> bool:
|
||||||
force: If True, forces re-download of the library even if it exists locally
|
return True
|
||||||
|
|
||||||
Returns:
|
return _Registry()
|
||||||
IDFComponent: The generated component object with resolved metadata
|
|
||||||
|
|
||||||
|
def _resolve_registry_version(
|
||||||
|
owner: str | None, pkgname: str, requirements: set[str]
|
||||||
|
) -> tuple[str, str, str, str]:
|
||||||
|
"""Resolve a registry package to the single highest version satisfying ALL
|
||||||
|
the given requirements; return ``(owner, name, version, download_url)``.
|
||||||
|
|
||||||
|
Intersecting every requirement (rather than resolving each consumer in
|
||||||
|
isolation) makes the result independent of processing order and guarantees
|
||||||
|
no stated constraint is violated -- e.g. ``esphome/libsodium`` requested as
|
||||||
|
both ``==1.10021.0`` and ``^1.10018.1`` resolves to ``1.10021.0``.
|
||||||
"""
|
"""
|
||||||
_LOGGER.info("Generate IDF component for %s library ...", library)
|
from platformio.package.meta import PackageSpec
|
||||||
|
|
||||||
# Resolve component name, version and url
|
registry = _make_registry_client()
|
||||||
component = _convert_library_to_component(library)
|
package = registry.fetch_registry_package(PackageSpec(owner=owner, name=pkgname))
|
||||||
name, version = component.name, component.version
|
owner = package["owner"]["username"]
|
||||||
|
name = package["name"]
|
||||||
|
|
||||||
# Download the library
|
# Chaining the per-requirement filter intersects all constraints.
|
||||||
component.download(force)
|
versions = package.get("versions") or []
|
||||||
|
for requirement in sorted(requirements):
|
||||||
# Paths to component metadata and build files
|
versions = registry.get_compatible_registry_versions(
|
||||||
library_json_path = component.path / "library.json"
|
versions, PackageSpec(owner=owner, name=name, requirements=requirement)
|
||||||
library_properties_path = component.path / "library.properties"
|
)
|
||||||
cmakelists_txt_path = component.path / "CMakeLists.txt"
|
if not versions:
|
||||||
idf_component_yml_path = component.path / "idf_component.yml"
|
|
||||||
|
|
||||||
# Bundled CMakeLists.txt / idf_component.yml are ignored -- library
|
|
||||||
# authors' IDF support is frequently broken (bogus REQUIRES, hard-coded
|
|
||||||
# arduino-esp32, etc.). We always regenerate.
|
|
||||||
|
|
||||||
if library_json_path.is_file():
|
|
||||||
component.data = _parse_library_json(library_json_path)
|
|
||||||
elif library_properties_path.is_file():
|
|
||||||
component.data = _parse_library_properties(library_properties_path)
|
|
||||||
else:
|
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"Invalid PIO library: missing library.json and/or library.properties"
|
f"No version of {owner}/{name} satisfies all requirements "
|
||||||
|
f"{sorted(requirements)} requested across the library tree"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check if the component is usable with ESP-IDF before executing any
|
best = registry.pick_best_registry_version(versions)
|
||||||
# third-party Python from the library (``_apply_extra_script`` below).
|
pkgfile = registry.pick_compatible_pkg_file(best["files"])
|
||||||
_check_library_data(component.data)
|
if not pkgfile:
|
||||||
|
raise RuntimeError(f"No package file for {owner}/{name}@{best['name']}")
|
||||||
# If the library declares a PIO ``extraScript``, run it against a
|
return owner, name, best["name"], pkgfile["download_url"]
|
||||||
# fake SCons env so we can fold its captured LIBPATH/LIBS/etc into
|
|
||||||
# the build-flag pipeline ``generate_cmakelists_txt`` consumes
|
|
||||||
# below. Without this, libraries that wire per-MCU archive linking
|
|
||||||
# via extraScript fail to link under native ESP-IDF.
|
|
||||||
_apply_extra_script(component)
|
|
||||||
|
|
||||||
# Handle the dependencies (convert PlatformIO library to ESP-IDF component if needed)
|
|
||||||
_process_dependencies(component)
|
|
||||||
|
|
||||||
_LOGGER.debug("Generating CMakeLists.txt for %s@%s ...", name, version)
|
|
||||||
write_file_if_changed(
|
|
||||||
cmakelists_txt_path,
|
|
||||||
generate_cmakelists_txt(component),
|
|
||||||
)
|
|
||||||
|
|
||||||
_LOGGER.debug("Generating idf_component.yml for %s@%s ...", name, version)
|
|
||||||
write_file_if_changed(
|
|
||||||
idf_component_yml_path,
|
|
||||||
generate_idf_component_yml(component),
|
|
||||||
)
|
|
||||||
|
|
||||||
return component
|
|
||||||
|
|
||||||
|
|
||||||
def generate_idf_component(
|
def _normalize_dependencies(dependencies: Any) -> list[dict]:
|
||||||
library: Library, force: bool = False
|
"""Normalize a library manifest's ``dependencies`` to a list of dicts.
|
||||||
) -> tuple[str, str, Path]:
|
|
||||||
|
PIO's library.json accepts both the list-of-dicts form and the shorthand
|
||||||
|
dict form (``{"owner/Name": "version_spec"}``); normalize the latter so
|
||||||
|
callers see a uniform list.
|
||||||
"""
|
"""
|
||||||
Generate an ESP-IDF component and return its name, version, and path.
|
if not dependencies:
|
||||||
|
return []
|
||||||
|
if isinstance(dependencies, dict):
|
||||||
|
normalized = []
|
||||||
|
for raw_name, spec in dependencies.items():
|
||||||
|
if "/" in raw_name:
|
||||||
|
owner, pkgname = raw_name.split("/", 1)
|
||||||
|
else:
|
||||||
|
owner, pkgname = None, raw_name
|
||||||
|
entry = {"name": pkgname, "owner": owner}
|
||||||
|
if isinstance(spec, dict):
|
||||||
|
entry.update(spec)
|
||||||
|
else:
|
||||||
|
entry["version"] = spec
|
||||||
|
normalized.append(entry)
|
||||||
|
return normalized
|
||||||
|
return [d for d in dependencies if isinstance(d, dict)]
|
||||||
|
|
||||||
This is a wrapper function that calls _generate_idf_component and returns
|
|
||||||
the standardized tuple format (name, version, path).
|
|
||||||
|
|
||||||
Args:
|
@dataclass
|
||||||
library: The library specification containing name, version, and repository URL
|
class _LibNode:
|
||||||
force: If True, forces re-download of the library even if it exists locally
|
"""A node in the library dependency graph being resolved as a batch."""
|
||||||
|
|
||||||
Returns:
|
key: str
|
||||||
tuple[str, str, Path]: A tuple containing (component_name, component_version, component_path)
|
is_git: bool
|
||||||
|
owner: str | None = None
|
||||||
|
pkgname: str | None = None
|
||||||
|
requirements: set[str] = field(default_factory=set)
|
||||||
|
url: str | None = None
|
||||||
|
ref: str | None = None
|
||||||
|
edges: set[str] = field(default_factory=set)
|
||||||
|
|
||||||
|
|
||||||
|
def _node_key(
|
||||||
|
name: str | None, version: str | None, repository: str | None
|
||||||
|
) -> tuple[str, bool, tuple[str | None, str | None]]:
|
||||||
|
"""Return ``(key, is_git, locator)`` for a library or dependency spec.
|
||||||
|
|
||||||
|
The key is derived from the *input* spec (the registry name as written, or
|
||||||
|
the git URL path), not the resolved canonical name. So a package referenced
|
||||||
|
inconsistently -- bare ``name`` vs ``owner/name``, or git vs registry -- maps
|
||||||
|
to distinct keys and isn't deduplicated; ``generate_idf_components`` warns
|
||||||
|
about that after resolution rather than merging the nodes.
|
||||||
"""
|
"""
|
||||||
component = _generate_idf_component(library, force)
|
if repository:
|
||||||
return component.get_sanitized_name(), component.version, component.path
|
split_result = urlsplit(repository)
|
||||||
|
key = str(split_result.path).strip("/").removesuffix(".git")
|
||||||
|
ref = split_result.fragment.strip() or None
|
||||||
|
url = urlunsplit(split_result._replace(fragment=""))
|
||||||
|
return key, True, (url, ref)
|
||||||
|
if name and "/" in name:
|
||||||
|
owner, pkgname = name.split("/", 1)
|
||||||
|
else:
|
||||||
|
owner, pkgname = None, name
|
||||||
|
return name, False, (owner, pkgname)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
|
||||||
|
"""Resolve and convert a batch of PlatformIO libraries to IDF components.
|
||||||
|
|
||||||
|
Resolves the whole set together rather than each library independently: it
|
||||||
|
walks the dependency graph collecting every version *requirement* per
|
||||||
|
component name, then resolves each name once to a single version satisfying
|
||||||
|
all of them. So a transitive dependency shared under
|
||||||
|
different specs (e.g. ``esphome/libsodium``, pulled by both ``noise-c`` and
|
||||||
|
``esp_wireguard``) becomes one component instead of two clashing
|
||||||
|
``override_path`` entries -- order-independently, and without ever violating
|
||||||
|
a stated constraint.
|
||||||
|
|
||||||
|
The returned list holds the top-level components (those directly requested);
|
||||||
|
transitive dependencies are converted too and wired into each component's
|
||||||
|
generated manifest.
|
||||||
|
"""
|
||||||
|
nodes: dict[str, _LibNode] = {}
|
||||||
|
|
||||||
|
def add_spec(name: str | None, version: str | None, repository: str | None) -> str:
|
||||||
|
key, is_git, locator = _node_key(name, version, repository)
|
||||||
|
node = nodes.get(key) or _LibNode(key=key, is_git=is_git)
|
||||||
|
nodes[key] = node
|
||||||
|
if is_git:
|
||||||
|
node.is_git = True
|
||||||
|
node.url, node.ref = locator
|
||||||
|
else:
|
||||||
|
node.owner, node.pkgname = locator
|
||||||
|
if version:
|
||||||
|
node.requirements.add(version)
|
||||||
|
return key
|
||||||
|
|
||||||
|
top_level = [
|
||||||
|
add_spec(library.name, library.version, library.repository)
|
||||||
|
for library in libraries
|
||||||
|
]
|
||||||
|
|
||||||
|
# Collect + resolve to a fixpoint: a node is (re)resolved whenever its
|
||||||
|
# requirement set has grown since the last time, so every requirement in the
|
||||||
|
# graph is accounted for before conversion.
|
||||||
|
components: dict[str, IDFComponent] = {}
|
||||||
|
resolved_requirements: dict[str, frozenset[str]] = {}
|
||||||
|
top_level_keys = set(top_level)
|
||||||
|
worklist = deque(dict.fromkeys(top_level))
|
||||||
|
while worklist:
|
||||||
|
key = worklist.popleft()
|
||||||
|
node = nodes[key]
|
||||||
|
|
||||||
|
# A node is queued once per referring edge; skip the (uncached) registry
|
||||||
|
# lookup + download + dependency walk unless its requirement set grew
|
||||||
|
# since the last resolve. Requirements only ever grow, so this still
|
||||||
|
# converges the fixpoint and terminates dependency cycles.
|
||||||
|
requirements = frozenset(node.requirements)
|
||||||
|
if resolved_requirements.get(key) == requirements:
|
||||||
|
continue
|
||||||
|
resolved_requirements[key] = requirements
|
||||||
|
|
||||||
|
if node.is_git:
|
||||||
|
component = IDFComponent(key, "*", GitSource(node.url, node.ref))
|
||||||
|
else:
|
||||||
|
owner, name, version, url = _resolve_registry_version(
|
||||||
|
node.owner, node.pkgname, node.requirements
|
||||||
|
)
|
||||||
|
component = IDFComponent(
|
||||||
|
_owner_pkgname_to_name(owner, name), version, URLSource(url)
|
||||||
|
)
|
||||||
|
component.download()
|
||||||
|
|
||||||
|
library_json_path = component.path / "library.json"
|
||||||
|
library_properties_path = component.path / "library.properties"
|
||||||
|
if library_json_path.is_file():
|
||||||
|
component.data = _parse_library_json(library_json_path)
|
||||||
|
elif library_properties_path.is_file():
|
||||||
|
component.data = _parse_library_properties(library_properties_path)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Invalid PIO library {key}: missing library.json and "
|
||||||
|
"library.properties"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
_check_library_data(component.data)
|
||||||
|
except InvalidIDFComponent as e:
|
||||||
|
# Skip an incompatible transitive dependency, but fail fast if a
|
||||||
|
# top-level library the build explicitly requested is incompatible.
|
||||||
|
if key in top_level_keys:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Requested library {key} is not compatible with ESP-IDF: {e}"
|
||||||
|
) from e
|
||||||
|
_LOGGER.debug("Skip incompatible dependency %s: %s", key, str(e))
|
||||||
|
continue
|
||||||
|
components[key] = component
|
||||||
|
|
||||||
|
# Requirements changed (we got past the short-circuit above), so
|
||||||
|
# (re)walk this component's dependencies.
|
||||||
|
node.edges = set()
|
||||||
|
for dependency in _normalize_dependencies(component.data.get("dependencies")):
|
||||||
|
if "name" not in dependency or "version" not in dependency:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
_check_library_data(dependency)
|
||||||
|
except InvalidIDFComponent as e:
|
||||||
|
_LOGGER.debug("Skip dependency %s: %s", dependency.get("name"), str(e))
|
||||||
|
continue
|
||||||
|
# The version field may actually be a URL (git/archive dependency).
|
||||||
|
dep_version = dependency["version"]
|
||||||
|
dep_url = None
|
||||||
|
try:
|
||||||
|
parsed = urlparse(dep_version)
|
||||||
|
if all([parsed.scheme, parsed.netloc]):
|
||||||
|
dep_url, dep_version = dep_version, None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
dep_key = add_spec(
|
||||||
|
_owner_pkgname_to_name(dependency.get("owner"), dependency.get("name")),
|
||||||
|
dep_version,
|
||||||
|
dep_url,
|
||||||
|
)
|
||||||
|
node.edges.add(dep_key)
|
||||||
|
worklist.append(dep_key)
|
||||||
|
|
||||||
|
# A git source wins over any registry version requested for the same
|
||||||
|
# component. That's intentional, but warn so a dropped registry pin isn't a
|
||||||
|
# silent surprise.
|
||||||
|
for node in nodes.values():
|
||||||
|
if node.is_git and node.requirements:
|
||||||
|
_LOGGER.warning(
|
||||||
|
"Library %s is requested both from a git source (%s) and as "
|
||||||
|
"registry version(s) %s; using the git source.",
|
||||||
|
node.key,
|
||||||
|
node.url,
|
||||||
|
sorted(node.requirements),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Two graph nodes that resolve to the same component name (e.g. a package
|
||||||
|
# referenced both bare and as ``owner/name``) are not deduplicated and can
|
||||||
|
# produce conflicting component definitions. Warn so it's not silent.
|
||||||
|
canonical_keys: dict[str, str] = {}
|
||||||
|
for node_key, component in components.items():
|
||||||
|
canonical = component.get_sanitized_name()
|
||||||
|
if canonical_keys.setdefault(canonical, node_key) != node_key:
|
||||||
|
_LOGGER.warning(
|
||||||
|
"Library %s is referenced under multiple names (%s and %s); these "
|
||||||
|
"are not deduplicated. Reference it consistently as %s.",
|
||||||
|
canonical,
|
||||||
|
canonical_keys[canonical],
|
||||||
|
node_key,
|
||||||
|
canonical,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Wire each component's dependencies to the single resolved instances, then
|
||||||
|
# regenerate build files.
|
||||||
|
for key, component in components.items():
|
||||||
|
component.dependencies = [
|
||||||
|
components[dep_key]
|
||||||
|
for dep_key in sorted(nodes[key].edges)
|
||||||
|
if dep_key in components
|
||||||
|
]
|
||||||
|
for component in components.values():
|
||||||
|
_apply_extra_script(component)
|
||||||
|
write_file_if_changed(
|
||||||
|
component.path / "CMakeLists.txt",
|
||||||
|
generate_cmakelists_txt(component),
|
||||||
|
)
|
||||||
|
write_file_if_changed(
|
||||||
|
component.path / "idf_component.yml",
|
||||||
|
generate_idf_component_yml(component),
|
||||||
|
)
|
||||||
|
|
||||||
|
return [components[key] for key in top_level if key in components]
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ section instead of static fields. The script runs under SCons during
|
|||||||
PIO's build and mutates the active ``Environment`` (``env.Append``,
|
PIO's build and mutates the active ``Environment`` (``env.Append``,
|
||||||
``env.Replace``, …) — chiefly to set ``LIBPATH``/``LIBS`` per chip MCU.
|
``env.Replace``, …) — chiefly to set ``LIBPATH``/``LIBS`` per chip MCU.
|
||||||
|
|
||||||
ESPHome's PIO→IDF converter (``_generate_idf_component``) doesn't run
|
ESPHome's PIO→IDF converter doesn't run SCons, so these scripts were
|
||||||
SCons, so these scripts were previously ignored and any library
|
previously ignored and any library
|
||||||
relying on them failed to link under ``toolchain: esp-idf``. This
|
relying on them failed to link under ``toolchain: esp-idf``. This
|
||||||
module provides a small shim that ``exec``s an extra-script with a
|
module provides a small shim that ``exec``s an extra-script with a
|
||||||
fake ``env`` object, captures the common ``env.Append(...)`` calls,
|
fake ``env`` object, captures the common ``env.Append(...)`` calls,
|
||||||
|
|||||||
@@ -21,13 +21,15 @@ from esphome.espidf.component import (
|
|||||||
URLSource,
|
URLSource,
|
||||||
_check_library_data,
|
_check_library_data,
|
||||||
_collect_filtered_files,
|
_collect_filtered_files,
|
||||||
_convert_library_to_component,
|
_node_key,
|
||||||
|
_normalize_dependencies,
|
||||||
_parse_library_json,
|
_parse_library_json,
|
||||||
_parse_library_properties,
|
_parse_library_properties,
|
||||||
_process_dependencies,
|
_resolve_registry_version,
|
||||||
_split_list_by_condition,
|
_split_list_by_condition,
|
||||||
generate_cmakelists_txt,
|
generate_cmakelists_txt,
|
||||||
generate_idf_component_yml,
|
generate_idf_component_yml,
|
||||||
|
generate_idf_components,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -162,43 +164,6 @@ def test_generate_cmakelists_txt_references_project_managed_components_variable(
|
|||||||
assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content
|
assert "${ESPHOME_PROJECT_MANAGED_COMPONENTS}" in content
|
||||||
|
|
||||||
|
|
||||||
def test_generate_idf_component_overwrites_bundled_files(
|
|
||||||
tmp_path: Path,
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
esp32_idf_core: None,
|
|
||||||
) -> None:
|
|
||||||
# A library that ships its own CMakeLists.txt + idf_component.yml must
|
|
||||||
# have both replaced by ESPHome's generated content. Library authors'
|
|
||||||
# bundled IDF metadata is frequently broken (bogus REQUIRES, hard-coded
|
|
||||||
# frameworks), so we always regenerate from library.json.
|
|
||||||
from esphome.espidf.component import _generate_idf_component
|
|
||||||
|
|
||||||
(tmp_path / "src").mkdir()
|
|
||||||
(tmp_path / "src" / "main.cpp").write_text("// dummy\n")
|
|
||||||
(tmp_path / "library.json").write_text(json.dumps({"name": "tripwire-lib"}))
|
|
||||||
(tmp_path / "CMakeLists.txt").write_text("# TRIPWIRE_BUNDLED_CMAKELISTS\n")
|
|
||||||
(tmp_path / "idf_component.yml").write_text("# TRIPWIRE_BUNDLED_MANIFEST\n")
|
|
||||||
|
|
||||||
fake_component = IDFComponent(
|
|
||||||
"owner/tripwire-lib", "1.0.0", source=URLSource("http://dummy")
|
|
||||||
)
|
|
||||||
fake_component.path = tmp_path
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component,
|
|
||||||
"_convert_library_to_component",
|
|
||||||
lambda _lib: fake_component,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(fake_component, "download", lambda force=False: None)
|
|
||||||
|
|
||||||
_generate_idf_component(Library("owner/tripwire-lib", "1.0.0", None))
|
|
||||||
|
|
||||||
cml = (tmp_path / "CMakeLists.txt").read_text()
|
|
||||||
manifest = (tmp_path / "idf_component.yml").read_text()
|
|
||||||
assert "TRIPWIRE_BUNDLED_CMAKELISTS" not in cml
|
|
||||||
assert "TRIPWIRE_BUNDLED_MANIFEST" not in manifest
|
|
||||||
assert "idf_component_register" in cml
|
|
||||||
|
|
||||||
|
|
||||||
def test_generate_idf_component_yml_basic(tmp_component):
|
def test_generate_idf_component_yml_basic(tmp_component):
|
||||||
tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
|
tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}}
|
||||||
result = generate_idf_component_yml(tmp_component)
|
result = generate_idf_component_yml(tmp_component)
|
||||||
@@ -419,200 +384,58 @@ empty=
|
|||||||
assert "empty" not in result
|
assert "empty" not in result
|
||||||
|
|
||||||
|
|
||||||
def test_convert_library_with_repository():
|
def test_node_key_git_with_ref():
|
||||||
lib = Library("name", None, "https://github.com/foo/bar.git#v1.2.3")
|
key, is_git, locator = _node_key(
|
||||||
|
"name", None, "https://github.com/foo/bar.git#v1.2.3"
|
||||||
result = _convert_library_to_component(lib)
|
|
||||||
|
|
||||||
assert result.name == "foo/bar"
|
|
||||||
assert result.version == "*"
|
|
||||||
assert isinstance(result.source, GitSource)
|
|
||||||
assert result.source.ref == "v1.2.3"
|
|
||||||
|
|
||||||
|
|
||||||
def test_convert_library_with_branch_ref():
|
|
||||||
lib = Library("name", None, "https://github.com/foo/bar.git#some-branch")
|
|
||||||
|
|
||||||
result = _convert_library_to_component(lib)
|
|
||||||
|
|
||||||
assert result.name == "foo/bar"
|
|
||||||
assert result.version == "*"
|
|
||||||
assert isinstance(result.source, GitSource)
|
|
||||||
assert result.source.ref == "some-branch"
|
|
||||||
|
|
||||||
|
|
||||||
def test_convert_library_missing_ref_uses_default_branch():
|
|
||||||
"""A bare URL with no #ref clones the remote's default branch.
|
|
||||||
|
|
||||||
Matches PIO's lib_deps behavior and external_components handling --
|
|
||||||
git.clone_or_update with ref=None leaves the depth-1 clone on
|
|
||||||
whatever branch the remote HEAD points at.
|
|
||||||
"""
|
|
||||||
lib = Library("name", None, "https://github.com/foo/bar.git")
|
|
||||||
|
|
||||||
result = _convert_library_to_component(lib)
|
|
||||||
|
|
||||||
assert result.name == "foo/bar"
|
|
||||||
assert result.version == "*"
|
|
||||||
assert isinstance(result.source, GitSource)
|
|
||||||
assert result.source.ref is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_convert_library_registry(monkeypatch):
|
|
||||||
lib = Library("foo/bar", "^1.0.0", None)
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component,
|
|
||||||
"_get_package_from_pio_registry",
|
|
||||||
lambda o, n, r: ("foo", "bar", "1.2.3", "http://example.com/pkg.zip"),
|
|
||||||
)
|
)
|
||||||
|
assert key == "foo/bar"
|
||||||
result = _convert_library_to_component(lib)
|
assert is_git is True
|
||||||
|
assert locator == ("https://github.com/foo/bar.git", "v1.2.3")
|
||||||
assert result.name == "foo/bar"
|
|
||||||
assert result.version == "1.2.3"
|
|
||||||
assert isinstance(result.source, URLSource)
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_dependencies_adds_valid_dependency(tmp_component, monkeypatch):
|
def test_node_key_git_branch_ref():
|
||||||
tmp_component.data = {
|
key, is_git, locator = _node_key(
|
||||||
"dependencies": [
|
"name", None, "https://github.com/foo/bar.git#some-branch"
|
||||||
{
|
|
||||||
"name": "foo",
|
|
||||||
"version": "1.0",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component,
|
|
||||||
"_generate_idf_component",
|
|
||||||
lambda lib: esphome.espidf.component.IDFComponent(
|
|
||||||
lib.name, lib.version, source=URLSource("http://dummy.com")
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
assert (key, is_git, locator[1]) == ("foo/bar", True, "some-branch")
|
||||||
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
|
|
||||||
|
|
||||||
_process_dependencies(tmp_component)
|
|
||||||
|
|
||||||
assert len(tmp_component.dependencies) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_dependencies_skips_invalid(tmp_component):
|
def test_node_key_git_no_ref():
|
||||||
tmp_component.data = {
|
_key, is_git, locator = _node_key("name", None, "https://github.com/foo/bar.git")
|
||||||
"dependencies": [
|
assert is_git is True
|
||||||
{"name": "foo", "version": "1.0", "platforms": ["arduino"]},
|
assert locator == ("https://github.com/foo/bar.git", None)
|
||||||
{"invalid": "entry"},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
_process_dependencies(tmp_component)
|
|
||||||
|
|
||||||
assert tmp_component.dependencies == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_dependencies_dict_form(tmp_component, monkeypatch):
|
def test_node_key_registry_owner_name():
|
||||||
"""PIO library.json shorthand ``{"owner/Name": "version"}`` is honored.
|
key, is_git, locator = _node_key("foo/bar", "^1.0.0", None)
|
||||||
|
assert (key, is_git, locator) == ("foo/bar", False, ("foo", "bar"))
|
||||||
|
|
||||||
Iterating a dict gives string keys, which would silently fail the
|
|
||||||
``"name" in dependency`` substring check. Normalize to list-of-dicts
|
|
||||||
first so the dict form (used by e.g. tesla-ble for its nanopb dep)
|
|
||||||
is treated the same as the verbose list form.
|
|
||||||
"""
|
|
||||||
captured: list[Library] = []
|
|
||||||
|
|
||||||
def fake_generate(library):
|
def test_node_key_registry_bare_name():
|
||||||
captured.append(library)
|
key, is_git, locator = _node_key("bar", "1.0", None)
|
||||||
return IDFComponent(
|
assert (key, is_git, locator) == ("bar", False, (None, "bar"))
|
||||||
library.name, library.version, source=URLSource("http://dummy.com")
|
|
||||||
)
|
|
||||||
|
|
||||||
tmp_component.data = {
|
|
||||||
"dependencies": {
|
def test_normalize_dependencies_none():
|
||||||
"nanopb/Nanopb": "^0.4.91",
|
assert _normalize_dependencies(None) == []
|
||||||
"BareName": "1.2.3",
|
|
||||||
}
|
|
||||||
}
|
def test_normalize_dependencies_list_form():
|
||||||
monkeypatch.setattr(
|
deps = [{"name": "foo", "version": "1.0"}]
|
||||||
esphome.espidf.component, "_generate_idf_component", fake_generate
|
assert _normalize_dependencies(deps) == [{"name": "foo", "version": "1.0"}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_dependencies_dict_form():
|
||||||
|
out = _normalize_dependencies({"nanopb/Nanopb": "^0.4.91", "BareName": "1.2.3"})
|
||||||
|
assert {"name": "Nanopb", "owner": "nanopb", "version": "^0.4.91"} in out
|
||||||
|
assert {"name": "BareName", "owner": None, "version": "1.2.3"} in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_dependencies_dict_form_nested_spec():
|
||||||
|
out = _normalize_dependencies(
|
||||||
|
{"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}}
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
|
assert out == [
|
||||||
|
|
||||||
_process_dependencies(tmp_component)
|
|
||||||
|
|
||||||
assert len(tmp_component.dependencies) == 2
|
|
||||||
names = sorted(lib.name for lib in captured)
|
|
||||||
versions = sorted(lib.version for lib in captured)
|
|
||||||
assert names == ["BareName", "nanopb/Nanopb"]
|
|
||||||
assert versions == ["1.2.3", "^0.4.91"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch):
|
|
||||||
"""A dict-value that's a URL gets routed to ``repository`` like the list form."""
|
|
||||||
captured: list[Library] = []
|
|
||||||
|
|
||||||
def fake_generate(library):
|
|
||||||
captured.append(library)
|
|
||||||
return IDFComponent(library.name, "*", source=URLSource("http://dummy.com"))
|
|
||||||
|
|
||||||
tmp_component.data = {
|
|
||||||
"dependencies": {
|
|
||||||
"foo/Bar": "https://github.com/foo/bar.git#main",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component, "_generate_idf_component", fake_generate
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None)
|
|
||||||
|
|
||||||
_process_dependencies(tmp_component)
|
|
||||||
|
|
||||||
assert len(captured) == 1
|
|
||||||
assert captured[0].name == "foo/Bar"
|
|
||||||
assert captured[0].version is None
|
|
||||||
assert captured[0].repository == "https://github.com/foo/bar.git#main"
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch):
|
|
||||||
"""A dict-value that's itself a dict is merged into the entry.
|
|
||||||
|
|
||||||
PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}``
|
|
||||||
for entries that need fields beyond just a version (platforms,
|
|
||||||
frameworks, etc.). The extra fields flow into _check_library_data
|
|
||||||
via the entry merge.
|
|
||||||
"""
|
|
||||||
captured: list[Library] = []
|
|
||||||
checked: list[dict] = []
|
|
||||||
|
|
||||||
def fake_generate(library):
|
|
||||||
captured.append(library)
|
|
||||||
return IDFComponent(
|
|
||||||
library.name, library.version, source=URLSource("http://dummy.com")
|
|
||||||
)
|
|
||||||
|
|
||||||
tmp_component.data = {
|
|
||||||
"dependencies": {
|
|
||||||
"nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component, "_generate_idf_component", fake_generate
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
esphome.espidf.component,
|
|
||||||
"_check_library_data",
|
|
||||||
checked.append,
|
|
||||||
)
|
|
||||||
|
|
||||||
_process_dependencies(tmp_component)
|
|
||||||
|
|
||||||
assert len(captured) == 1
|
|
||||||
assert captured[0].name == "nanopb/Nanopb"
|
|
||||||
assert captured[0].version == "^0.4.91"
|
|
||||||
# Extra spec fields reach _check_library_data so platform/framework
|
|
||||||
# gating still applies.
|
|
||||||
assert checked == [
|
|
||||||
{
|
{
|
||||||
"name": "Nanopb",
|
"name": "Nanopb",
|
||||||
"owner": "nanopb",
|
"owner": "nanopb",
|
||||||
@@ -620,3 +443,364 @@ def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypa
|
|||||||
"platforms": "espidf",
|
"platforms": "espidf",
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_registry(monkeypatch, versions):
|
||||||
|
"""Patch the registry client to serve a canned version list (no network).
|
||||||
|
|
||||||
|
Only ``fetch_registry_package`` is faked; the real
|
||||||
|
``get_compatible_registry_versions`` / ``pick_best_registry_version`` run on
|
||||||
|
the canned data so the intersection logic is exercised for real.
|
||||||
|
"""
|
||||||
|
registry = esphome.espidf.component._make_registry_client()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
registry,
|
||||||
|
"fetch_registry_package",
|
||||||
|
lambda spec: {
|
||||||
|
"owner": {"username": spec.owner or "owner"},
|
||||||
|
"name": spec.name,
|
||||||
|
"versions": [
|
||||||
|
{"name": v, "files": [{"download_url": f"http://x/{v}.tar.gz"}]}
|
||||||
|
for v in versions
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component, "_make_registry_client", lambda: registry
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_registry_version_intersects_constraints(monkeypatch):
|
||||||
|
_patch_registry(monkeypatch, ["1.10018.1", "1.10021.0", "1.10021.1"])
|
||||||
|
owner, name, version, url = _resolve_registry_version(
|
||||||
|
"esphome", "libsodium", {"==1.10021.0", "^1.10018.1"}
|
||||||
|
)
|
||||||
|
assert (owner, name, version) == ("esphome", "libsodium", "1.10021.0")
|
||||||
|
assert url == "http://x/1.10021.0.tar.gz"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_registry_version_picks_highest_satisfying(monkeypatch):
|
||||||
|
_patch_registry(monkeypatch, ["1.0.0", "1.5.0", "2.0.0"])
|
||||||
|
_owner, _name, version, _url = _resolve_registry_version("o", "p", {"^1.0.0"})
|
||||||
|
assert version == "1.5.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_registry_version_conflict_raises(monkeypatch):
|
||||||
|
_patch_registry(monkeypatch, ["1.0.0", "2.0.0"])
|
||||||
|
with pytest.raises(RuntimeError, match="satisfies all requirements"):
|
||||||
|
_resolve_registry_version("o", "p", {"==1.0.0", "==2.0.0"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_dedupes_shared_dependency(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
) -> None:
|
||||||
|
# A and B both depend on shared C under different version specs. The batch
|
||||||
|
# must resolve C once with BOTH requirements collected, wire a single C
|
||||||
|
# instance into both, and regenerate (overwrite) each library's build files.
|
||||||
|
manifests = {
|
||||||
|
"esphome/A": {
|
||||||
|
"name": "A",
|
||||||
|
"dependencies": [
|
||||||
|
{"owner": "esphome", "name": "C", "version": "==1.10021.0"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"esphome/B": {
|
||||||
|
"name": "B",
|
||||||
|
"dependencies": [
|
||||||
|
{"owner": "esphome", "name": "C", "version": "^1.10018.1"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"esphome/C": {"name": "C"},
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "src" / "x.c").write_text("int x;")
|
||||||
|
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
|
||||||
|
(self.path / "CMakeLists.txt").write_text("# TRIPWIRE\n")
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
|
||||||
|
captured: dict[str, set[str]] = {}
|
||||||
|
resolve_calls: list[str] = []
|
||||||
|
|
||||||
|
def fake_resolve(owner, pkgname, requirements):
|
||||||
|
resolve_calls.append(pkgname)
|
||||||
|
captured[f"{owner}/{pkgname}"] = set(requirements)
|
||||||
|
version = "1.10021.0" if pkgname == "C" else "1.0.0"
|
||||||
|
return owner, pkgname, version, f"http://x/{pkgname}.tar.gz"
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component, "_resolve_registry_version", fake_resolve
|
||||||
|
)
|
||||||
|
|
||||||
|
top = generate_idf_components(
|
||||||
|
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
|
||||||
|
)
|
||||||
|
|
||||||
|
# C resolved once (not once per consumer) with BOTH requirements gathered.
|
||||||
|
assert captured["esphome/C"] == {"==1.10021.0", "^1.10018.1"}
|
||||||
|
assert resolve_calls.count("C") == 1
|
||||||
|
# Top-level components returned in request order.
|
||||||
|
assert [c.name for c in top] == ["esphome/A", "esphome/B"]
|
||||||
|
# A and B reference the SAME single C instance (deduped).
|
||||||
|
a_dep = top[0].dependencies[0]
|
||||||
|
b_dep = top[1].dependencies[0]
|
||||||
|
assert a_dep.name == "esphome/C"
|
||||||
|
assert a_dep is b_dep
|
||||||
|
# The bundled CMakeLists was overwritten with generated content.
|
||||||
|
generated = (a_dep.path / "CMakeLists.txt").read_text()
|
||||||
|
assert "TRIPWIRE" not in generated
|
||||||
|
assert "idf_component_register" in generated
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_handles_dependency_cycle(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
) -> None:
|
||||||
|
# A -> B -> A. Must terminate (not recurse forever) and wire the cycle with
|
||||||
|
# a single instance per component.
|
||||||
|
manifests = {
|
||||||
|
"esphome/A": {
|
||||||
|
"name": "A",
|
||||||
|
"dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}],
|
||||||
|
},
|
||||||
|
"esphome/B": {
|
||||||
|
"name": "B",
|
||||||
|
"dependencies": [{"owner": "esphome", "name": "A", "version": "1.0.0"}],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "src" / "x.c").write_text("int x;")
|
||||||
|
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner,
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
top = generate_idf_components([Library("esphome/A", "1.0.0", None)])
|
||||||
|
|
||||||
|
assert [c.name for c in top] == ["esphome/A"]
|
||||||
|
component_a = top[0]
|
||||||
|
component_b = component_a.dependencies[0]
|
||||||
|
assert component_b.name == "esphome/B"
|
||||||
|
# The cycle is wired back to the same A instance, not a duplicate.
|
||||||
|
assert component_b.dependencies[0] is component_a
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_git_overrides_registry_warns(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
# A pulls shared as a registry pin; B pulls the same component from a git
|
||||||
|
# source. The git source wins, but the dropped registry pin must be warned
|
||||||
|
# about (not silently discarded).
|
||||||
|
manifests = {
|
||||||
|
"esphome/A": {
|
||||||
|
"name": "A",
|
||||||
|
"dependencies": [
|
||||||
|
{"owner": "esphome", "name": "shared", "version": "==1.0.0"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"esphome/B": {
|
||||||
|
"name": "B",
|
||||||
|
"dependencies": [
|
||||||
|
{
|
||||||
|
"owner": "esphome",
|
||||||
|
"name": "shared",
|
||||||
|
"version": "https://github.com/esphome/shared.git#main",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"esphome/shared": {"name": "shared"},
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "src" / "x.c").write_text("int x;")
|
||||||
|
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner,
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
top = generate_idf_components(
|
||||||
|
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
|
||||||
|
)
|
||||||
|
|
||||||
|
# shared resolved from the git source (version "*"), not the registry pin.
|
||||||
|
shared = top[0].dependencies[0]
|
||||||
|
assert shared.name == "esphome/shared"
|
||||||
|
assert isinstance(shared.source, GitSource)
|
||||||
|
assert "using the git source" in caplog.text
|
||||||
|
assert "==1.0.0" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_missing_manifest_raises(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
) -> None:
|
||||||
|
# A library with neither library.json nor library.properties is invalid;
|
||||||
|
# fail loudly rather than silently generating build files for it.
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
# no library.json / library.properties written
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner,
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="missing library.json"):
|
||||||
|
generate_idf_components([Library("esphome/A", "1.0.0", None)])
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_warns_on_noncanonical_duplicate(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
# A references "shared" (bare) and B references "owner/shared"; both resolve
|
||||||
|
# to the same canonical name but as distinct graph nodes, so they aren't
|
||||||
|
# deduplicated -- warn about it.
|
||||||
|
manifests = {
|
||||||
|
"esphome/A": {
|
||||||
|
"name": "A",
|
||||||
|
"dependencies": [{"name": "shared", "version": "1.0.0"}],
|
||||||
|
},
|
||||||
|
"esphome/B": {
|
||||||
|
"name": "B",
|
||||||
|
"dependencies": [{"owner": "owner", "name": "shared", "version": "1.0.0"}],
|
||||||
|
},
|
||||||
|
"owner/shared": {"name": "shared"},
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "src" / "x.c").write_text("int x;")
|
||||||
|
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
# Bare "shared" and "owner/shared" both resolve to canonical owner/shared.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner or "owner",
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
generate_idf_components(
|
||||||
|
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "referenced under multiple names" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_incompatible_top_level_raises(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
) -> None:
|
||||||
|
# A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
|
||||||
|
# not be silently dropped.
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "library.json").write_text(
|
||||||
|
json.dumps({"name": "A", "platforms": ["espressif8266"]})
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner,
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="not compatible with ESP-IDF"):
|
||||||
|
generate_idf_components([Library("esphome/A", "1.0.0", None)])
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_idf_components_incompatible_dependency_skipped(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
esp32_idf_core: None,
|
||||||
|
) -> None:
|
||||||
|
# An incompatible *transitive* dependency is skipped (not fatal): A is fine,
|
||||||
|
# its esp8266-only dep B is dropped and not wired.
|
||||||
|
manifests = {
|
||||||
|
"esphome/A": {
|
||||||
|
"name": "A",
|
||||||
|
"dependencies": [{"owner": "esphome", "name": "B", "version": "1.0.0"}],
|
||||||
|
},
|
||||||
|
"esphome/B": {"name": "B", "platforms": ["espressif8266"]},
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_download(self, force=False):
|
||||||
|
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
|
||||||
|
(self.path / "src").mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
|
||||||
|
|
||||||
|
monkeypatch.setattr(IDFComponent, "download", fake_download)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
esphome.espidf.component,
|
||||||
|
"_resolve_registry_version",
|
||||||
|
lambda owner, pkgname, requirements: (
|
||||||
|
owner,
|
||||||
|
pkgname,
|
||||||
|
"1.0.0",
|
||||||
|
f"http://x/{pkgname}.tar.gz",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
top = generate_idf_components([Library("esphome/A", "1.0.0", None)])
|
||||||
|
|
||||||
|
assert [c.name for c in top] == ["esphome/A"]
|
||||||
|
# The incompatible dependency was dropped, not wired in.
|
||||||
|
assert top[0].dependencies == []
|
||||||
|
|||||||
Reference in New Issue
Block a user