[nrf52] Build PlatformIO libraries as Zephyr modules (sdk-nrf) (#17250)

This commit is contained in:
Jonathan Swoboda
2026-07-02 10:26:34 -04:00
committed by GitHub
parent 41cf842d5d
commit 65fc10d627
7 changed files with 388 additions and 25 deletions
+26
View File
@@ -411,6 +411,17 @@ async def _dfu_to_code(dfu_config):
def copy_files() -> None:
"""Copy files to the build directory."""
# Library conversion to Zephyr modules is wired into the sdk-nrf
# CMakeLists only; the PlatformIO toolchain's forked platform package
# cannot compile external libraries at all, so the build would fail at
# link time anyway. Fail fast with a clear message instead.
if CORE.using_toolchain_platformio and CORE.platformio_libraries:
raise EsphomeError(
f"Libraries ({', '.join(sorted(CORE.platformio_libraries))}) are "
"not supported on the nRF52 'platformio' toolchain; use toolchain "
"'sdk-nrf' to build them as Zephyr modules."
)
if CORE.using_toolchain_platformio and (
zephyr_data()[KEY_BOOTLOADER] == BOOTLOADER_MCUBOOT
or zephyr_data()[KEY_BOARD] == "xiao_ble"
@@ -702,11 +713,26 @@ def _generate_cmake_lists() -> bool:
compile_flags = get_project_compile_flags()
link_flags = get_project_link_flags()
# Convert any PlatformIO libraries added via cg.add_library() into Zephyr
# modules and discover them through EXTRA_ZEPHYR_MODULES (a CMake list, set
# before find_package(Zephyr) so the modules are picked up). Only
# framework-agnostic libraries actually compile under Zephyr.
from esphome.components.zephyr.library import generate_zephyr_modules
module_dirs = generate_zephyr_modules(list(CORE.platformio_libraries.values()))
lines = [
"cmake_minimum_required(VERSION 3.20.0)",
"",
'set(Zephyr_DIR "$ENV{ZEPHYR_BASE}/share/zephyr-package/cmake/")',
"",
]
if module_dirs:
modules = ";".join(str(d).replace("\\", "/") for d in module_dirs)
lines += [f'set(EXTRA_ZEPHYR_MODULES "{modules}")', ""]
lines += [
"find_package(Zephyr REQUIRED)",
"",
f"project({CORE.name})",
+180
View File
@@ -0,0 +1,180 @@
"""Zephyr backend for the shared PlatformIO library converter.
For each PlatformIO library added via ``cg.add_library()``, emit a Zephyr
external module (``zephyr/module.yml`` + ``zephyr/CMakeLists.txt`` built with the
``zephyr_library*`` API) into the shared ``pio_components`` cache. The caller
wires the resulting module directories into the build via
``EXTRA_ZEPHYR_MODULES``; Zephyr then compiles each module and links it into the
final image.
Only framework-agnostic libraries (plain C/C++ that doesn't depend on the Arduino
API) will actually compile under Zephyr — this converter shares the
fetch/parse/cache plumbing, not API compatibility.
"""
from pathlib import Path
from esphome import yaml_util
from esphome.core import EsphomeError, Library
from esphome.helpers import write_file_if_changed
from esphome.platformio.library import (
DEFAULT_BUILD_FLAGS,
DEFAULT_BUILD_INCLUDE_DIR,
DEFAULT_BUILD_SRC_FILTER,
SRC_FILE_EXTENSIONS,
ConvertedLibrary,
LibraryBackend,
PathType,
collect_filtered_files,
convert_libraries,
ensure_list,
split_list_by_condition,
)
# Zephyr libraries declare frameworks rarely and the PIO ``platforms`` token for
# nRF is seldom present, so the platform check is disabled (None) and only the
# framework mismatch warning fires.
ZEPHYR_FRAMEWORK = "zephyr"
def _escape(p: PathType) -> str:
# In CMakeLists.txt, backslashes need to be escaped (mirrors the ESP-IDF
# backend's escape_entry). Doubling -- rather than rewriting '\' -> '/' --
# preserves content, so it's safe for arbitrary build flags (e.g. a -D value
# containing a backslash) as well as Windows paths.
return f'"{str(p)}"'.replace("\\", "\\\\")
def generate_module_yml(component: ConvertedLibrary) -> str:
"""Render the ``zephyr/module.yml`` manifest for a converted library."""
return yaml_util.dump(
{
"name": component.get_require_name(),
"build": {"cmake": "zephyr"},
}
)
def generate_cmakelists_txt(component: ConvertedLibrary) -> str:
"""Render the ``zephyr/CMakeLists.txt`` that builds a converted library.
Sources/includes are emitted as absolute paths since the CMakeLists lives in
the library's ``zephyr/`` subdir while its sources sit alongside it. Include
dirs are published globally so the app (and sibling libraries) can include the
library's headers, mirroring ESP-IDF's public ``INCLUDE_DIRS``.
"""
build = component.data.get("build", {})
build_src_dir = build.get("srcDir")
if not build_src_dir:
for d in ["src", "Src", "."]:
if (component.path / Path(d)).is_dir():
build_src_dir = d
break
build_include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR)
build_src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER))
build_flags = ensure_list(build.get("flags", DEFAULT_BUILD_FLAGS))
src_files = collect_filtered_files(
component.path / Path(build_src_dir), build_src_filter
)
src_files = sorted(
str(Path(p).resolve())
for p in src_files
if Path(p).suffix in SRC_FILE_EXTENSIONS
)
include_dir_flags, build_flags = split_list_by_condition(
build_flags, lambda a: a[2:].strip() if a.startswith("-I") else None
)
link_directories, build_flags = split_list_by_condition(
build_flags, lambda a: a[2:].strip() if a.startswith("-L") else None
)
link_libraries, build_flags = split_list_by_condition(
build_flags, lambda a: a[2:].strip() if a.startswith("-l") else None
)
include_dirs = [build_include_dir, build_src_dir, *include_dir_flags]
include_dirs = [
str((component.path / Path(d)).resolve())
for d in include_dirs
if (component.path / Path(d)).is_dir()
]
lines = [f"zephyr_library_named({component.get_require_name()})"]
if src_files:
lines += [
"zephyr_library_sources(",
*[f" {_escape(p)}" for p in src_files],
")",
]
if include_dirs:
lines += [
"zephyr_include_directories(",
*[f" {_escape(p)}" for p in include_dirs],
")",
]
if build_flags:
lines += [
"zephyr_library_compile_options(",
*[f" {_escape(f)}" for f in build_flags],
")",
]
# Best-effort link wiring; most Zephyr-portable libraries don't need it.
link_flags = [f"-L{d}" for d in link_directories] + [
f"-l{lib}" for lib in link_libraries
]
if link_flags:
lines += [
"zephyr_link_libraries(",
*[f" {_escape(f)}" for f in link_flags],
")",
]
return "\n".join(lines) + "\n"
def _emit_zephyr_module(component: ConvertedLibrary) -> None:
zephyr_dir = component.path / "zephyr"
write_file_if_changed(zephyr_dir / "module.yml", generate_module_yml(component))
write_file_if_changed(
zephyr_dir / "CMakeLists.txt", generate_cmakelists_txt(component)
)
def generate_zephyr_modules(libraries: list[Library]) -> list[Path]:
"""Convert ``libraries`` to Zephyr modules and return all module directories.
The returned list includes transitive dependencies (each converted library is
its own module). Every directory should be added to ``EXTRA_ZEPHYR_MODULES``;
Zephyr links all module libraries into the image, so cross-library symbols
resolve without explicit dependency declarations.
Raises ``EsphomeError`` if two libraries resolve to the same Zephyr module
name -- each module's CMakeLists calls ``zephyr_library_named(<name>)``, so a
duplicate would otherwise fail the build with a CMake "target already exists".
The converter already warns when a library is referenced under inconsistent
specs (bare ``name`` vs ``owner/name``, git vs registry); this turns that into
an actionable error at the Zephyr boundary where it is fatal.
"""
module_dirs: list[Path] = []
by_name: dict[str, Path] = {}
def emit(component: ConvertedLibrary) -> None:
name = component.get_require_name()
if name in by_name:
raise EsphomeError(
f"Two libraries resolve to the same Zephyr module '{name}' "
f"({by_name[name]} and {component.path}). Reference the library "
f"consistently (e.g. always as 'owner/name') so it resolves once."
)
by_name[name] = component.path
_emit_zephyr_module(component)
module_dirs.append(component.path)
backend = LibraryBackend(
platform=None, framework=ZEPHYR_FRAMEWORK, emit=emit, cache_key="zephyr"
)
convert_libraries(libraries, backend)
return module_dirs
+1
View File
@@ -264,5 +264,6 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
platform=ESP32_PLATFORM,
framework=_idf_framework(),
emit=_emit_idf_component,
cache_key="idf",
)
return convert_libraries(libraries, backend)
+33 -12
View File
@@ -68,7 +68,9 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE"
class Source:
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
def download(
self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = ""
) -> Path:
raise NotImplementedError
@@ -76,8 +78,14 @@ class URLSource(Source):
def __init__(self, url: str):
self.url = url
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
def download(
self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = ""
) -> Path:
# Namespace the cache per backend (e.g. pio_components/idf, .../zephyr) so
# the build files each backend writes into the library dir can't collide.
base_dir = Path(CORE.data_dir) / DOMAIN
if namespace:
base_dir = base_dir / namespace
h = hashlib.new("sha256")
h.update(self.url.encode())
if salt:
@@ -113,12 +121,19 @@ class GitSource(Source):
self.url = url
self.ref = ref
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
def download(
self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = ""
) -> Path:
domain = DOMAIN
if namespace:
domain = f"{domain}/{namespace}"
if salt:
domain = f"{domain}/{salt}"
path, _ = git.clone_or_update(
url=self.url,
ref=self.ref,
refresh=git.NEVER_REFRESH if not force else None,
domain=f"{DOMAIN}/{salt}" if salt else DOMAIN,
domain=domain,
submodules=[],
subpath=Path(dir_suffix),
)
@@ -167,16 +182,16 @@ class ConvertedLibrary:
def get_require_name(self):
return self.get_sanitized_name().replace("/", "__")
def download(self, force: bool = False, salt: str = ""):
def download(self, force: bool = False, salt: str = "", namespace: str = ""):
"""Fetch the library into the shared cache and record its ``path``.
The cache directory is named after the sanitized library name; backends
rely on that name to identify the unit they build (e.g. ESP-IDF uses the
directory name as the component name, replacing ``/`` with ``__`` via
``get_require_name``).
``get_require_name``). ``namespace`` keeps each backend's cache separate.
"""
self.path = self.source.download(
self.get_sanitized_name(), force=force, salt=salt
self.get_sanitized_name(), force=force, salt=salt, namespace=namespace
)
@@ -188,11 +203,15 @@ class LibraryBackend:
``emit`` writes the toolchain-specific build files into a resolved library's
``path`` (e.g. the ESP-IDF ``CMakeLists.txt`` + ``idf_component.yml``, or a
Zephyr ``module.yml`` + ``CMakeLists.txt``).
``cache_key`` namespaces the download cache (``pio_components/<cache_key>/``)
so the differing build files two backends emit into a library dir never
collide when the same config dir hosts both an ESP-IDF and a Zephyr build.
"""
platform: str
platform: str | None
framework: str
emit: Callable[["ConvertedLibrary"], None]
cache_key: str
def ensure_list[T](obj: T | list[T]) -> list[T]:
@@ -306,7 +325,7 @@ def split_list_by_condition(
return matched, non_matched
def check_library_data(data: dict, platform: str, framework: str):
def check_library_data(data: dict, platform: str | None, framework: str):
"""
Check whether a library manifest is compatible with the target toolchain.
@@ -319,7 +338,9 @@ def check_library_data(data: dict, platform: str, framework: str):
Args:
data: PIO library manifest dict being processed.
platform: The PlatformIO platform token the build targets (e.g.
``espressif32``).
``espressif32``). ``None`` skips the platform check entirely — useful
for targets (e.g. Zephyr) where PIO manifests rarely declare the
platform yet portable libraries still build.
framework: The active framework name (e.g. ``espidf``, ``arduino``,
``zephyr``) the manifest is expected to declare.
@@ -332,7 +353,7 @@ def check_library_data(data: dict, platform: str, framework: str):
platforms = ensure_list(platforms)
# Check if library supports the target platform
valid_platforms = "*" in platforms or platform in platforms
valid_platforms = platform is None or "*" in platforms or platform in platforms
if not valid_platforms:
raise InvalidLibrary(f"Unsupported library platforms: {platforms}")
@@ -613,7 +634,7 @@ def convert_libraries(
component = ConvertedLibrary(
_owner_pkgname_to_name(owner, name), version, URLSource(url)
)
component.download(salt=salt)
component.download(salt=salt, namespace=backend.cache_key)
library_json_path = component.path / "library.json"
library_properties_path = component.path / "library.properties"
+27 -11
View File
@@ -481,7 +481,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
"esphome/C": {"name": "C"},
}
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -543,7 +543,7 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
download_salts: list[str] = []
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
download_salts.append(salt)
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
@@ -597,7 +597,7 @@ def test_generate_idf_components_handles_dependency_cycle(
},
}
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -654,7 +654,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
"esphome/shared": {"name": "shared"},
}
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -691,7 +691,7 @@ def test_generate_idf_components_missing_manifest_raises(
) -> None:
# A library with neither library.json nor library.properties is invalid;
# fail loudly rather than silently generating build files for it.
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
# no library.json / library.properties written
@@ -733,7 +733,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
"owner/shared": {"name": "shared"},
}
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -766,7 +766,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
) -> None:
# A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
# not be silently dropped.
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(
@@ -804,7 +804,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
"esphome/B": {"name": "B", "platforms": ["espressif8266"]},
}
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
@@ -847,6 +847,13 @@ def test_url_source_salt_changes_cache_path(
assert source.download("lib") == expected[""]
assert source.download("lib", salt="abcd1234") == expected["abcd1234"]
# A backend namespace adds a pio_components/<namespace>/ subdir.
digest = hashlib.sha256(url.encode()).hexdigest()[:8]
ns_expected = base / "idf" / digest / "lib"
ns_expected.mkdir(parents=True)
(ns_expected / ".esphome_extracted").touch()
assert source.download("lib", namespace="idf") == ns_expected
def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
"""The salt becomes a subdirectory of the git clone domain."""
@@ -863,7 +870,14 @@ def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
source = GitSource("https://github.com/esphome/noise-c.git", "v1.0")
source.download("noise-c")
source.download("noise-c", salt="abcd1234")
assert domains == ["pio_components", "pio_components/abcd1234"]
source.download("noise-c", namespace="idf")
source.download("noise-c", namespace="zephyr", salt="abcd1234")
assert domains == [
"pio_components",
"pio_components/abcd1234",
"pio_components/idf",
"pio_components/zephyr/abcd1234",
]
def test_idf_component_download_passes_salt() -> None:
@@ -873,7 +887,9 @@ def test_idf_component_download_passes_salt() -> None:
source.download.return_value = Path("/converted/owner/name")
c = IDFComponent("owner/name", "1.0", source=source)
c.download(force=True, salt="abcd1234")
c.download(force=True, salt="abcd1234", namespace="idf")
source.download.assert_called_once_with("owner/name", force=True, salt="abcd1234")
source.download.assert_called_once_with(
"owner/name", force=True, salt="abcd1234", namespace="idf"
)
assert c.path == Path("/converted/owner/name")
+4 -2
View File
@@ -26,7 +26,9 @@ from esphome.platformio.library import (
def _backend(emit=lambda component: None) -> LibraryBackend:
return LibraryBackend(platform="espressif32", framework="espidf", emit=emit)
return LibraryBackend(
platform="espressif32", framework="espidf", emit=emit, cache_key="idf"
)
def test_check_library_data_accepts_wildcards():
@@ -134,7 +136,7 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch):
def _patch_download_with_manifests(monkeypatch, tmp_path, manifests, *, properties=()):
"""Fake ConvertedLibrary.download to materialize canned manifests on disk."""
def fake_download(self, force=False, salt=""):
def fake_download(self, force=False, salt="", namespace=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
self.path.mkdir(parents=True, exist_ok=True)
if self.name in properties:
+117
View File
@@ -0,0 +1,117 @@
"""Tests for the Zephyr backend of the shared PlatformIO library converter."""
from pathlib import Path
import pytest
import esphome.components.zephyr.library as zlib
from esphome.components.zephyr.library import (
generate_cmakelists_txt,
generate_module_yml,
generate_zephyr_modules,
)
from esphome.core import EsphomeError, Library
from esphome.platformio.library import ConvertedLibrary, URLSource
def _make_component(path: Path, name: str = "mylib") -> ConvertedLibrary:
c = ConvertedLibrary(name, "1.0", source=URLSource("http://dummy"))
c.path = path
return c
def test_generate_module_yml_uses_sanitized_name():
c = ConvertedLibrary("owner/My Lib", "1.0", source=URLSource("http://dummy"))
out = generate_module_yml(c)
# "/" -> "__" and " " -> "_" so it's a valid Zephyr module name.
assert "name: owner__My_Lib" in out
assert "cmake: zephyr" in out
def test_generate_cmakelists_txt_basic(tmp_path):
c = _make_component(tmp_path)
src = tmp_path / "src"
src.mkdir()
(src / "main.c").write_text("int main() {}")
c.data = {}
out = generate_cmakelists_txt(c)
assert "zephyr_library_named(mylib)" in out
assert "zephyr_library_sources(" in out
# Sources are emitted as absolute paths (CMakeLists lives in zephyr/ subdir),
# backslash-escaped for CMake (matching the output on Windows).
assert str((src / "main.c").resolve()).replace("\\", "\\\\") in out
def test_generate_cmakelists_txt_flags_and_includes(tmp_path):
c = _make_component(tmp_path)
(tmp_path / "src").mkdir()
(tmp_path / "src" / "a.c").write_text("")
(tmp_path / "include").mkdir()
c.data = {"build": {"flags": ["-Iinclude", "-DFOO", "-Wall", "-Llibdir", "-lm"]}}
out = generate_cmakelists_txt(c)
assert "zephyr_include_directories(" in out
assert str((tmp_path / "include").resolve()).replace("\\", "\\\\") in out
assert "zephyr_library_compile_options(" in out
assert "-DFOO" in out
assert "-Wall" in out
assert "zephyr_link_libraries(" in out
assert "-Llibdir" in out
assert "-lm" in out
def test_generate_zephyr_modules_collects_all_dirs_and_writes(tmp_path, monkeypatch):
# Two converted libraries: one top-level, one transitive dependency. The
# converter calls backend.emit for both; generate_zephyr_modules must return
# *all* module dirs (not just top-level) so every module is discoverable.
top = _make_component(tmp_path / "top", "top")
(top.path / "src").mkdir(parents=True)
(top.path / "src" / "t.c").write_text("")
dep = _make_component(tmp_path / "dep", "dep")
(dep.path / "src").mkdir(parents=True)
(dep.path / "src" / "d.c").write_text("")
captured = {}
def fake_convert(libraries, backend):
captured["platform"] = backend.platform
captured["framework"] = backend.framework
backend.emit(top)
backend.emit(dep)
return [top]
monkeypatch.setattr(zlib, "convert_libraries", fake_convert)
dirs = generate_zephyr_modules([Library("top", "1.0", None)])
assert dirs == [top.path, dep.path]
# Platform check disabled for Zephyr; framework declared as zephyr.
assert captured["platform"] is None
assert captured["framework"] == "zephyr"
for comp in (top, dep):
assert (comp.path / "zephyr" / "module.yml").is_file()
assert (comp.path / "zephyr" / "CMakeLists.txt").is_file()
def test_generate_zephyr_modules_errors_on_duplicate_module_name(tmp_path, monkeypatch):
# The same library referenced under inconsistent specs (e.g. bare vs
# owner-qualified, or git vs registry) resolves to two components with the
# same Zephyr module name, which would collide in zephyr_library_named().
a = _make_component(tmp_path / "a", "esphome/noise-c")
a.path.mkdir(parents=True)
b = _make_component(tmp_path / "b", "esphome/noise-c")
b.path.mkdir(parents=True)
assert a.get_require_name() == b.get_require_name()
def fake_convert(libraries, backend):
backend.emit(a)
backend.emit(b)
return [a]
monkeypatch.setattr(zlib, "convert_libraries", fake_convert)
with pytest.raises(EsphomeError, match="same Zephyr module"):
generate_zephyr_modules([Library("esphome/noise-c", "1.0", None)])