github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-09-30 00:10:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[nrf52] Fetch only the nRF Connect SDK projects a build needs (#19735)

Browse Source
This commit is contained in:
J. Nick Koston
2026-09-27 19:18:08 +02:00
committed by GitHub
parent a91023e87c
commit 1f175c1942
9 changed files with 747 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit_tests/test_nrf52_build.py
+23
View File
@@ -0,0 +1,23 @@
"""Tests for the nRF52 native west build command."""
from pathlib import Path
from esphome.components.nrf52 import _west_build_command
def test_west_build_sets_the_cmake_build_type() -> None:
"""The picolibc module used to force MinSizeRel, and with it the -DNDEBUG
that keeps libc assert() out of the image; the build sets it itself now."""
python = Path("/penv/python")
cmd = _west_build_command(
python,
"adafruit_feather_nrf52840",
Path("/build/pio"),
Path("/build/zephyr"),
)
# str(), not a literal: the separator differs on Windows
assert cmd[:4] == [str(python), "-m", "west", "build"]
assert "-b" in cmd and "adafruit_feather_nrf52840" in cmd
# CMake arguments come after west's own
assert cmd[cmd.index("--") + 1 :] == ["-DCMAKE_BUILD_TYPE=MinSizeRel"]
tests/unit_tests/test_nrf52_framework.py
+439 -32
View File
@@ -1,5 +1,6 @@
"""Tests for esphome.components.nrf52.framework helpers."""
import errno
import hashlib
import os
from pathlib import Path
@@ -14,16 +15,20 @@ from esphome.components.nrf52 import _resolve_toolchain
from esphome.components.nrf52.framework import (
_PLATFORMIO_PENV_REQUIREMENTS,
_REQUIREMENTS,
DEFAULT_WEST_PROJECTS,
TOOLCHAIN_VERSION,
_get_penv_site_packages,
_get_platformio_penv_path,
_get_toolchain_platform_info,
_needs_venv_rebuild,
_wanted_west_projects,
check_and_install,
get_build_env,
get_sdk_nrf_tools_path,
include_west_project,
setup_platformio_python_env,
)
from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR
import esphome.config_validation as cv
from esphome.config_validation import Version
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain
@@ -76,6 +81,8 @@ def test_get_toolchain_platform_info(
# ---------------------------------------------------------------------------
_TEST_SDK_VERSION = "2.9.0"
# The filter that keeps only DEFAULT_WEST_PROJECTS
_DEFAULTS_FILTER = "-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr"
@pytest.fixture
@@ -142,6 +149,33 @@ def _touch_penv_python(penv: Path) -> None:
python.touch()
def _subcommand(cmd: list[str]) -> str:
tool = "west" if "west" in cmd else "pip"
return cmd[cmd.index(tool) + 1]
def _subcommands(run_command_ok) -> list[str]:
"""The west or pip subcommand of each command run, in order."""
return [_subcommand(c.args[0]) for c in run_command_ok.call_args_list]
def _project_filter(run_command_ok) -> str:
"""The manifest.project-filter value of the last ``west config`` run."""
for west_call in reversed(run_command_ok.call_args_list):
cmd = west_call.args[0]
if "manifest.project-filter" in cmd:
return cmd[-1]
raise AssertionError("no west config command ran")
def _mark_installed(dirs: SimpleNamespace) -> None:
"""Every install step finished: venv, zephyr requirements, SDK, toolchain."""
_mark_venv_ready(dirs.python_env)
(dirs.python_env / ".zephyr_reqs_ready").touch()
(dirs.framework / ".ready").touch()
(dirs.toolchain / ".ready").touch()
def _mark_venv_ready(python_env: Path) -> None:
"""Write the venv sentinel with the current requirements hash and a
present interpreter so the rebuild gate passes."""
@@ -157,10 +191,7 @@ class TestCheckAndInstall:
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""All three sentinels present → nothing downloaded or compiled."""
_mark_venv_ready(nrf52_dirs.python_env)
(nrf52_dirs.python_env / ".zephyr_reqs_ready").touch()
(nrf52_dirs.framework / ".ready").touch()
(nrf52_dirs.toolchain / ".ready").touch()
_mark_installed(nrf52_dirs)
check_and_install()
@@ -195,8 +226,14 @@ class TestCheckAndInstall:
check_and_install()
mock_nrf52_ops.create_venv.assert_called_once()
# pip install requirements, west init, west update, pip install zephyr reqs
assert mock_nrf52_ops.run_command_ok.call_count == 4
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"install", # requirements
"init",
"config",
"update",
"list",
"install", # zephyr requirements
]
# minimal SDK + per-arch toolchain
assert mock_nrf52_ops.download_from_mirrors.call_count == 2
assert mock_nrf52_ops.archive_extract_all.call_count == 2
@@ -216,8 +253,13 @@ class TestCheckAndInstall:
check_and_install()
mock_nrf52_ops.create_venv.assert_not_called()
# west init, west update, pip install zephyr reqs
assert mock_nrf52_ops.run_command_ok.call_count == 3
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"init",
"config",
"update",
"list",
"install",
]
# minimal SDK + per-arch toolchain
assert mock_nrf52_ops.download_from_mirrors.call_count == 2
@@ -249,16 +291,12 @@ class TestCheckAndInstall:
check_and_install()
init_cmd, update_cmd = (
call.args[0] for call in mock_nrf52_ops.run_command_ok.call_args_list[:2]
)
assert "init" in init_cmd
assert "-o=--depth=1" in init_cmd
assert "update" in update_cmd
assert "--fetch-opt=--depth=1" in update_cmd
init, _, update = mock_nrf52_ops.run_command_ok.call_args_list[:3]
assert "-o=--depth=1" in init.args[0]
assert "--fetch-opt=--depth=1" in update.args[0]
# Streamed, so the long clone's progress reaches the log
for west_call in mock_nrf52_ops.run_command_ok.call_args_list[:2]:
assert west_call.kwargs["stream_output"] is True
assert init.kwargs["stream_output"] is True
assert update.kwargs["stream_output"] is True
def test_interrupted_download_resumes(
self,
@@ -280,12 +318,15 @@ class TestCheckAndInstall:
not in mock_nrf52_ops.rmdir.call_args_list
)
assert not (nrf52_dirs.framework / ".resume_failed").exists()
# west update (no init), then pip install zephyr reqs
first = mock_nrf52_ops.run_command_ok.call_args_list[0]
assert "update" in first.args[0]
assert "init" not in first.args[0]
assert first.kwargs["cwd"] == nrf52_dirs.framework
assert mock_nrf52_ops.run_command_ok.call_count == 2
# west update in the workspace (no init), then pip install zephyr reqs
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"config",
"update",
"list",
"install",
]
update = mock_nrf52_ops.run_command_ok.call_args_list[1]
assert update.kwargs["cwd"] == nrf52_dirs.framework
assert (nrf52_dirs.framework / ".ready").exists()
def test_failed_resume_keeps_the_download_once(
@@ -297,7 +338,8 @@ class TestCheckAndInstall:
dropped again) and is retried on the next build."""
_mark_venv_ready(nrf52_dirs.python_env)
_mark_west_initialized(nrf52_dirs.framework)
mock_nrf52_ops.run_command_ok.return_value = False
# config succeeds, the resumed update fails
mock_nrf52_ops.run_command_ok.side_effect = [True, False]
with pytest.raises(EsphomeError, match="Can't resume"):
check_and_install()
@@ -332,16 +374,21 @@ class TestCheckAndInstall:
_mark_venv_ready(nrf52_dirs.python_env)
_mark_west_initialized(nrf52_dirs.framework)
(nrf52_dirs.framework / ".resume_failed").touch()
# resumed update fails; clean init, update and zephyr reqs succeed
mock_nrf52_ops.run_command_ok.side_effect = [False, True, True, True]
# resumed update fails; the clean clone and zephyr reqs succeed
mock_nrf52_ops.run_command_ok.side_effect = [True, False, *[True] * 5]
check_and_install()
mock_nrf52_ops.rmdir.assert_any_call(nrf52_dirs.framework, msg=ANY)
commands = [c.args[0] for c in mock_nrf52_ops.run_command_ok.call_args_list]
assert "update" in commands[0]
assert "init" in commands[1]
assert "update" in commands[2]
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"config",
"update",
"init",
"config",
"update",
"list",
"install",
]
def test_requirements_install_failure_raises(
self,
@@ -373,12 +420,372 @@ class TestCheckAndInstall:
) -> None:
"""Failing west update raises EsphomeError."""
_mark_venv_ready(nrf52_dirs.python_env)
# init succeeds, update fails
mock_nrf52_ops.run_command_ok.side_effect = [True, False]
# init and config succeed, update fails
mock_nrf52_ops.run_command_ok.side_effect = [True, True, False]
with pytest.raises(EsphomeError, match="Can't update"):
check_and_install()
def test_fresh_install_fetches_only_default_projects(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A fresh clone leaves every west project out except the defaults."""
_mark_venv_ready(nrf52_dirs.python_env)
check_and_install()
assert _project_filter(mock_nrf52_ops.run_command_ok) == (
"-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr"
)
stamp = nrf52_dirs.framework / ".west_projects"
assert stamp.read_text(encoding="utf-8").split() == sorted(
DEFAULT_WEST_PROJECTS
)
def test_included_project_joins_the_filter(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A project a component includes is fetched with the defaults."""
_mark_venv_ready(nrf52_dirs.python_env)
include_west_project("oberon-psa-crypto")
check_and_install()
assert "+oberon-psa-crypto" in _project_filter(
mock_nrf52_ops.run_command_ok
).split(",")
def test_installed_sdk_fetches_a_newly_needed_project(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A finished install gains a project another config left out, keeping
what it already has."""
_mark_installed(nrf52_dirs)
(nrf52_dirs.framework / ".west_projects").write_text(
"cmsis\nhal_nordic\nnrfxlib\ntinycrypt\nzephyr", encoding="utf-8"
)
include_west_project("openthread")
check_and_install()
# The names are checked before anything is fetched
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"list",
"config",
"update",
]
wanted = "-.*,+cmsis,+hal_nordic,+nrfxlib,+openthread,+tinycrypt,+zephyr"
assert _project_filter(mock_nrf52_ops.run_command_ok) == wanted
mock_nrf52_ops.rmdir.assert_not_called()
@pytest.mark.parametrize(
"stamp",
[
pytest.param(None, id="install_from_before_the_filter"),
pytest.param(
"cmsis\nhal_nordic\nnrfxlib\nopenthread\nzephyr",
id="project_already_fetched",
),
],
)
def test_installed_sdk_with_the_project_fetches_nothing(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
stamp: str | None,
) -> None:
"""No fetch when the install already has every wanted project; an
install without the stamp (west config, no filter) has them all. The
names are still checked, which only reads the manifest."""
_mark_installed(nrf52_dirs)
_mark_west_initialized(nrf52_dirs.framework)
if stamp is not None:
(nrf52_dirs.framework / ".west_projects").write_text(
stamp, encoding="utf-8"
)
include_west_project("openthread")
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
def test_sysbuild_fetches_mcuboot(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""Sysbuild always builds the MCUboot image, so it needs the project."""
_mark_venv_ready(nrf52_dirs.python_env)
CORE.data[KEY_ZEPHYR] = {KEY_SYSBUILD: True}
check_and_install()
assert "+mcuboot" in _project_filter(mock_nrf52_ops.run_command_ok).split(",")
@pytest.mark.parametrize(
("sdk_version", "has_cmsis_6"),
[("2.9.2", False), ("3.1.0", True), ("3.2.0", True)],
)
def test_sdk_3_1_and_later_want_cmsis_6(
self, setup_core: Path, sdk_version: str, has_cmsis_6: bool
) -> None:
"""Zephyr 4.1 moved the Cortex-M core headers to the cmsis_6 module."""
CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse(sdk_version)}
assert ("cmsis_6" in _wanted_west_projects()) is has_cmsis_6
def test_default_projects_never_read_the_stamp(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A build wanting only the defaults has them on any install; an
unreadable stamp shows the check never looked."""
_mark_installed(nrf52_dirs)
(nrf52_dirs.framework / ".west_projects").mkdir()
check_and_install()
mock_nrf52_ops.run_command_ok.assert_not_called()
def test_failed_project_fetch_raises(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A failed fetch of a newly needed project raises, keeps the stamp and
puts the workspace filter back to the stamp's projects and the defaults."""
_mark_venv_ready(nrf52_dirs.python_env)
(nrf52_dirs.framework / ".ready").touch()
stamp = nrf52_dirs.framework / ".west_projects"
stamp.write_text("zephyr", encoding="utf-8")
include_west_project("openthread")
# list and config succeed, update fails, the restoring config succeeds
mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True]
with pytest.raises(EsphomeError, match="Can't update"):
check_and_install()
assert stamp.read_text(encoding="utf-8") == "zephyr"
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"list",
"config",
"update",
"config",
]
# The defaults always stay in the restored filter
assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER
def test_failed_fetch_with_a_lost_stamp_keeps_the_defaults_active(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""An unknown installed set must not leave a filter with every module off."""
_mark_installed(nrf52_dirs)
_mark_west_initialized(nrf52_dirs.framework)
(nrf52_dirs.framework / ".west" / "config").write_text(
"[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8"
)
include_west_project("openthread")
# list and config succeed, update fails, the restoring config succeeds
mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True]
with pytest.raises(EsphomeError, match="Can't update"):
check_and_install()
assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER
def test_failed_filter_restore_is_logged(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
caplog: pytest.LogCaptureFixture,
) -> None:
"""When the filter can't be put back after a failed fetch, the user is told."""
_mark_venv_ready(nrf52_dirs.python_env)
(nrf52_dirs.framework / ".ready").touch()
(nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8")
include_west_project("openthread")
# list and config succeed, update fails, the restoring config fails too
mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, False]
with pytest.raises(EsphomeError, match="Can't update"):
check_and_install()
assert "Couldn't put the nRF Connect SDK" in caplog.text
def test_lost_stamp_on_a_filtered_install_fetches_again(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A workspace with a project filter but no stamp is a filtered install
whose record was lost, so the wanted projects are fetched, not assumed."""
_mark_installed(nrf52_dirs)
_mark_west_initialized(nrf52_dirs.framework)
(nrf52_dirs.framework / ".west" / "config").write_text(
"[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8"
)
include_west_project("openthread")
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"list",
"config",
"update",
]
def test_install_waits_for_another_build_holding_the_lock(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
caplog: pytest.LogCaptureFixture,
) -> None:
"""A held lock is waited on, never mistaken for a filesystem that
cannot lock (filelock's Timeout is an OSError as well)."""
from filelock import Timeout
_mark_installed(nrf52_dirs)
with (
caplog.at_level("INFO"),
patch("filelock.FileLock") as file_lock,
):
file_lock.return_value.acquire.side_effect = [Timeout("install.lock"), None]
check_and_install()
assert file_lock.return_value.acquire.call_count == 2
assert "Waiting for another build" in caplog.text
assert "continuing without a lock" not in caplog.text
def test_install_from_before_the_filter_still_checks_the_names(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""An install with every project fetches nothing, but an unknown name
is still rejected so a mistake shows on every install alike."""
_mark_installed(nrf52_dirs)
_mark_west_initialized(nrf52_dirs.framework)
include_west_project("openthread")
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
mock_nrf52_ops.run_command_ok.reset_mock()
mock_nrf52_ops.run_command_ok.return_value = False
include_west_project("no_such_project")
with pytest.raises(EsphomeError, match="west list failed"):
check_and_install()
def test_install_lock_is_per_sdk_version(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""Installs of different SDK versions do not wait on each other; the
toolchain they share is only locked while it is missing."""
_mark_installed(nrf52_dirs)
with patch("filelock.FileLock") as file_lock:
check_and_install()
lock_files = [Path(c.args[0]).name for c in file_lock.call_args_list]
assert lock_files == [f"sdk-v{_TEST_SDK_VERSION}.lock"]
def test_install_runs_unlocked_where_the_filesystem_cannot_lock(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
caplog: pytest.LogCaptureFixture,
) -> None:
"""No soft-lock fallback: its marker outlives a killed build and would
hang every later one, so the install goes ahead without a lock."""
_mark_installed(nrf52_dirs)
with patch("filelock.FileLock") as file_lock:
file_lock.return_value.acquire.side_effect = OSError(
errno.ENOSYS, "Function not implemented"
)
check_and_install()
assert file_lock.call_args.kwargs == {"fallback_to_soft": False}
assert "continuing without a lock" in caplog.text
def test_unknown_project_raises_before_anything_is_written(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A filter naming a project the manifest lacks fetches nothing, so
the names are checked once the update resolved the manifest, and an
unknown one is never recorded as installed."""
_mark_venv_ready(nrf52_dirs.python_env)
include_west_project("no_such_project")
# init, config and update succeed, list fails
mock_nrf52_ops.run_command_ok.side_effect = [True, True, True, False]
with pytest.raises(EsphomeError, match="west list failed .*no_such_project"):
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"init",
"config",
"update",
"list",
]
assert "no_such_project" in mock_nrf52_ops.run_command_ok.call_args.args[0]
assert not (nrf52_dirs.framework / ".west_projects").exists()
def test_unknown_project_on_an_installed_sdk_fetches_nothing(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""On a finished install the names are checked first, so an unknown one
costs no fetch and leaves the workspace as it was."""
_mark_venv_ready(nrf52_dirs.python_env)
(nrf52_dirs.framework / ".ready").touch()
(nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8")
include_west_project("no_such_project")
# list fails before anything is fetched or changed
mock_nrf52_ops.run_command_ok.side_effect = [False]
with pytest.raises(EsphomeError, match="west list failed"):
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
assert (nrf52_dirs.framework / ".west_projects").read_text(
encoding="utf-8"
) == "zephyr"
def test_missing_stamp_and_west_config_fetches_again(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""With neither the stamp nor .west/config left, nothing says what the
install has, so the wanted projects are fetched rather than assumed."""
_mark_installed(nrf52_dirs)
include_west_project("openthread")
check_and_install()
assert _subcommands(mock_nrf52_ops.run_command_ok) == [
"list",
"config",
"update",
]
def test_toolchain_download_passes_platform_substitutions(
self,
nrf52_dirs: SimpleNamespace,