From 1f175c1942ecb5c47741314ada4f907856da8c5a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 27 Sep 2026 19:18:08 +0200 Subject: [PATCH] [nrf52] Fetch only the nRF Connect SDK projects a build needs (#19735) --- esphome/components/debug/__init__.py | 4 +- esphome/components/nrf52/__init__.py | 35 +- esphome/components/nrf52/clang_tidy.py | 55 +- esphome/components/nrf52/framework.py | 202 +++++++- esphome/components/openthread/__init__.py | 8 + .../components/zephyr_ble_server/__init__.py | 6 + .../components/zephyr_mcumgr/ota/__init__.py | 4 + tests/unit_tests/test_nrf52_build.py | 23 + tests/unit_tests/test_nrf52_framework.py | 471 ++++++++++++++++-- 9 files changed, 747 insertions(+), 61 deletions(-) create mode 100644 tests/unit_tests/test_nrf52_build.py diff --git a/esphome/components/debug/__init__.py b/esphome/components/debug/__init__.py index a889d13329..e4692dfa9a 100644 --- a/esphome/components/debug/__init__.py +++ b/esphome/components/debug/__init__.py @@ -1,4 +1,5 @@ import esphome.codegen as cg +from esphome.components.nrf52.framework import include_west_project from esphome.components.zephyr import zephyr_add_prj_conf from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv @@ -51,7 +52,8 @@ async def to_code(config: ConfigType) -> None: zephyr_add_prj_conf("HWINFO", True) # gdb thread support zephyr_add_prj_conf("DEBUG_THREAD_INFO", True) - # RTT + # RTT sources live in the segger project + include_west_project("segger") zephyr_add_prj_conf("USE_SEGGER_RTT", True) zephyr_add_prj_conf("RTT_CONSOLE", True) zephyr_add_prj_conf("LOG", True) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 249c28ac5f..b32d4f075e 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -832,6 +832,26 @@ def _copy_if_exists(src: Path, dst: Path) -> None: shutil.copy2(src, dst) +def _west_build_command( + python_executable: Path, board: str, build_dir: Path, source_dir: Path +) -> list[str]: + return [ + str(python_executable), + "-m", + "west", + "build", + "--pristine=auto", + "-b", + board, + "-d", + str(build_dir), + str(source_dir), + "--", + # Only adds -DNDEBUG (Kconfig sets the optimization level); picolibc used to force it + "-DCMAKE_BUILD_TYPE=MinSizeRel", + ] + + def run_compile(args, config: ConfigType) -> bool: if CORE.using_toolchain_platformio: # The actual build is done by PlatformIO (the caller falls through to @@ -866,18 +886,9 @@ def run_compile(args, config: ConfigType) -> bool: _LOGGER.info("Build inputs changed, cleaning %s", build_dir) rmtree(build_dir) - west_cmd = [ - str(paths["python_executable"]), - "-m", - "west", - "build", - "--pristine=auto", - "-b", - board, - "-d", - str(build_dir), - str(source_dir), - ] + west_cmd = _west_build_command( + paths["python_executable"], board, build_dir, source_dir + ) if not run_command_ok( west_cmd, diff --git a/esphome/components/nrf52/clang_tidy.py b/esphome/components/nrf52/clang_tidy.py index 2dd4b7bd09..89272cd406 100644 --- a/esphome/components/nrf52/clang_tidy.py +++ b/esphome/components/nrf52/clang_tidy.py @@ -20,9 +20,13 @@ commands. from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass import logging from pathlib import Path +from .framework import bluetooth_west_projects + TIDY_PROJECT_NAME = "esphome_tidy" # Analyzed against the native toolchain's default SDK version @@ -33,19 +37,36 @@ _TIDY_BOARD = "adafruit_itsybitsy_nrf52840" # app target emits a C++ compile command to harvest flags/includes from. _TIDY_MAIN_CPP = "int main() { return 0; }\n" -# Kconfig superset enabling every subsystem an ESPHome nrf52 component may -# use, so the compile commands carry all of their include paths. -_TIDY_PRJ_CONF = """\ + +@dataclass(frozen=True) +class _TidySubsystem: + """A subsystem's Kconfig and the west projects its headers come from.""" + + name: str + prj_conf: str + # A callable when the projects differ by SDK version + west_projects: tuple[str, ...] | Callable[[], tuple[str, ...]] = () + + +# Kconfig superset for the compile commands and the projects it needs +_TIDY_SUBSYSTEMS = ( + _TidySubsystem( + "base", + """\ CONFIG_CPP=y CONFIG_STD_CPP20=y CONFIG_REQUIRES_FULL_LIBCPP=y CONFIG_NEWLIB_LIBC=y -CONFIG_BT=y CONFIG_ADC=y # posix (time sets POSIX_CLOCK, socket sets POSIX_API); without it the # Zephyr POSIX headers clash with the libc ones under analysis CONFIG_POSIX_API=y -#mcumgr begin +""", + ), + _TidySubsystem("bluetooth", "CONFIG_BT=y\n", bluetooth_west_projects), + _TidySubsystem( + "mcumgr", + """\ CONFIG_NET_BUF=y CONFIG_ZCBOR=y CONFIG_MCUMGR=y @@ -60,14 +81,23 @@ CONFIG_MCUMGR_MGMT_NOTIFICATION_HOOKS=y CONFIG_MCUMGR_GRP_IMG_STATUS_HOOKS=y CONFIG_MCUMGR_GRP_IMG_UPLOAD_CHECK_HOOK=y CONFIG_MCUMGR_TRANSPORT_UART=y -#mcumgr end -#zigbee begin +""", + ("mcuboot", "zcbor"), + ), + _TidySubsystem( + "zigbee", + """\ CONFIG_ZIGBEE=y CONFIG_CRYPTO=y CONFIG_NVS=y CONFIG_SETTINGS=y -#zigbee end -""" +""", + ), +) + +_TIDY_PRJ_CONF = "".join( + f"# {subsystem.name}\n{subsystem.prj_conf}" for subsystem in _TIDY_SUBSYSTEMS +) def _tidy_cmakelists(library_include_dirs: str) -> str: @@ -166,6 +196,7 @@ def _setup_core(work_dir: Path) -> None: from esphome.core import CORE from . import RECOMMENDED_SDK_NRF_VERSION + from .framework import include_west_project CORE.name = TIDY_PROJECT_NAME # config_path's parent is the data-dir root for per-run artifacts. The @@ -178,6 +209,10 @@ def _setup_core(work_dir: Path) -> None: CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse( RECOMMENDED_SDK_NRF_VERSION ) + for subsystem in _TIDY_SUBSYSTEMS: + projects = subsystem.west_projects + for project in projects() if callable(projects) else projects: + include_west_project(project) def generate_compile_commands(work_dir: Path, platformio_ini: Path) -> Path: @@ -237,6 +272,8 @@ def generate_compile_commands(work_dir: Path, platformio_ini: Path) -> Path: "zephyr_generated_headers", "--", "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + # As in a real build, so NDEBUG is set + "-DCMAKE_BUILD_TYPE=MinSizeRel", ] if not run_command_ok( west_cmd, diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 6e57c46c7e..48af47e8ff 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -1,3 +1,7 @@ +from collections.abc import Iterator +import configparser +from contextlib import contextmanager +from dataclasses import dataclass, field import hashlib import logging import os @@ -7,6 +11,7 @@ import shutil import sys from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path +from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError @@ -258,7 +263,84 @@ def _patch_uf2conv_escape_sequences(framework_path: Path) -> None: tmp.replace(uf2conv) -def _west_update(env_python_path: Path, framework_path: Path) -> bool: +# West projects every build needs; components add others with include_west_project() +DEFAULT_WEST_PROJECTS = ("cmsis", "hal_nordic", "nrfxlib", "zephyr") + +_KEY_NRF52 = "nrf52" +# The projects a finished install fetched +_WEST_PROJECTS_FILE = ".west_projects" + + +@dataclass +class _Nrf52Data: + west_projects: set[str] = field(default_factory=lambda: set(DEFAULT_WEST_PROJECTS)) + + +def _get_data() -> _Nrf52Data: + if _KEY_NRF52 not in CORE.data: + CORE.data[_KEY_NRF52] = _Nrf52Data() + return CORE.data[_KEY_NRF52] + + +def include_west_project(name: str) -> None: + """Fetch a west project left out by default; call from to_code().""" + _get_data().west_projects.add(name) + + +def bluetooth_west_projects() -> tuple[str, ...]: + """Bluetooth's crypto: TinyCrypt up to SDK 3.1, PSA (mbedtls, Oberon) from 3.2.""" + if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 2, 0): + return ("mbedtls", "oberon-psa-crypto") + return ("tinycrypt",) + + +def _wanted_west_projects() -> set[str]: + projects = set(_get_data().west_projects) + # Zephyr 4.1 moved the Cortex-M core headers to cmsis_6 + if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 1, 0): + projects.add("cmsis_6") + # Sysbuild builds the MCUboot image with any bootloader + if CORE.data.get(KEY_ZEPHYR, {}).get(KEY_SYSBUILD): + projects.add("mcuboot") + return projects + + +def _set_project_filter( + env_python_path: Path, framework_path: Path, projects: set[str] +) -> bool: + # "--" keeps west from reading the leading "-" as an option + project_filter = ",".join(["-.*", *(f"+{p}" for p in sorted(projects))]) + cmd = [str(env_python_path), "-m", "west", "config", "manifest.project-filter"] + return run_command_ok([*cmd, "--", project_filter], cwd=framework_path) + + +def _check_west_projects( + env_python_path: Path, framework_path: Path, version: str, projects: set[str] +) -> None: + """Raise when the manifest lacks one of ``projects``; needs zephyr cloned. + + west update quietly skips an unknown name in the filter, west list fails on it. + """ + names = sorted(projects) + cmd = [str(env_python_path), "-m", "west", "list", "-f", "{name}", *names] + if not run_command_ok(cmd, cwd=framework_path): + raise EsphomeError( + f"west list failed for the requested nRF Connect SDK {version} projects " + f"({', '.join(names)}); a project the manifest does not have is the " + "usual cause, see west's output above" + ) + + +def _west_update( + env_python_path: Path, + framework_path: Path, + version: str, + projects: set[str], + checked: bool = False, +) -> bool: + """Fetch ``projects``; False when the fetch fails.""" + if not _set_project_filter(env_python_path, framework_path, projects): + return False cmd = [ str(env_python_path), "-m", @@ -267,12 +349,103 @@ def _west_update(env_python_path: Path, framework_path: Path) -> bool: "--narrow", "--fetch-opt=--depth=1", ] - # Streamed so the per-project progress of the long clone reaches the log - return run_command_ok(cmd, cwd=framework_path, stream_output=True) + # Streamed so the clone's progress reaches the log + if not run_command_ok(cmd, cwd=framework_path, stream_output=True): + return False + if not checked: + _check_west_projects(env_python_path, framework_path, version, projects) + (framework_path / _WEST_PROJECTS_FILE).write_text( + "\n".join(sorted(projects)), encoding="utf-8" + ) + return True + + +def _installed_west_projects(framework_path: Path) -> set[str] | None: + """The projects a finished install fetched; None when it has every project.""" + try: + stamp = (framework_path / _WEST_PROJECTS_FILE).read_text(encoding="utf-8") + except FileNotFoundError: + pass + else: + return set(stamp.split()) + # No stamp: an install from before the filter has every project, a filtered + # one that lost its stamp fetches again + config = configparser.ConfigParser() + if not config.read(framework_path / ".west" / "config", encoding="utf-8"): + return set() + if config.has_option("manifest", "project-filter"): + return set() + return None + + +def _restore_project_filter( + env_python_path: Path, framework_path: Path, version: str, installed: set[str] +) -> None: + """Put the filter back to the stamp's projects; the defaults always stay in.""" + projects = installed | set(DEFAULT_WEST_PROJECTS) + if not _set_project_filter(env_python_path, framework_path, projects): + _LOGGER.warning( + "Couldn't put the nRF Connect SDK %s project filter back; " + "the next build that fetches a project sets it again", + version, + ) + + +# Lock wait slices, so Ctrl-C stays responsive +_INSTALL_LOCK_POLL = 1 + + +@contextmanager +def _install_lock(name: str) -> Iterator[None]: + """Serialize a shared install step across builds running at once.""" + from filelock import FileLock, Timeout + + lock_path = get_sdk_nrf_tools_path() / f"{name}.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + # No soft lock: its marker outlives a killed build and hangs every later one + lock = FileLock(str(lock_path), fallback_to_soft=False) + waiting = False + while True: + try: + lock.acquire(timeout=_INSTALL_LOCK_POLL) + break + except Timeout: # before OSError, which it subclasses + if not waiting: + waiting = True + _LOGGER.info("Waiting for another build installing %s ...", name) + except OSError as err: + _LOGGER.warning( + "Can't lock %s (%s), continuing without a lock", lock_path, err + ) + break + try: + yield + finally: + lock.release() + + +def _fetch_missing_west_projects( + env_python_path: Path, framework_path: Path, version: str, projects: set[str] +) -> None: + """Fetch the wanted projects a finished install lacks; it only ever gains.""" + if projects <= set(DEFAULT_WEST_PROJECTS): + return + installed = _installed_west_projects(framework_path) + # Before the fetch, so an unknown name costs nothing on any install + _check_west_projects(env_python_path, framework_path, version, projects) + if installed is None or not (missing := projects - installed): + return + _LOGGER.info( + "Fetching nRF Connect SDK %s projects: %s", version, ", ".join(sorted(missing)) + ) + wanted = installed | projects + if not _west_update(env_python_path, framework_path, version, wanted, checked=True): + _restore_project_filter(env_python_path, framework_path, version, installed) + raise EsphomeError(f"Can't update nRF Connect SDK {version}") def _install_framework( - env_python_path: Path, framework_path: Path, version: str + env_python_path: Path, framework_path: Path, version: str, projects: set[str] ) -> None: """Clone the nRF Connect SDK into ``framework_path`` with west. @@ -289,7 +462,7 @@ def _install_framework( initialized = (framework_path / ".west" / "config").is_file() if initialized and not (framework_path / ".ready").exists(): _LOGGER.info("Resuming the nRF Connect SDK %s download ...", version) - if _west_update(env_python_path, framework_path): + if _west_update(env_python_path, framework_path, version, projects): resume_failed.unlink(missing_ok=True) return if not resume_failed.exists(): @@ -318,12 +491,17 @@ def _install_framework( if not run_command_ok(cmd, stream_output=True): raise EsphomeError(f"Can't initialize nRF Connect SDK {version}") _LOGGER.info("Updating nRF Connect SDK %s (this may take a while) ...", version) - if not _west_update(env_python_path, framework_path): + if not _west_update(env_python_path, framework_path, version, projects): raise EsphomeError(f"Can't update nRF Connect SDK {version}") def check_and_install() -> None: version = _get_version_str() + with _install_lock(f"sdk-{version}"): + _check_and_install(version) + + +def _check_and_install(version: str) -> None: python_env_path = _get_python_env_path(version) env_python_path = get_python_env_executable_path(python_env_path, "python") sentinel = python_env_path / ".ready" @@ -354,12 +532,15 @@ def check_and_install() -> None: framework_path = _get_framework_path(version) sentinel = framework_path / ".ready" zephyr_reqs = framework_path / "zephyr" / "scripts" / "requirements.txt" + projects = _wanted_west_projects() if not sentinel.exists() or not zephyr_reqs.exists(): - _install_framework(env_python_path, framework_path, version) + _install_framework(env_python_path, framework_path, version, projects) framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] if framework_ver < cv.Version(2, 9, 2): _patch_uf2conv_escape_sequences(framework_path) sentinel.touch() + else: + _fetch_missing_west_projects(env_python_path, framework_path, version, projects) zephyr_sentinel = python_env_path / ".zephyr_reqs_ready" if ( @@ -380,6 +561,13 @@ def check_and_install() -> None: raise EsphomeError(f"Install Zephyr requirements for {version} failure") zephyr_sentinel.touch() + # Shared by every SDK version; locked only while missing + if not (_get_toolchain_path(TOOLCHAIN_VERSION) / ".ready").exists(): + with _install_lock(f"toolchain-{TOOLCHAIN_VERSION}"): + _install_toolchain() + + +def _install_toolchain() -> None: toolchains_dir = _get_toolchain_path(TOOLCHAIN_VERSION) sentinel = toolchains_dir / ".ready" if not sentinel.exists(): diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index 907200d5ec..ac14efc8e6 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -19,6 +19,7 @@ from esphome.components.esp32 import ( ) from esphome.components.mdns import MDNSComponent, enable_mdns_storage from esphome.components.network import add_use_address +from esphome.components.nrf52.framework import include_west_project from esphome.components.zephyr import zephyr_add_prj_conf from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv @@ -30,6 +31,8 @@ from esphome.const import ( CONF_LOG_LEVEL, CONF_OUTPUT_POWER, CONF_USE_ADDRESS, + KEY_CORE, + KEY_FRAMEWORK_VERSION, PLATFORM_ESP32, PlatformFramework, ) @@ -317,6 +320,11 @@ async def to_code(config: ConfigType) -> None: if CORE.is_esp32: set_sdkconfig_options(config) elif CORE.using_zephyr: + # Crypto through PSA: mbedtls, plus Oberon from SDK 2.7 + include_west_project("mbedtls") + include_west_project("openthread") + if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(2, 7, 0): + include_west_project("oberon-psa-crypto") zephyr_add_prj_conf("NET_L2_OPENTHREAD", True) zephyr_add_prj_conf( f"OPENTHREAD_NORDIC_LIBRARY_{config.get(CONF_DEVICE_TYPE)}", True diff --git a/esphome/components/zephyr_ble_server/__init__.py b/esphome/components/zephyr_ble_server/__init__.py index 59e5bc556b..a012e832c7 100644 --- a/esphome/components/zephyr_ble_server/__init__.py +++ b/esphome/components/zephyr_ble_server/__init__.py @@ -1,5 +1,9 @@ from esphome import automation import esphome.codegen as cg +from esphome.components.nrf52.framework import ( + bluetooth_west_projects, + include_west_project, +) from esphome.components.zephyr import zephyr_add_prj_conf import esphome.config_validation as cv from esphome.const import CONF_ID, Framework @@ -36,6 +40,8 @@ _CALLBACK_AUTOMATIONS = ( async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) zephyr_add_prj_conf("BT", True) + for project in bluetooth_west_projects(): + include_west_project(project) zephyr_add_prj_conf("BT_PERIPHERAL", True) zephyr_add_prj_conf("BT_RX_STACK_SIZE", 1536) zephyr_add_prj_conf("BT_DEVICE_NAME", CORE.name) diff --git a/esphome/components/zephyr_mcumgr/ota/__init__.py b/esphome/components/zephyr_mcumgr/ota/__init__.py index ad89c1ac79..109d1f7dd6 100644 --- a/esphome/components/zephyr_mcumgr/ota/__init__.py +++ b/esphome/components/zephyr_mcumgr/ota/__init__.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components.nrf52.boards import BOOTLOADER_CONFIG +from esphome.components.nrf52.framework import include_west_project from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code from esphome.components.zephyr import ( HexValue, @@ -115,6 +116,9 @@ async def to_code(config: ConfigType) -> None: zephyr_add_prj_conf("NET_BUF", True) zephyr_add_prj_conf("ZCBOR", True) + include_west_project("zcbor") + # The image manager includes MCUboot headers with any bootloader + include_west_project("mcuboot") zephyr_add_prj_conf("MCUMGR", True) zephyr_add_prj_conf("MCUMGR_GRP_IMG", True) diff --git a/tests/unit_tests/test_nrf52_build.py b/tests/unit_tests/test_nrf52_build.py new file mode 100644 index 0000000000..868bc9549f --- /dev/null +++ b/tests/unit_tests/test_nrf52_build.py @@ -0,0 +1,23 @@ +"""Tests for the nRF52 native west build command.""" + +from pathlib import Path + +from esphome.components.nrf52 import _west_build_command + + +def test_west_build_sets_the_cmake_build_type() -> None: + """The picolibc module used to force MinSizeRel, and with it the -DNDEBUG + that keeps libc assert() out of the image; the build sets it itself now.""" + python = Path("/penv/python") + cmd = _west_build_command( + python, + "adafruit_feather_nrf52840", + Path("/build/pio"), + Path("/build/zephyr"), + ) + + # str(), not a literal: the separator differs on Windows + assert cmd[:4] == [str(python), "-m", "west", "build"] + assert "-b" in cmd and "adafruit_feather_nrf52840" in cmd + # CMake arguments come after west's own + assert cmd[cmd.index("--") + 1 :] == ["-DCMAKE_BUILD_TYPE=MinSizeRel"] diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index 05805a1d57..d6b27bdbef 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -1,5 +1,6 @@ """Tests for esphome.components.nrf52.framework helpers.""" +import errno import hashlib import os from pathlib import Path @@ -14,16 +15,20 @@ from esphome.components.nrf52 import _resolve_toolchain from esphome.components.nrf52.framework import ( _PLATFORMIO_PENV_REQUIREMENTS, _REQUIREMENTS, + DEFAULT_WEST_PROJECTS, TOOLCHAIN_VERSION, _get_penv_site_packages, _get_platformio_penv_path, _get_toolchain_platform_info, _needs_venv_rebuild, + _wanted_west_projects, check_and_install, get_build_env, get_sdk_nrf_tools_path, + include_west_project, setup_platformio_python_env, ) +from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR import esphome.config_validation as cv from esphome.config_validation import Version from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain @@ -76,6 +81,8 @@ def test_get_toolchain_platform_info( # --------------------------------------------------------------------------- _TEST_SDK_VERSION = "2.9.0" +# The filter that keeps only DEFAULT_WEST_PROJECTS +_DEFAULTS_FILTER = "-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr" @pytest.fixture @@ -142,6 +149,33 @@ def _touch_penv_python(penv: Path) -> None: python.touch() +def _subcommand(cmd: list[str]) -> str: + tool = "west" if "west" in cmd else "pip" + return cmd[cmd.index(tool) + 1] + + +def _subcommands(run_command_ok) -> list[str]: + """The west or pip subcommand of each command run, in order.""" + return [_subcommand(c.args[0]) for c in run_command_ok.call_args_list] + + +def _project_filter(run_command_ok) -> str: + """The manifest.project-filter value of the last ``west config`` run.""" + for west_call in reversed(run_command_ok.call_args_list): + cmd = west_call.args[0] + if "manifest.project-filter" in cmd: + return cmd[-1] + raise AssertionError("no west config command ran") + + +def _mark_installed(dirs: SimpleNamespace) -> None: + """Every install step finished: venv, zephyr requirements, SDK, toolchain.""" + _mark_venv_ready(dirs.python_env) + (dirs.python_env / ".zephyr_reqs_ready").touch() + (dirs.framework / ".ready").touch() + (dirs.toolchain / ".ready").touch() + + def _mark_venv_ready(python_env: Path) -> None: """Write the venv sentinel with the current requirements hash and a present interpreter so the rebuild gate passes.""" @@ -157,10 +191,7 @@ class TestCheckAndInstall: mock_nrf52_ops: SimpleNamespace, ) -> None: """All three sentinels present → nothing downloaded or compiled.""" - _mark_venv_ready(nrf52_dirs.python_env) - (nrf52_dirs.python_env / ".zephyr_reqs_ready").touch() - (nrf52_dirs.framework / ".ready").touch() - (nrf52_dirs.toolchain / ".ready").touch() + _mark_installed(nrf52_dirs) check_and_install() @@ -195,8 +226,14 @@ class TestCheckAndInstall: check_and_install() mock_nrf52_ops.create_venv.assert_called_once() - # pip install requirements, west init, west update, pip install zephyr reqs - assert mock_nrf52_ops.run_command_ok.call_count == 4 + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "install", # requirements + "init", + "config", + "update", + "list", + "install", # zephyr requirements + ] # minimal SDK + per-arch toolchain assert mock_nrf52_ops.download_from_mirrors.call_count == 2 assert mock_nrf52_ops.archive_extract_all.call_count == 2 @@ -216,8 +253,13 @@ class TestCheckAndInstall: check_and_install() mock_nrf52_ops.create_venv.assert_not_called() - # west init, west update, pip install zephyr reqs - assert mock_nrf52_ops.run_command_ok.call_count == 3 + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "init", + "config", + "update", + "list", + "install", + ] # minimal SDK + per-arch toolchain assert mock_nrf52_ops.download_from_mirrors.call_count == 2 @@ -249,16 +291,12 @@ class TestCheckAndInstall: check_and_install() - init_cmd, update_cmd = ( - call.args[0] for call in mock_nrf52_ops.run_command_ok.call_args_list[:2] - ) - assert "init" in init_cmd - assert "-o=--depth=1" in init_cmd - assert "update" in update_cmd - assert "--fetch-opt=--depth=1" in update_cmd + init, _, update = mock_nrf52_ops.run_command_ok.call_args_list[:3] + assert "-o=--depth=1" in init.args[0] + assert "--fetch-opt=--depth=1" in update.args[0] # Streamed, so the long clone's progress reaches the log - for west_call in mock_nrf52_ops.run_command_ok.call_args_list[:2]: - assert west_call.kwargs["stream_output"] is True + assert init.kwargs["stream_output"] is True + assert update.kwargs["stream_output"] is True def test_interrupted_download_resumes( self, @@ -280,12 +318,15 @@ class TestCheckAndInstall: not in mock_nrf52_ops.rmdir.call_args_list ) assert not (nrf52_dirs.framework / ".resume_failed").exists() - # west update (no init), then pip install zephyr reqs - first = mock_nrf52_ops.run_command_ok.call_args_list[0] - assert "update" in first.args[0] - assert "init" not in first.args[0] - assert first.kwargs["cwd"] == nrf52_dirs.framework - assert mock_nrf52_ops.run_command_ok.call_count == 2 + # west update in the workspace (no init), then pip install zephyr reqs + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "config", + "update", + "list", + "install", + ] + update = mock_nrf52_ops.run_command_ok.call_args_list[1] + assert update.kwargs["cwd"] == nrf52_dirs.framework assert (nrf52_dirs.framework / ".ready").exists() def test_failed_resume_keeps_the_download_once( @@ -297,7 +338,8 @@ class TestCheckAndInstall: dropped again) and is retried on the next build.""" _mark_venv_ready(nrf52_dirs.python_env) _mark_west_initialized(nrf52_dirs.framework) - mock_nrf52_ops.run_command_ok.return_value = False + # config succeeds, the resumed update fails + mock_nrf52_ops.run_command_ok.side_effect = [True, False] with pytest.raises(EsphomeError, match="Can't resume"): check_and_install() @@ -332,16 +374,21 @@ class TestCheckAndInstall: _mark_venv_ready(nrf52_dirs.python_env) _mark_west_initialized(nrf52_dirs.framework) (nrf52_dirs.framework / ".resume_failed").touch() - # resumed update fails; clean init, update and zephyr reqs succeed - mock_nrf52_ops.run_command_ok.side_effect = [False, True, True, True] + # resumed update fails; the clean clone and zephyr reqs succeed + mock_nrf52_ops.run_command_ok.side_effect = [True, False, *[True] * 5] check_and_install() mock_nrf52_ops.rmdir.assert_any_call(nrf52_dirs.framework, msg=ANY) - commands = [c.args[0] for c in mock_nrf52_ops.run_command_ok.call_args_list] - assert "update" in commands[0] - assert "init" in commands[1] - assert "update" in commands[2] + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "config", + "update", + "init", + "config", + "update", + "list", + "install", + ] def test_requirements_install_failure_raises( self, @@ -373,12 +420,372 @@ class TestCheckAndInstall: ) -> None: """Failing west update raises EsphomeError.""" _mark_venv_ready(nrf52_dirs.python_env) - # init succeeds, update fails - mock_nrf52_ops.run_command_ok.side_effect = [True, False] + # init and config succeed, update fails + mock_nrf52_ops.run_command_ok.side_effect = [True, True, False] with pytest.raises(EsphomeError, match="Can't update"): check_and_install() + def test_fresh_install_fetches_only_default_projects( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A fresh clone leaves every west project out except the defaults.""" + _mark_venv_ready(nrf52_dirs.python_env) + + check_and_install() + + assert _project_filter(mock_nrf52_ops.run_command_ok) == ( + "-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr" + ) + stamp = nrf52_dirs.framework / ".west_projects" + assert stamp.read_text(encoding="utf-8").split() == sorted( + DEFAULT_WEST_PROJECTS + ) + + def test_included_project_joins_the_filter( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A project a component includes is fetched with the defaults.""" + _mark_venv_ready(nrf52_dirs.python_env) + include_west_project("oberon-psa-crypto") + + check_and_install() + + assert "+oberon-psa-crypto" in _project_filter( + mock_nrf52_ops.run_command_ok + ).split(",") + + def test_installed_sdk_fetches_a_newly_needed_project( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A finished install gains a project another config left out, keeping + what it already has.""" + _mark_installed(nrf52_dirs) + (nrf52_dirs.framework / ".west_projects").write_text( + "cmsis\nhal_nordic\nnrfxlib\ntinycrypt\nzephyr", encoding="utf-8" + ) + include_west_project("openthread") + + check_and_install() + + # The names are checked before anything is fetched + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "list", + "config", + "update", + ] + wanted = "-.*,+cmsis,+hal_nordic,+nrfxlib,+openthread,+tinycrypt,+zephyr" + assert _project_filter(mock_nrf52_ops.run_command_ok) == wanted + mock_nrf52_ops.rmdir.assert_not_called() + + @pytest.mark.parametrize( + "stamp", + [ + pytest.param(None, id="install_from_before_the_filter"), + pytest.param( + "cmsis\nhal_nordic\nnrfxlib\nopenthread\nzephyr", + id="project_already_fetched", + ), + ], + ) + def test_installed_sdk_with_the_project_fetches_nothing( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + stamp: str | None, + ) -> None: + """No fetch when the install already has every wanted project; an + install without the stamp (west config, no filter) has them all. The + names are still checked, which only reads the manifest.""" + _mark_installed(nrf52_dirs) + _mark_west_initialized(nrf52_dirs.framework) + if stamp is not None: + (nrf52_dirs.framework / ".west_projects").write_text( + stamp, encoding="utf-8" + ) + include_west_project("openthread") + + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"] + + def test_sysbuild_fetches_mcuboot( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """Sysbuild always builds the MCUboot image, so it needs the project.""" + _mark_venv_ready(nrf52_dirs.python_env) + CORE.data[KEY_ZEPHYR] = {KEY_SYSBUILD: True} + + check_and_install() + + assert "+mcuboot" in _project_filter(mock_nrf52_ops.run_command_ok).split(",") + + @pytest.mark.parametrize( + ("sdk_version", "has_cmsis_6"), + [("2.9.2", False), ("3.1.0", True), ("3.2.0", True)], + ) + def test_sdk_3_1_and_later_want_cmsis_6( + self, setup_core: Path, sdk_version: str, has_cmsis_6: bool + ) -> None: + """Zephyr 4.1 moved the Cortex-M core headers to the cmsis_6 module.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse(sdk_version)} + + assert ("cmsis_6" in _wanted_west_projects()) is has_cmsis_6 + + def test_default_projects_never_read_the_stamp( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A build wanting only the defaults has them on any install; an + unreadable stamp shows the check never looked.""" + _mark_installed(nrf52_dirs) + (nrf52_dirs.framework / ".west_projects").mkdir() + + check_and_install() + + mock_nrf52_ops.run_command_ok.assert_not_called() + + def test_failed_project_fetch_raises( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A failed fetch of a newly needed project raises, keeps the stamp and + puts the workspace filter back to the stamp's projects and the defaults.""" + _mark_venv_ready(nrf52_dirs.python_env) + (nrf52_dirs.framework / ".ready").touch() + stamp = nrf52_dirs.framework / ".west_projects" + stamp.write_text("zephyr", encoding="utf-8") + include_west_project("openthread") + # list and config succeed, update fails, the restoring config succeeds + mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True] + + with pytest.raises(EsphomeError, match="Can't update"): + check_and_install() + + assert stamp.read_text(encoding="utf-8") == "zephyr" + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "list", + "config", + "update", + "config", + ] + # The defaults always stay in the restored filter + assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER + + def test_failed_fetch_with_a_lost_stamp_keeps_the_defaults_active( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """An unknown installed set must not leave a filter with every module off.""" + _mark_installed(nrf52_dirs) + _mark_west_initialized(nrf52_dirs.framework) + (nrf52_dirs.framework / ".west" / "config").write_text( + "[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8" + ) + include_west_project("openthread") + # list and config succeed, update fails, the restoring config succeeds + mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True] + + with pytest.raises(EsphomeError, match="Can't update"): + check_and_install() + + assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER + + def test_failed_filter_restore_is_logged( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + caplog: pytest.LogCaptureFixture, + ) -> None: + """When the filter can't be put back after a failed fetch, the user is told.""" + _mark_venv_ready(nrf52_dirs.python_env) + (nrf52_dirs.framework / ".ready").touch() + (nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8") + include_west_project("openthread") + # list and config succeed, update fails, the restoring config fails too + mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, False] + + with pytest.raises(EsphomeError, match="Can't update"): + check_and_install() + + assert "Couldn't put the nRF Connect SDK" in caplog.text + + def test_lost_stamp_on_a_filtered_install_fetches_again( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A workspace with a project filter but no stamp is a filtered install + whose record was lost, so the wanted projects are fetched, not assumed.""" + _mark_installed(nrf52_dirs) + _mark_west_initialized(nrf52_dirs.framework) + (nrf52_dirs.framework / ".west" / "config").write_text( + "[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8" + ) + include_west_project("openthread") + + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "list", + "config", + "update", + ] + + def test_install_waits_for_another_build_holding_the_lock( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + caplog: pytest.LogCaptureFixture, + ) -> None: + """A held lock is waited on, never mistaken for a filesystem that + cannot lock (filelock's Timeout is an OSError as well).""" + from filelock import Timeout + + _mark_installed(nrf52_dirs) + + with ( + caplog.at_level("INFO"), + patch("filelock.FileLock") as file_lock, + ): + file_lock.return_value.acquire.side_effect = [Timeout("install.lock"), None] + check_and_install() + + assert file_lock.return_value.acquire.call_count == 2 + assert "Waiting for another build" in caplog.text + assert "continuing without a lock" not in caplog.text + + def test_install_from_before_the_filter_still_checks_the_names( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """An install with every project fetches nothing, but an unknown name + is still rejected so a mistake shows on every install alike.""" + _mark_installed(nrf52_dirs) + _mark_west_initialized(nrf52_dirs.framework) + include_west_project("openthread") + + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"] + + mock_nrf52_ops.run_command_ok.reset_mock() + mock_nrf52_ops.run_command_ok.return_value = False + include_west_project("no_such_project") + with pytest.raises(EsphomeError, match="west list failed"): + check_and_install() + + def test_install_lock_is_per_sdk_version( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """Installs of different SDK versions do not wait on each other; the + toolchain they share is only locked while it is missing.""" + _mark_installed(nrf52_dirs) + + with patch("filelock.FileLock") as file_lock: + check_and_install() + + lock_files = [Path(c.args[0]).name for c in file_lock.call_args_list] + assert lock_files == [f"sdk-v{_TEST_SDK_VERSION}.lock"] + + def test_install_runs_unlocked_where_the_filesystem_cannot_lock( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + caplog: pytest.LogCaptureFixture, + ) -> None: + """No soft-lock fallback: its marker outlives a killed build and would + hang every later one, so the install goes ahead without a lock.""" + _mark_installed(nrf52_dirs) + + with patch("filelock.FileLock") as file_lock: + file_lock.return_value.acquire.side_effect = OSError( + errno.ENOSYS, "Function not implemented" + ) + check_and_install() + + assert file_lock.call_args.kwargs == {"fallback_to_soft": False} + assert "continuing without a lock" in caplog.text + + def test_unknown_project_raises_before_anything_is_written( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """A filter naming a project the manifest lacks fetches nothing, so + the names are checked once the update resolved the manifest, and an + unknown one is never recorded as installed.""" + _mark_venv_ready(nrf52_dirs.python_env) + include_west_project("no_such_project") + # init, config and update succeed, list fails + mock_nrf52_ops.run_command_ok.side_effect = [True, True, True, False] + + with pytest.raises(EsphomeError, match="west list failed .*no_such_project"): + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "init", + "config", + "update", + "list", + ] + assert "no_such_project" in mock_nrf52_ops.run_command_ok.call_args.args[0] + assert not (nrf52_dirs.framework / ".west_projects").exists() + + def test_unknown_project_on_an_installed_sdk_fetches_nothing( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """On a finished install the names are checked first, so an unknown one + costs no fetch and leaves the workspace as it was.""" + _mark_venv_ready(nrf52_dirs.python_env) + (nrf52_dirs.framework / ".ready").touch() + (nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8") + include_west_project("no_such_project") + # list fails before anything is fetched or changed + mock_nrf52_ops.run_command_ok.side_effect = [False] + + with pytest.raises(EsphomeError, match="west list failed"): + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"] + assert (nrf52_dirs.framework / ".west_projects").read_text( + encoding="utf-8" + ) == "zephyr" + + def test_missing_stamp_and_west_config_fetches_again( + self, + nrf52_dirs: SimpleNamespace, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """With neither the stamp nor .west/config left, nothing says what the + install has, so the wanted projects are fetched rather than assumed.""" + _mark_installed(nrf52_dirs) + include_west_project("openthread") + + check_and_install() + + assert _subcommands(mock_nrf52_ops.run_command_ok) == [ + "list", + "config", + "update", + ] + def test_toolchain_download_passes_platform_substitutions( self, nrf52_dirs: SimpleNamespace,