[nrf52] Fetch only the nRF Connect SDK projects a build needs (#19735)

esphome/components/debug/__init__.py

@@ -1,4 +1,5 @@
 import esphome.codegen as cg
+from esphome.components.nrf52.framework import include_west_project
 from esphome.components.zephyr import zephyr_add_prj_conf
 from esphome.config_helpers import filter_source_files_from_platform
 import esphome.config_validation as cv
@@ -51,7 +52,8 @@ async def to_code(config: ConfigType) -> None:
         zephyr_add_prj_conf("HWINFO", True)
         # gdb thread support
         zephyr_add_prj_conf("DEBUG_THREAD_INFO", True)
-        # RTT
+        # RTT sources live in the segger project
+        include_west_project("segger")
         zephyr_add_prj_conf("USE_SEGGER_RTT", True)
         zephyr_add_prj_conf("RTT_CONSOLE", True)
         zephyr_add_prj_conf("LOG", True)

esphome/components/nrf52/__init__.py

@@ -832,6 +832,26 @@ def _copy_if_exists(src: Path, dst: Path) -> None:
         shutil.copy2(src, dst)
 
 
+def _west_build_command(
+    python_executable: Path, board: str, build_dir: Path, source_dir: Path
+) -> list[str]:
+    return [
+        str(python_executable),
+        "-m",
+        "west",
+        "build",
+        "--pristine=auto",
+        "-b",
+        board,
+        "-d",
+        str(build_dir),
+        str(source_dir),
+        "--",
+        # Only adds -DNDEBUG (Kconfig sets the optimization level); picolibc used to force it
+        "-DCMAKE_BUILD_TYPE=MinSizeRel",
+    ]
+
+
 def run_compile(args, config: ConfigType) -> bool:
     if CORE.using_toolchain_platformio:
         # The actual build is done by PlatformIO (the caller falls through to
@@ -866,18 +886,9 @@ def run_compile(args, config: ConfigType) -> bool:
         _LOGGER.info("Build inputs changed, cleaning %s", build_dir)
         rmtree(build_dir)
 
-    west_cmd = [
-        str(paths["python_executable"]),
-        "-m",
-        "west",
-        "build",
-        "--pristine=auto",
-        "-b",
-        board,
-        "-d",
-        str(build_dir),
-        str(source_dir),
-    ]
+    west_cmd = _west_build_command(
+        paths["python_executable"], board, build_dir, source_dir
+    )
 
     if not run_command_ok(
         west_cmd,

esphome/components/nrf52/clang_tidy.py

@@ -20,9 +20,13 @@ commands.
 
 from __future__ import annotations
 
+from collections.abc import Callable
+from dataclasses import dataclass
 import logging
 from pathlib import Path
 
+from .framework import bluetooth_west_projects
+
 TIDY_PROJECT_NAME = "esphome_tidy"
 
 # Analyzed against the native toolchain's default SDK version
@@ -33,19 +37,36 @@ _TIDY_BOARD = "adafruit_itsybitsy_nrf52840"
 # app target emits a C++ compile command to harvest flags/includes from.
 _TIDY_MAIN_CPP = "int main() { return 0; }\n"
 
-# Kconfig superset enabling every subsystem an ESPHome nrf52 component may
-# use, so the compile commands carry all of their include paths.
-_TIDY_PRJ_CONF = """\
+
+@dataclass(frozen=True)
+class _TidySubsystem:
+    """A subsystem's Kconfig and the west projects its headers come from."""
+
+    name: str
+    prj_conf: str
+    # A callable when the projects differ by SDK version
+    west_projects: tuple[str, ...] | Callable[[], tuple[str, ...]] = ()
+
+
+# Kconfig superset for the compile commands and the projects it needs
+_TIDY_SUBSYSTEMS = (
+    _TidySubsystem(
+        "base",
+        """\
 CONFIG_CPP=y
 CONFIG_STD_CPP20=y
 CONFIG_REQUIRES_FULL_LIBCPP=y
 CONFIG_NEWLIB_LIBC=y
-CONFIG_BT=y
 CONFIG_ADC=y
 # posix (time sets POSIX_CLOCK, socket sets POSIX_API); without it the
 # Zephyr POSIX headers clash with the libc ones under analysis
 CONFIG_POSIX_API=y
-#mcumgr begin
+""",
+    ),
+    _TidySubsystem("bluetooth", "CONFIG_BT=y\n", bluetooth_west_projects),
+    _TidySubsystem(
+        "mcumgr",
+        """\
 CONFIG_NET_BUF=y
 CONFIG_ZCBOR=y
 CONFIG_MCUMGR=y
@@ -60,14 +81,23 @@ CONFIG_MCUMGR_MGMT_NOTIFICATION_HOOKS=y
 CONFIG_MCUMGR_GRP_IMG_STATUS_HOOKS=y
 CONFIG_MCUMGR_GRP_IMG_UPLOAD_CHECK_HOOK=y
 CONFIG_MCUMGR_TRANSPORT_UART=y
-#mcumgr end
-#zigbee begin
+""",
+        ("mcuboot", "zcbor"),
+    ),
+    _TidySubsystem(
+        "zigbee",
+        """\
 CONFIG_ZIGBEE=y
 CONFIG_CRYPTO=y
 CONFIG_NVS=y
 CONFIG_SETTINGS=y
-#zigbee end
-"""
+""",
+    ),
+)
+
+_TIDY_PRJ_CONF = "".join(
+    f"# {subsystem.name}\n{subsystem.prj_conf}" for subsystem in _TIDY_SUBSYSTEMS
+)
 
 
 def _tidy_cmakelists(library_include_dirs: str) -> str:
@@ -166,6 +196,7 @@ def _setup_core(work_dir: Path) -> None:
     from esphome.core import CORE
 
     from . import RECOMMENDED_SDK_NRF_VERSION
+    from .framework import include_west_project
 
     CORE.name = TIDY_PROJECT_NAME
     # config_path's parent is the data-dir root for per-run artifacts. The
@@ -178,6 +209,10 @@ def _setup_core(work_dir: Path) -> None:
     CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse(
         RECOMMENDED_SDK_NRF_VERSION
     )
+    for subsystem in _TIDY_SUBSYSTEMS:
+        projects = subsystem.west_projects
+        for project in projects() if callable(projects) else projects:
+            include_west_project(project)
 
 
 def generate_compile_commands(work_dir: Path, platformio_ini: Path) -> Path:
@@ -237,6 +272,8 @@ def generate_compile_commands(work_dir: Path, platformio_ini: Path) -> Path:
         "zephyr_generated_headers",
         "--",
         "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
+        # As in a real build, so NDEBUG is set
+        "-DCMAKE_BUILD_TYPE=MinSizeRel",
     ]
     if not run_command_ok(
         west_cmd,

esphome/components/nrf52/framework.py

@@ -1,3 +1,7 @@
+from collections.abc import Iterator
+import configparser
+from contextlib import contextmanager
+from dataclasses import dataclass, field
 import hashlib
 import logging
 import os
@@ -7,6 +11,7 @@ import shutil
 import sys
 
 from esphome.build_helpers.tools_cache import SDK_NRF_TOOLS_CACHE, tools_cache_path
+from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR
 import esphome.config_validation as cv
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
@@ -258,7 +263,84 @@ def _patch_uf2conv_escape_sequences(framework_path: Path) -> None:
     tmp.replace(uf2conv)
 
 
-def _west_update(env_python_path: Path, framework_path: Path) -> bool:
+# West projects every build needs; components add others with include_west_project()
+DEFAULT_WEST_PROJECTS = ("cmsis", "hal_nordic", "nrfxlib", "zephyr")
+
+_KEY_NRF52 = "nrf52"
+# The projects a finished install fetched
+_WEST_PROJECTS_FILE = ".west_projects"
+
+
+@dataclass
+class _Nrf52Data:
+    west_projects: set[str] = field(default_factory=lambda: set(DEFAULT_WEST_PROJECTS))
+
+
+def _get_data() -> _Nrf52Data:
+    if _KEY_NRF52 not in CORE.data:
+        CORE.data[_KEY_NRF52] = _Nrf52Data()
+    return CORE.data[_KEY_NRF52]
+
+
+def include_west_project(name: str) -> None:
+    """Fetch a west project left out by default; call from to_code()."""
+    _get_data().west_projects.add(name)
+
+
+def bluetooth_west_projects() -> tuple[str, ...]:
+    """Bluetooth's crypto: TinyCrypt up to SDK 3.1, PSA (mbedtls, Oberon) from 3.2."""
+    if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 2, 0):
+        return ("mbedtls", "oberon-psa-crypto")
+    return ("tinycrypt",)
+
+
+def _wanted_west_projects() -> set[str]:
+    projects = set(_get_data().west_projects)
+    # Zephyr 4.1 moved the Cortex-M core headers to cmsis_6
+    if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 1, 0):
+        projects.add("cmsis_6")
+    # Sysbuild builds the MCUboot image with any bootloader
+    if CORE.data.get(KEY_ZEPHYR, {}).get(KEY_SYSBUILD):
+        projects.add("mcuboot")
+    return projects
+
+
+def _set_project_filter(
+    env_python_path: Path, framework_path: Path, projects: set[str]
+) -> bool:
+    # "--" keeps west from reading the leading "-" as an option
+    project_filter = ",".join(["-.*", *(f"+{p}" for p in sorted(projects))])
+    cmd = [str(env_python_path), "-m", "west", "config", "manifest.project-filter"]
+    return run_command_ok([*cmd, "--", project_filter], cwd=framework_path)
+
+
+def _check_west_projects(
+    env_python_path: Path, framework_path: Path, version: str, projects: set[str]
+) -> None:
+    """Raise when the manifest lacks one of ``projects``; needs zephyr cloned.
+
+    west update quietly skips an unknown name in the filter, west list fails on it.
+    """
+    names = sorted(projects)
+    cmd = [str(env_python_path), "-m", "west", "list", "-f", "{name}", *names]
+    if not run_command_ok(cmd, cwd=framework_path):
+        raise EsphomeError(
+            f"west list failed for the requested nRF Connect SDK {version} projects "
+            f"({', '.join(names)}); a project the manifest does not have is the "
+            "usual cause, see west's output above"
+        )
+
+
+def _west_update(
+    env_python_path: Path,
+    framework_path: Path,
+    version: str,
+    projects: set[str],
+    checked: bool = False,
+) -> bool:
+    """Fetch ``projects``; False when the fetch fails."""
+    if not _set_project_filter(env_python_path, framework_path, projects):
+        return False
     cmd = [
         str(env_python_path),
         "-m",
@@ -267,12 +349,103 @@ def _west_update(env_python_path: Path, framework_path: Path) -> bool:
         "--narrow",
         "--fetch-opt=--depth=1",
     ]
-    # Streamed so the per-project progress of the long clone reaches the log
-    return run_command_ok(cmd, cwd=framework_path, stream_output=True)
+    # Streamed so the clone's progress reaches the log
+    if not run_command_ok(cmd, cwd=framework_path, stream_output=True):
+        return False
+    if not checked:
+        _check_west_projects(env_python_path, framework_path, version, projects)
+    (framework_path / _WEST_PROJECTS_FILE).write_text(
+        "\n".join(sorted(projects)), encoding="utf-8"
+    )
+    return True
+
+
+def _installed_west_projects(framework_path: Path) -> set[str] | None:
+    """The projects a finished install fetched; None when it has every project."""
+    try:
+        stamp = (framework_path / _WEST_PROJECTS_FILE).read_text(encoding="utf-8")
+    except FileNotFoundError:
+        pass
+    else:
+        return set(stamp.split())
+    # No stamp: an install from before the filter has every project, a filtered
+    # one that lost its stamp fetches again
+    config = configparser.ConfigParser()
+    if not config.read(framework_path / ".west" / "config", encoding="utf-8"):
+        return set()
+    if config.has_option("manifest", "project-filter"):
+        return set()
+    return None
+
+
+def _restore_project_filter(
+    env_python_path: Path, framework_path: Path, version: str, installed: set[str]
+) -> None:
+    """Put the filter back to the stamp's projects; the defaults always stay in."""
+    projects = installed | set(DEFAULT_WEST_PROJECTS)
+    if not _set_project_filter(env_python_path, framework_path, projects):
+        _LOGGER.warning(
+            "Couldn't put the nRF Connect SDK %s project filter back; "
+            "the next build that fetches a project sets it again",
+            version,
+        )
+
+
+# Lock wait slices, so Ctrl-C stays responsive
+_INSTALL_LOCK_POLL = 1
+
+
+@contextmanager
+def _install_lock(name: str) -> Iterator[None]:
+    """Serialize a shared install step across builds running at once."""
+    from filelock import FileLock, Timeout
+
+    lock_path = get_sdk_nrf_tools_path() / f"{name}.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    # No soft lock: its marker outlives a killed build and hangs every later one
+    lock = FileLock(str(lock_path), fallback_to_soft=False)
+    waiting = False
+    while True:
+        try:
+            lock.acquire(timeout=_INSTALL_LOCK_POLL)
+            break
+        except Timeout:  # before OSError, which it subclasses
+            if not waiting:
+                waiting = True
+                _LOGGER.info("Waiting for another build installing %s ...", name)
+        except OSError as err:
+            _LOGGER.warning(
+                "Can't lock %s (%s), continuing without a lock", lock_path, err
+            )
+            break
+    try:
+        yield
+    finally:
+        lock.release()
+
+
+def _fetch_missing_west_projects(
+    env_python_path: Path, framework_path: Path, version: str, projects: set[str]
+) -> None:
+    """Fetch the wanted projects a finished install lacks; it only ever gains."""
+    if projects <= set(DEFAULT_WEST_PROJECTS):
+        return
+    installed = _installed_west_projects(framework_path)
+    # Before the fetch, so an unknown name costs nothing on any install
+    _check_west_projects(env_python_path, framework_path, version, projects)
+    if installed is None or not (missing := projects - installed):
+        return
+    _LOGGER.info(
+        "Fetching nRF Connect SDK %s projects: %s", version, ", ".join(sorted(missing))
+    )
+    wanted = installed | projects
+    if not _west_update(env_python_path, framework_path, version, wanted, checked=True):
+        _restore_project_filter(env_python_path, framework_path, version, installed)
+        raise EsphomeError(f"Can't update nRF Connect SDK {version}")
 
 
 def _install_framework(
-    env_python_path: Path, framework_path: Path, version: str
+    env_python_path: Path, framework_path: Path, version: str, projects: set[str]
 ) -> None:
     """Clone the nRF Connect SDK into ``framework_path`` with west.
 
@@ -289,7 +462,7 @@ def _install_framework(
     initialized = (framework_path / ".west" / "config").is_file()
     if initialized and not (framework_path / ".ready").exists():
         _LOGGER.info("Resuming the nRF Connect SDK %s download ...", version)
-        if _west_update(env_python_path, framework_path):
+        if _west_update(env_python_path, framework_path, version, projects):
             resume_failed.unlink(missing_ok=True)
             return
         if not resume_failed.exists():
@@ -318,12 +491,17 @@ def _install_framework(
     if not run_command_ok(cmd, stream_output=True):
         raise EsphomeError(f"Can't initialize nRF Connect SDK {version}")
     _LOGGER.info("Updating nRF Connect SDK %s (this may take a while) ...", version)
-    if not _west_update(env_python_path, framework_path):
+    if not _west_update(env_python_path, framework_path, version, projects):
         raise EsphomeError(f"Can't update nRF Connect SDK {version}")
 
 
 def check_and_install() -> None:
     version = _get_version_str()
+    with _install_lock(f"sdk-{version}"):
+        _check_and_install(version)
+
+
+def _check_and_install(version: str) -> None:
     python_env_path = _get_python_env_path(version)
     env_python_path = get_python_env_executable_path(python_env_path, "python")
     sentinel = python_env_path / ".ready"
@@ -354,12 +532,15 @@ def check_and_install() -> None:
     framework_path = _get_framework_path(version)
     sentinel = framework_path / ".ready"
     zephyr_reqs = framework_path / "zephyr" / "scripts" / "requirements.txt"
+    projects = _wanted_west_projects()
     if not sentinel.exists() or not zephyr_reqs.exists():
-        _install_framework(env_python_path, framework_path, version)
+        _install_framework(env_python_path, framework_path, version, projects)
         framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
         if framework_ver < cv.Version(2, 9, 2):
             _patch_uf2conv_escape_sequences(framework_path)
         sentinel.touch()
+    else:
+        _fetch_missing_west_projects(env_python_path, framework_path, version, projects)
 
     zephyr_sentinel = python_env_path / ".zephyr_reqs_ready"
     if (
@@ -380,6 +561,13 @@ def check_and_install() -> None:
             raise EsphomeError(f"Install Zephyr requirements for {version} failure")
         zephyr_sentinel.touch()
 
+    # Shared by every SDK version; locked only while missing
+    if not (_get_toolchain_path(TOOLCHAIN_VERSION) / ".ready").exists():
+        with _install_lock(f"toolchain-{TOOLCHAIN_VERSION}"):
+            _install_toolchain()
+
+
+def _install_toolchain() -> None:
     toolchains_dir = _get_toolchain_path(TOOLCHAIN_VERSION)
     sentinel = toolchains_dir / ".ready"
     if not sentinel.exists():

esphome/components/openthread/__init__.py

@@ -19,6 +19,7 @@ from esphome.components.esp32 import (
 )
 from esphome.components.mdns import MDNSComponent, enable_mdns_storage
 from esphome.components.network import add_use_address
+from esphome.components.nrf52.framework import include_west_project
 from esphome.components.zephyr import zephyr_add_prj_conf
 from esphome.config_helpers import filter_source_files_from_platform
 import esphome.config_validation as cv
@@ -30,6 +31,8 @@ from esphome.const import (
     CONF_LOG_LEVEL,
     CONF_OUTPUT_POWER,
     CONF_USE_ADDRESS,
+    KEY_CORE,
+    KEY_FRAMEWORK_VERSION,
     PLATFORM_ESP32,
     PlatformFramework,
 )
@@ -317,6 +320,11 @@ async def to_code(config: ConfigType) -> None:
     if CORE.is_esp32:
         set_sdkconfig_options(config)
     elif CORE.using_zephyr:
+        # Crypto through PSA: mbedtls, plus Oberon from SDK 2.7
+        include_west_project("mbedtls")
+        include_west_project("openthread")
+        if CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(2, 7, 0):
+            include_west_project("oberon-psa-crypto")
         zephyr_add_prj_conf("NET_L2_OPENTHREAD", True)
         zephyr_add_prj_conf(
             f"OPENTHREAD_NORDIC_LIBRARY_{config.get(CONF_DEVICE_TYPE)}", True

esphome/components/zephyr_ble_server/__init__.py

@@ -1,5 +1,9 @@
 from esphome import automation
 import esphome.codegen as cg
+from esphome.components.nrf52.framework import (
+    bluetooth_west_projects,
+    include_west_project,
+)
 from esphome.components.zephyr import zephyr_add_prj_conf
 import esphome.config_validation as cv
 from esphome.const import CONF_ID, Framework
@@ -36,6 +40,8 @@ _CALLBACK_AUTOMATIONS = (
 async def to_code(config: ConfigType) -> None:
     var = cg.new_Pvariable(config[CONF_ID])
     zephyr_add_prj_conf("BT", True)
+    for project in bluetooth_west_projects():
+        include_west_project(project)
     zephyr_add_prj_conf("BT_PERIPHERAL", True)
     zephyr_add_prj_conf("BT_RX_STACK_SIZE", 1536)
     zephyr_add_prj_conf("BT_DEVICE_NAME", CORE.name)

esphome/components/zephyr_mcumgr/ota/__init__.py

@@ -1,5 +1,6 @@
 import esphome.codegen as cg
 from esphome.components.nrf52.boards import BOOTLOADER_CONFIG
+from esphome.components.nrf52.framework import include_west_project
 from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code
 from esphome.components.zephyr import (
     HexValue,
@@ -115,6 +116,9 @@ async def to_code(config: ConfigType) -> None:
 
     zephyr_add_prj_conf("NET_BUF", True)
     zephyr_add_prj_conf("ZCBOR", True)
+    include_west_project("zcbor")
+    # The image manager includes MCUboot headers with any bootloader
+    include_west_project("mcuboot")
     zephyr_add_prj_conf("MCUMGR", True)
 
     zephyr_add_prj_conf("MCUMGR_GRP_IMG", True)

tests/unit_tests/test_nrf52_build.py

@@ -0,0 +1,23 @@
+"""Tests for the nRF52 native west build command."""
+
+from pathlib import Path
+
+from esphome.components.nrf52 import _west_build_command
+
+
+def test_west_build_sets_the_cmake_build_type() -> None:
+    """The picolibc module used to force MinSizeRel, and with it the -DNDEBUG
+    that keeps libc assert() out of the image; the build sets it itself now."""
+    python = Path("/penv/python")
+    cmd = _west_build_command(
+        python,
+        "adafruit_feather_nrf52840",
+        Path("/build/pio"),
+        Path("/build/zephyr"),
+    )
+
+    # str(), not a literal: the separator differs on Windows
+    assert cmd[:4] == [str(python), "-m", "west", "build"]
+    assert "-b" in cmd and "adafruit_feather_nrf52840" in cmd
+    # CMake arguments come after west's own
+    assert cmd[cmd.index("--") + 1 :] == ["-DCMAKE_BUILD_TYPE=MinSizeRel"]

tests/unit_tests/test_nrf52_framework.py

@@ -1,5 +1,6 @@
 """Tests for esphome.components.nrf52.framework helpers."""
 
+import errno
 import hashlib
 import os
 from pathlib import Path
@@ -14,16 +15,20 @@ from esphome.components.nrf52 import _resolve_toolchain
 from esphome.components.nrf52.framework import (
     _PLATFORMIO_PENV_REQUIREMENTS,
     _REQUIREMENTS,
+    DEFAULT_WEST_PROJECTS,
     TOOLCHAIN_VERSION,
     _get_penv_site_packages,
     _get_platformio_penv_path,
     _get_toolchain_platform_info,
     _needs_venv_rebuild,
+    _wanted_west_projects,
     check_and_install,
     get_build_env,
     get_sdk_nrf_tools_path,
+    include_west_project,
     setup_platformio_python_env,
 )
+from esphome.components.zephyr.const import KEY_SYSBUILD, KEY_ZEPHYR
 import esphome.config_validation as cv
 from esphome.config_validation import Version
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain
@@ -76,6 +81,8 @@ def test_get_toolchain_platform_info(
 # ---------------------------------------------------------------------------
 
 _TEST_SDK_VERSION = "2.9.0"
+# The filter that keeps only DEFAULT_WEST_PROJECTS
+_DEFAULTS_FILTER = "-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr"
 
 
 @pytest.fixture
@@ -142,6 +149,33 @@ def _touch_penv_python(penv: Path) -> None:
     python.touch()
 
 
+def _subcommand(cmd: list[str]) -> str:
+    tool = "west" if "west" in cmd else "pip"
+    return cmd[cmd.index(tool) + 1]
+
+
+def _subcommands(run_command_ok) -> list[str]:
+    """The west or pip subcommand of each command run, in order."""
+    return [_subcommand(c.args[0]) for c in run_command_ok.call_args_list]
+
+
+def _project_filter(run_command_ok) -> str:
+    """The manifest.project-filter value of the last ``west config`` run."""
+    for west_call in reversed(run_command_ok.call_args_list):
+        cmd = west_call.args[0]
+        if "manifest.project-filter" in cmd:
+            return cmd[-1]
+    raise AssertionError("no west config command ran")
+
+
+def _mark_installed(dirs: SimpleNamespace) -> None:
+    """Every install step finished: venv, zephyr requirements, SDK, toolchain."""
+    _mark_venv_ready(dirs.python_env)
+    (dirs.python_env / ".zephyr_reqs_ready").touch()
+    (dirs.framework / ".ready").touch()
+    (dirs.toolchain / ".ready").touch()
+
+
 def _mark_venv_ready(python_env: Path) -> None:
     """Write the venv sentinel with the current requirements hash and a
     present interpreter so the rebuild gate passes."""
@@ -157,10 +191,7 @@ class TestCheckAndInstall:
         mock_nrf52_ops: SimpleNamespace,
     ) -> None:
         """All three sentinels present → nothing downloaded or compiled."""
-        _mark_venv_ready(nrf52_dirs.python_env)
-        (nrf52_dirs.python_env / ".zephyr_reqs_ready").touch()
-        (nrf52_dirs.framework / ".ready").touch()
-        (nrf52_dirs.toolchain / ".ready").touch()
+        _mark_installed(nrf52_dirs)
 
         check_and_install()
 
@@ -195,8 +226,14 @@ class TestCheckAndInstall:
         check_and_install()
 
         mock_nrf52_ops.create_venv.assert_called_once()
-        # pip install requirements, west init, west update, pip install zephyr reqs
-        assert mock_nrf52_ops.run_command_ok.call_count == 4
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "install",  # requirements
+            "init",
+            "config",
+            "update",
+            "list",
+            "install",  # zephyr requirements
+        ]
         # minimal SDK + per-arch toolchain
         assert mock_nrf52_ops.download_from_mirrors.call_count == 2
         assert mock_nrf52_ops.archive_extract_all.call_count == 2
@@ -216,8 +253,13 @@ class TestCheckAndInstall:
         check_and_install()
 
         mock_nrf52_ops.create_venv.assert_not_called()
-        # west init, west update, pip install zephyr reqs
-        assert mock_nrf52_ops.run_command_ok.call_count == 3
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "init",
+            "config",
+            "update",
+            "list",
+            "install",
+        ]
         # minimal SDK + per-arch toolchain
         assert mock_nrf52_ops.download_from_mirrors.call_count == 2
 
@@ -249,16 +291,12 @@ class TestCheckAndInstall:
 
         check_and_install()
 
-        init_cmd, update_cmd = (
-            call.args[0] for call in mock_nrf52_ops.run_command_ok.call_args_list[:2]
-        )
-        assert "init" in init_cmd
-        assert "-o=--depth=1" in init_cmd
-        assert "update" in update_cmd
-        assert "--fetch-opt=--depth=1" in update_cmd
+        init, _, update = mock_nrf52_ops.run_command_ok.call_args_list[:3]
+        assert "-o=--depth=1" in init.args[0]
+        assert "--fetch-opt=--depth=1" in update.args[0]
         # Streamed, so the long clone's progress reaches the log
-        for west_call in mock_nrf52_ops.run_command_ok.call_args_list[:2]:
-            assert west_call.kwargs["stream_output"] is True
+        assert init.kwargs["stream_output"] is True
+        assert update.kwargs["stream_output"] is True
 
     def test_interrupted_download_resumes(
         self,
@@ -280,12 +318,15 @@ class TestCheckAndInstall:
             not in mock_nrf52_ops.rmdir.call_args_list
         )
         assert not (nrf52_dirs.framework / ".resume_failed").exists()
-        # west update (no init), then pip install zephyr reqs
-        first = mock_nrf52_ops.run_command_ok.call_args_list[0]
-        assert "update" in first.args[0]
-        assert "init" not in first.args[0]
-        assert first.kwargs["cwd"] == nrf52_dirs.framework
-        assert mock_nrf52_ops.run_command_ok.call_count == 2
+        # west update in the workspace (no init), then pip install zephyr reqs
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "config",
+            "update",
+            "list",
+            "install",
+        ]
+        update = mock_nrf52_ops.run_command_ok.call_args_list[1]
+        assert update.kwargs["cwd"] == nrf52_dirs.framework
         assert (nrf52_dirs.framework / ".ready").exists()
 
     def test_failed_resume_keeps_the_download_once(
@@ -297,7 +338,8 @@ class TestCheckAndInstall:
         dropped again) and is retried on the next build."""
         _mark_venv_ready(nrf52_dirs.python_env)
         _mark_west_initialized(nrf52_dirs.framework)
-        mock_nrf52_ops.run_command_ok.return_value = False
+        # config succeeds, the resumed update fails
+        mock_nrf52_ops.run_command_ok.side_effect = [True, False]
 
         with pytest.raises(EsphomeError, match="Can't resume"):
             check_and_install()
@@ -332,16 +374,21 @@ class TestCheckAndInstall:
         _mark_venv_ready(nrf52_dirs.python_env)
         _mark_west_initialized(nrf52_dirs.framework)
         (nrf52_dirs.framework / ".resume_failed").touch()
-        # resumed update fails; clean init, update and zephyr reqs succeed
-        mock_nrf52_ops.run_command_ok.side_effect = [False, True, True, True]
+        # resumed update fails; the clean clone and zephyr reqs succeed
+        mock_nrf52_ops.run_command_ok.side_effect = [True, False, *[True] * 5]
 
         check_and_install()
 
         mock_nrf52_ops.rmdir.assert_any_call(nrf52_dirs.framework, msg=ANY)
-        commands = [c.args[0] for c in mock_nrf52_ops.run_command_ok.call_args_list]
-        assert "update" in commands[0]
-        assert "init" in commands[1]
-        assert "update" in commands[2]
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "config",
+            "update",
+            "init",
+            "config",
+            "update",
+            "list",
+            "install",
+        ]
 
     def test_requirements_install_failure_raises(
         self,
@@ -373,12 +420,372 @@ class TestCheckAndInstall:
     ) -> None:
         """Failing west update raises EsphomeError."""
         _mark_venv_ready(nrf52_dirs.python_env)
-        # init succeeds, update fails
-        mock_nrf52_ops.run_command_ok.side_effect = [True, False]
+        # init and config succeed, update fails
+        mock_nrf52_ops.run_command_ok.side_effect = [True, True, False]
 
         with pytest.raises(EsphomeError, match="Can't update"):
             check_and_install()
 
+    def test_fresh_install_fetches_only_default_projects(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A fresh clone leaves every west project out except the defaults."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+
+        check_and_install()
+
+        assert _project_filter(mock_nrf52_ops.run_command_ok) == (
+            "-.*,+cmsis,+hal_nordic,+nrfxlib,+zephyr"
+        )
+        stamp = nrf52_dirs.framework / ".west_projects"
+        assert stamp.read_text(encoding="utf-8").split() == sorted(
+            DEFAULT_WEST_PROJECTS
+        )
+
+    def test_included_project_joins_the_filter(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A project a component includes is fetched with the defaults."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        include_west_project("oberon-psa-crypto")
+
+        check_and_install()
+
+        assert "+oberon-psa-crypto" in _project_filter(
+            mock_nrf52_ops.run_command_ok
+        ).split(",")
+
+    def test_installed_sdk_fetches_a_newly_needed_project(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A finished install gains a project another config left out, keeping
+        what it already has."""
+        _mark_installed(nrf52_dirs)
+        (nrf52_dirs.framework / ".west_projects").write_text(
+            "cmsis\nhal_nordic\nnrfxlib\ntinycrypt\nzephyr", encoding="utf-8"
+        )
+        include_west_project("openthread")
+
+        check_and_install()
+
+        # The names are checked before anything is fetched
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "list",
+            "config",
+            "update",
+        ]
+        wanted = "-.*,+cmsis,+hal_nordic,+nrfxlib,+openthread,+tinycrypt,+zephyr"
+        assert _project_filter(mock_nrf52_ops.run_command_ok) == wanted
+        mock_nrf52_ops.rmdir.assert_not_called()
+
+    @pytest.mark.parametrize(
+        "stamp",
+        [
+            pytest.param(None, id="install_from_before_the_filter"),
+            pytest.param(
+                "cmsis\nhal_nordic\nnrfxlib\nopenthread\nzephyr",
+                id="project_already_fetched",
+            ),
+        ],
+    )
+    def test_installed_sdk_with_the_project_fetches_nothing(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+        stamp: str | None,
+    ) -> None:
+        """No fetch when the install already has every wanted project; an
+        install without the stamp (west config, no filter) has them all. The
+        names are still checked, which only reads the manifest."""
+        _mark_installed(nrf52_dirs)
+        _mark_west_initialized(nrf52_dirs.framework)
+        if stamp is not None:
+            (nrf52_dirs.framework / ".west_projects").write_text(
+                stamp, encoding="utf-8"
+            )
+        include_west_project("openthread")
+
+        check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
+
+    def test_sysbuild_fetches_mcuboot(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """Sysbuild always builds the MCUboot image, so it needs the project."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        CORE.data[KEY_ZEPHYR] = {KEY_SYSBUILD: True}
+
+        check_and_install()
+
+        assert "+mcuboot" in _project_filter(mock_nrf52_ops.run_command_ok).split(",")
+
+    @pytest.mark.parametrize(
+        ("sdk_version", "has_cmsis_6"),
+        [("2.9.2", False), ("3.1.0", True), ("3.2.0", True)],
+    )
+    def test_sdk_3_1_and_later_want_cmsis_6(
+        self, setup_core: Path, sdk_version: str, has_cmsis_6: bool
+    ) -> None:
+        """Zephyr 4.1 moved the Cortex-M core headers to the cmsis_6 module."""
+        CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse(sdk_version)}
+
+        assert ("cmsis_6" in _wanted_west_projects()) is has_cmsis_6
+
+    def test_default_projects_never_read_the_stamp(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A build wanting only the defaults has them on any install; an
+        unreadable stamp shows the check never looked."""
+        _mark_installed(nrf52_dirs)
+        (nrf52_dirs.framework / ".west_projects").mkdir()
+
+        check_and_install()
+
+        mock_nrf52_ops.run_command_ok.assert_not_called()
+
+    def test_failed_project_fetch_raises(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A failed fetch of a newly needed project raises, keeps the stamp and
+        puts the workspace filter back to the stamp's projects and the defaults."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        (nrf52_dirs.framework / ".ready").touch()
+        stamp = nrf52_dirs.framework / ".west_projects"
+        stamp.write_text("zephyr", encoding="utf-8")
+        include_west_project("openthread")
+        # list and config succeed, update fails, the restoring config succeeds
+        mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True]
+
+        with pytest.raises(EsphomeError, match="Can't update"):
+            check_and_install()
+
+        assert stamp.read_text(encoding="utf-8") == "zephyr"
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "list",
+            "config",
+            "update",
+            "config",
+        ]
+        # The defaults always stay in the restored filter
+        assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER
+
+    def test_failed_fetch_with_a_lost_stamp_keeps_the_defaults_active(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """An unknown installed set must not leave a filter with every module off."""
+        _mark_installed(nrf52_dirs)
+        _mark_west_initialized(nrf52_dirs.framework)
+        (nrf52_dirs.framework / ".west" / "config").write_text(
+            "[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8"
+        )
+        include_west_project("openthread")
+        # list and config succeed, update fails, the restoring config succeeds
+        mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, True]
+
+        with pytest.raises(EsphomeError, match="Can't update"):
+            check_and_install()
+
+        assert _project_filter(mock_nrf52_ops.run_command_ok) == _DEFAULTS_FILTER
+
+    def test_failed_filter_restore_is_logged(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """When the filter can't be put back after a failed fetch, the user is told."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        (nrf52_dirs.framework / ".ready").touch()
+        (nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8")
+        include_west_project("openthread")
+        # list and config succeed, update fails, the restoring config fails too
+        mock_nrf52_ops.run_command_ok.side_effect = [True, True, False, False]
+
+        with pytest.raises(EsphomeError, match="Can't update"):
+            check_and_install()
+
+        assert "Couldn't put the nRF Connect SDK" in caplog.text
+
+    def test_lost_stamp_on_a_filtered_install_fetches_again(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A workspace with a project filter but no stamp is a filtered install
+        whose record was lost, so the wanted projects are fetched, not assumed."""
+        _mark_installed(nrf52_dirs)
+        _mark_west_initialized(nrf52_dirs.framework)
+        (nrf52_dirs.framework / ".west" / "config").write_text(
+            "[manifest]\nproject-filter = -.*,+zephyr\n", encoding="utf-8"
+        )
+        include_west_project("openthread")
+
+        check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "list",
+            "config",
+            "update",
+        ]
+
+    def test_install_waits_for_another_build_holding_the_lock(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """A held lock is waited on, never mistaken for a filesystem that
+        cannot lock (filelock's Timeout is an OSError as well)."""
+        from filelock import Timeout
+
+        _mark_installed(nrf52_dirs)
+
+        with (
+            caplog.at_level("INFO"),
+            patch("filelock.FileLock") as file_lock,
+        ):
+            file_lock.return_value.acquire.side_effect = [Timeout("install.lock"), None]
+            check_and_install()
+
+        assert file_lock.return_value.acquire.call_count == 2
+        assert "Waiting for another build" in caplog.text
+        assert "continuing without a lock" not in caplog.text
+
+    def test_install_from_before_the_filter_still_checks_the_names(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """An install with every project fetches nothing, but an unknown name
+        is still rejected so a mistake shows on every install alike."""
+        _mark_installed(nrf52_dirs)
+        _mark_west_initialized(nrf52_dirs.framework)
+        include_west_project("openthread")
+
+        check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
+
+        mock_nrf52_ops.run_command_ok.reset_mock()
+        mock_nrf52_ops.run_command_ok.return_value = False
+        include_west_project("no_such_project")
+        with pytest.raises(EsphomeError, match="west list failed"):
+            check_and_install()
+
+    def test_install_lock_is_per_sdk_version(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """Installs of different SDK versions do not wait on each other; the
+        toolchain they share is only locked while it is missing."""
+        _mark_installed(nrf52_dirs)
+
+        with patch("filelock.FileLock") as file_lock:
+            check_and_install()
+
+        lock_files = [Path(c.args[0]).name for c in file_lock.call_args_list]
+        assert lock_files == [f"sdk-v{_TEST_SDK_VERSION}.lock"]
+
+    def test_install_runs_unlocked_where_the_filesystem_cannot_lock(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """No soft-lock fallback: its marker outlives a killed build and would
+        hang every later one, so the install goes ahead without a lock."""
+        _mark_installed(nrf52_dirs)
+
+        with patch("filelock.FileLock") as file_lock:
+            file_lock.return_value.acquire.side_effect = OSError(
+                errno.ENOSYS, "Function not implemented"
+            )
+            check_and_install()
+
+        assert file_lock.call_args.kwargs == {"fallback_to_soft": False}
+        assert "continuing without a lock" in caplog.text
+
+    def test_unknown_project_raises_before_anything_is_written(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """A filter naming a project the manifest lacks fetches nothing, so
+        the names are checked once the update resolved the manifest, and an
+        unknown one is never recorded as installed."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        include_west_project("no_such_project")
+        # init, config and update succeed, list fails
+        mock_nrf52_ops.run_command_ok.side_effect = [True, True, True, False]
+
+        with pytest.raises(EsphomeError, match="west list failed .*no_such_project"):
+            check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "init",
+            "config",
+            "update",
+            "list",
+        ]
+        assert "no_such_project" in mock_nrf52_ops.run_command_ok.call_args.args[0]
+        assert not (nrf52_dirs.framework / ".west_projects").exists()
+
+    def test_unknown_project_on_an_installed_sdk_fetches_nothing(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """On a finished install the names are checked first, so an unknown one
+        costs no fetch and leaves the workspace as it was."""
+        _mark_venv_ready(nrf52_dirs.python_env)
+        (nrf52_dirs.framework / ".ready").touch()
+        (nrf52_dirs.framework / ".west_projects").write_text("zephyr", encoding="utf-8")
+        include_west_project("no_such_project")
+        # list fails before anything is fetched or changed
+        mock_nrf52_ops.run_command_ok.side_effect = [False]
+
+        with pytest.raises(EsphomeError, match="west list failed"):
+            check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == ["list"]
+        assert (nrf52_dirs.framework / ".west_projects").read_text(
+            encoding="utf-8"
+        ) == "zephyr"
+
+    def test_missing_stamp_and_west_config_fetches_again(
+        self,
+        nrf52_dirs: SimpleNamespace,
+        mock_nrf52_ops: SimpleNamespace,
+    ) -> None:
+        """With neither the stamp nor .west/config left, nothing says what the
+        install has, so the wanted projects are fetched rather than assumed."""
+        _mark_installed(nrf52_dirs)
+        include_west_project("openthread")
+
+        check_and_install()
+
+        assert _subcommands(mock_nrf52_ops.run_command_ok) == [
+            "list",
+            "config",
+            "update",
+        ]
+
     def test_toolchain_download_passes_platform_substitutions(
         self,
         nrf52_dirs: SimpleNamespace,