github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-09-28 15:30:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

[nrf52] Stream the SDK clone and resume it after an interruption (#19731)

Browse Source
This commit is contained in:
J. Nick Koston
2026-09-27 05:04:41 +01:00
committed by GitHub
parent 2d17ad2c91
commit 5d415ad505
2 changed files with 158 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/components/nrf52/framework.py
+65 -27
View File
@@ -247,6 +247,70 @@ def _patch_uf2conv_escape_sequences(framework_path: Path) -> None:
tmp.replace(uf2conv)
def _west_update(env_python_path: Path, framework_path: Path) -> bool:
cmd = [
str(env_python_path),
"-m",
"west",
"update",
"--narrow",
"--fetch-opt=--depth=1",
]
# Streamed so the per-project progress of the long clone reaches the log
return run_command_ok(cmd, cwd=framework_path, stream_output=True)
def _install_framework(
env_python_path: Path, framework_path: Path, version: str
) -> None:
"""Clone the nRF Connect SDK into ``framework_path`` with west.
A download cut short after ``west init`` leaves the workspace behind;
rerunning ``west update`` there only fetches what is missing, so it resumes
instead of cloning about 2 GB again. A resume that fails keeps what was
fetched (a flaky network is the likely cause) and is retried on the next
build; only a second failure in a row starts over clean.
"""
resume_failed = framework_path / ".resume_failed"
# Resume only a workspace whose ``west init`` finished (it writes the
# config last). ``.ready`` with missing requirements is a damaged install,
# not an interrupted one, so it goes the clean way.
initialized = (framework_path / ".west" / "config").is_file()
if initialized and not (framework_path / ".ready").exists():
_LOGGER.info("Resuming the nRF Connect SDK %s download ...", version)
if _west_update(env_python_path, framework_path):
resume_failed.unlink(missing_ok=True)
return
if not resume_failed.exists():
resume_failed.touch()
raise EsphomeError(
f"Can't resume the nRF Connect SDK {version} download; "
"the next build retries it"
)
_LOGGER.warning(
"Resuming failed again; downloading nRF Connect SDK %s anew", version
)
rmdir(framework_path, msg=f"Clean up {version} framework environment")
_LOGGER.info("Initializing nRF Connect SDK %s ...", version)
cmd = [
str(env_python_path),
"-m",
"west",
"init",
"-m",
"https://github.com/nrfconnect/sdk-nrf",
"-o=--depth=1",
"--mr",
version,
str(framework_path),
]
if not run_command_ok(cmd, stream_output=True):
raise EsphomeError(f"Can't initialize nRF Connect SDK {version}")
_LOGGER.info("Updating nRF Connect SDK %s (this may take a while) ...", version)
if not _west_update(env_python_path, framework_path):
raise EsphomeError(f"Can't update nRF Connect SDK {version}")
def check_and_install() -> None:
version = _get_version_str()
python_env_path = _get_python_env_path(version)
@@ -280,33 +344,7 @@ def check_and_install() -> None:
sentinel = framework_path / ".ready"
zephyr_reqs = framework_path / "zephyr" / "scripts" / "requirements.txt"
if not sentinel.exists() or not zephyr_reqs.exists():
rmdir(framework_path, msg=f"Clean up {version} framework environment")
_LOGGER.info("Initializing nRF Connect SDK %s ...", version)
cmd = [
str(env_python_path),
"-m",
"west",
"init",
"-m",
"https://github.com/nrfconnect/sdk-nrf",
"-o=--depth=1",
"--mr",
version,
str(framework_path),
]
if not run_command_ok(cmd):
raise EsphomeError(f"Can't initialize nRF Connect SDK {version}")
_LOGGER.info("Updating nRF Connect SDK %s (this may take a while) ...", version)
cmd = [
str(env_python_path),
"-m",
"west",
"update",
"--narrow",
"--fetch-opt=--depth=1",
]
if not run_command_ok(cmd, cwd=framework_path):
raise EsphomeError(f"Can't update nRF Connect SDK {version}")
_install_framework(env_python_path, framework_path, version)
framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
if framework_ver < cv.Version(2, 9, 2):
_patch_uf2conv_escape_sequences(framework_path)
tests/unit_tests/test_nrf52_framework.py
+93 -1
View File
@@ -5,7 +5,7 @@ import os
from pathlib import Path
import sys
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import ANY, call, patch
import platformdirs
import pytest
@@ -129,6 +129,12 @@ def mock_nrf52_ops():
# ---------------------------------------------------------------------------
def _mark_west_initialized(framework: Path) -> None:
"""What a finished ``west init`` leaves behind."""
(framework / ".west").mkdir()
(framework / ".west" / "config").touch()
def _touch_penv_python(penv: Path) -> None:
"""Create the interpreter file so the rebuild gate sees a live venv."""
python = get_python_env_executable_path(penv, "python")
@@ -250,6 +256,92 @@ class TestCheckAndInstall:
assert "-o=--depth=1" in init_cmd
assert "update" in update_cmd
assert "--fetch-opt=--depth=1" in update_cmd
# Streamed, so the long clone's progress reaches the log
for west_call in mock_nrf52_ops.run_command_ok.call_args_list[:2]:
assert west_call.kwargs["stream_output"] is True
def test_interrupted_download_resumes(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A workspace left by a cut-short download is updated in place, not
wiped and cloned again."""
_mark_venv_ready(nrf52_dirs.python_env)
_mark_west_initialized(nrf52_dirs.framework)
# A marker left by an earlier failed resume
(nrf52_dirs.framework / ".resume_failed").touch()
check_and_install()
assert (
call(nrf52_dirs.framework, msg=ANY)
not in mock_nrf52_ops.rmdir.call_args_list
)
assert not (nrf52_dirs.framework / ".resume_failed").exists()
# west update (no init), then pip install zephyr reqs
first = mock_nrf52_ops.run_command_ok.call_args_list[0]
assert "update" in first.args[0]
assert "init" not in first.args[0]
assert first.kwargs["cwd"] == nrf52_dirs.framework
assert mock_nrf52_ops.run_command_ok.call_count == 2
assert (nrf52_dirs.framework / ".ready").exists()
def test_failed_resume_keeps_the_download_once(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A first failed resume keeps what was fetched (the network likely
dropped again) and is retried on the next build."""
_mark_venv_ready(nrf52_dirs.python_env)
_mark_west_initialized(nrf52_dirs.framework)
mock_nrf52_ops.run_command_ok.return_value = False
with pytest.raises(EsphomeError, match="Can't resume"):
check_and_install()
assert (
call(nrf52_dirs.framework, msg=ANY)
not in mock_nrf52_ops.rmdir.call_args_list
)
assert (nrf52_dirs.framework / ".resume_failed").exists()
def test_cut_short_init_starts_over(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A ``.west`` without its config (init cut short) clones clean."""
_mark_venv_ready(nrf52_dirs.python_env)
(nrf52_dirs.framework / ".west").mkdir()
check_and_install()
mock_nrf52_ops.rmdir.assert_any_call(nrf52_dirs.framework, msg=ANY)
first = mock_nrf52_ops.run_command_ok.call_args_list[0]
assert "init" in first.args[0]
def test_second_failed_resume_starts_over(
self,
nrf52_dirs: SimpleNamespace,
mock_nrf52_ops: SimpleNamespace,
) -> None:
"""A resume failing twice in a row wipes the workspace and clones clean."""
_mark_venv_ready(nrf52_dirs.python_env)
_mark_west_initialized(nrf52_dirs.framework)
(nrf52_dirs.framework / ".resume_failed").touch()
# resumed update fails; clean init, update and zephyr reqs succeed
mock_nrf52_ops.run_command_ok.side_effect = [False, True, True, True]
check_and_install()
mock_nrf52_ops.rmdir.assert_any_call(nrf52_dirs.framework, msg=ANY)
commands = [c.args[0] for c in mock_nrf52_ops.run_command_ok.call_args_list]
assert "update" in commands[0]
assert "init" in commands[1]
assert "update" in commands[2]
def test_requirements_install_failure_raises(
self,