diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 7ce973a2a9..7c17eadd1a 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -117,7 +117,7 @@ def set_core_data(config: ConfigType) -> ConfigType: def _resolve_toolchain(config: ConfigType) -> ConfigType: if CORE.toolchain is None: - CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.PLATFORMIO) + CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.SDK_NRF) return config @@ -439,8 +439,8 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: types = [] UF2_PATH = "zephyr/zephyr.uf2" DFU_PATH = "firmware.zip" - HEX_PATH = "zephyr/zephyr.hex" - HEX_MERGED_PATH = "zephyr/merged.hex" + HEX_PATH = "zephyr/zephyr.hex" # SDK 2.6.1, only generated when OTA is disabled + HEX_MERGED_PATH = "zephyr/merged.hex" # SDK 2.9.2, always generated APP_IMAGE_PATH = "zephyr/app_update.bin" build_dir = Path(storage_json.firmware_bin_path).parent if (build_dir / UF2_PATH).is_file(): @@ -777,6 +777,11 @@ def _generate_cmake_lists() -> bool: ) +def _copy_if_exists(src: Path, dst: Path) -> None: + if src.is_file(): + shutil.copy2(src, dst) + + def run_compile(args, config: ConfigType) -> bool: if CORE.using_toolchain_platformio: return False @@ -828,15 +833,18 @@ def run_compile(args, config: ConfigType) -> bool: ): raise EsphomeError("nRF52 native build failed") - # Zephyr's cmake places kernel artifacts in build_dir/zephyr/zephyr/ and - # merged.hex at build_dir/. Normalize to build_dir/zephyr/ so paths match - # get_download_types (which mirrors the platformio build output layout). zephyr_dir = build_dir / "zephyr" - west_out = zephyr_dir / "zephyr" - for filename in ["zephyr.uf2"]: - src = west_out / filename - if src.is_file(): - shutil.copy2(src, zephyr_dir / filename) + framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] + # SDK < 2.9.2 places artifacts directly in build_dir/zephyr/. + # SDK >= 2.9.2 nests them one level deeper (build_dir/zephyr/zephyr/); + # copy files to match get_download_types layout. + if framework_ver < cv.Version(2, 9, 2): + west_out = zephyr_dir + else: + west_out = zephyr_dir / "zephyr" + _copy_if_exists(west_out / "zephyr.uf2", zephyr_dir / "zephyr.uf2") + _copy_if_exists(west_out / "zephyr.signed.bin", zephyr_dir / "app_update.bin") + _copy_if_exists(build_dir / "merged.hex", zephyr_dir / "merged.hex") # (dev_type, sd_req) per bootloader — values from Nordic SoftDevice release notes _GENPKG_PARAMS = { diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 7cb1164482..640aa07fbf 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -2,10 +2,12 @@ import logging import os from pathlib import Path import platform +import shutil import tempfile import platformdirs +import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError from esphome.framework_helpers import ( @@ -134,6 +136,23 @@ def get_build_env() -> dict: return env +def _patch_uf2conv_escape_sequences(framework_path: Path) -> None: + # SDK v2.6.1 ships uf2conv.py with '\s+' — an unrecognised escape that + # Python 3.12+ flags with SyntaxWarning (a future version will reject it). + uf2conv = framework_path / "zephyr" / "scripts" / "build" / "uf2conv.py" + if not uf2conv.exists(): + return + content = uf2conv.read_text(encoding="utf-8") + patched = content.replace("re.split('\\s+', line)", "re.split('\\\\s+', line)") + if patched == content: + return + # Write atomically so a concurrent build never sees a truncated file + tmp = uf2conv.with_suffix(".py.tmp") + tmp.write_text(patched, encoding="utf-8") + shutil.copymode(uf2conv, tmp) + tmp.replace(uf2conv) + + def check_and_install() -> None: version = _get_version_str() python_env_path = _get_python_env_path(version) @@ -195,6 +214,9 @@ def check_and_install() -> None: ] if not run_command_ok(cmd, cwd=framework_path): raise EsphomeError(f"Can't update nRF Connect SDK {version}") + framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] + if framework_ver < cv.Version(2, 9, 2): + _patch_uf2conv_escape_sequences(framework_path) sentinel.touch() zephyr_sentinel = python_env_path / ".zephyr_reqs_ready" diff --git a/esphome/components/zephyr/__init__.py b/esphome/components/zephyr/__init__.py index cd077a142f..d6c45a744c 100644 --- a/esphome/components/zephyr/__init__.py +++ b/esphome/components/zephyr/__init__.py @@ -18,6 +18,7 @@ from .const import ( KEY_OVERLAY, KEY_PM_STATIC, KEY_PRJ_CONF, + KEY_SYSBUILD, KEY_USER, KEY_ZEPHYR, zephyr_ns, @@ -55,6 +56,7 @@ class ZephyrData(TypedDict): pm_static: list[Section] user: dict[str, list[str]] kconfig: str + sysbuild: bool def zephyr_set_core_data(config: ConfigType) -> None: @@ -69,6 +71,10 @@ def zephyr_set_core_data(config: ConfigType) -> None: pm_static=[], user={}, kconfig="", + # When OTA is disabled, the image is built without a bootloader even if the + # config says `bootloader: mcuboot`, so the image can be smaller. This was + # the default behaviour in SDK 2.6.1. + sysbuild=False, ) @@ -286,6 +292,13 @@ def copy_files() -> None: CORE.relative_build_path("zephyr/Kconfig"), kconfig ) + sysbuild_conf = "" + if zephyr_data()[KEY_SYSBUILD]: + sysbuild_conf = "SB_CONFIG_BOOTLOADER_MCUBOOT=y\n" + changed |= _write_file_if_changed_or_remove_when_empty( + CORE.relative_build_path("zephyr/sysbuild.conf"), sysbuild_conf + ) + if changed: # A configure-time input changed; drop the CMake cache so the build # can't reuse stale configure results (the native sdk-nrf toolchain diff --git a/esphome/components/zephyr/const.py b/esphome/components/zephyr/const.py index f2de861e31..497e5f3ce5 100644 --- a/esphome/components/zephyr/const.py +++ b/esphome/components/zephyr/const.py @@ -13,6 +13,7 @@ KEY_PRJ_CONF: Final = "prj_conf" KEY_ZEPHYR = "zephyr" KEY_BOARD: Final = "board" KEY_USER: Final = "user" +KEY_SYSBUILD: Final = "sysbuild" zephyr_ns = cg.esphome_ns.namespace("zephyr") CdcAcm = zephyr_ns.class_("CdcAcm", cg.Component) diff --git a/esphome/components/zephyr_mcumgr/ota/__init__.py b/esphome/components/zephyr_mcumgr/ota/__init__.py index b0d86190b8..0ff1825bd1 100644 --- a/esphome/components/zephyr_mcumgr/ota/__init__.py +++ b/esphome/components/zephyr_mcumgr/ota/__init__.py @@ -6,9 +6,19 @@ from esphome.components.zephyr import ( zephyr_add_prj_conf, zephyr_data, ) -from esphome.components.zephyr.const import BOOTLOADER_MCUBOOT, KEY_BOOTLOADER +from esphome.components.zephyr.const import ( + BOOTLOADER_MCUBOOT, + KEY_BOOTLOADER, + KEY_SYSBUILD, +) import esphome.config_validation as cv -from esphome.const import CONF_HARDWARE_UART, CONF_ID, Framework +from esphome.const import ( + CONF_HARDWARE_UART, + CONF_ID, + KEY_CORE, + KEY_FRAMEWORK_VERSION, + Framework, +) from esphome.core import CORE, coroutine_with_priority from esphome.coroutine import CoroPriority from esphome.types import ConfigType @@ -139,3 +149,6 @@ async def to_code(config: ConfigType) -> None: }}; """ ) + framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] + if framework_ver >= cv.Version(2, 9, 2): + zephyr_data()[KEY_SYSBUILD] = True diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index feacc2b1af..20a737cdbf 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Extract memory usage statistics from ESPHome build output. -This script parses the PlatformIO build output to extract RAM and flash -usage statistics for a compiled component. It's used by the CI workflow to +This script parses the build output to extract RAM and flash usage +statistics for a compiled component. It's used by the CI workflow to compare memory usage between branches. The script reads compile output from stdin and looks for the standard @@ -10,6 +10,13 @@ PlatformIO output format: RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) +or the linker memory usage table printed by Zephyr native builds +(e.g. nRF52 with the sdk-nrf toolchain): + Memory region Used Size Region Size %age Used + FLASH: 90624 B 796 KB 11.12% + RAM: 22432 B 256 KB 8.56% + IDT_LIST: 0 GB 32 KB 0.00% + Optionally performs detailed memory analysis if a build directory is provided. """ @@ -34,20 +41,43 @@ _RAM_PATTERN = re.compile(r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes" _FLASH_PATTERN = re.compile(r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes") _BUILD_PATH_PATTERN = re.compile(r"Build path: (.+)") +# Zephyr native builds print the GNU ld --print-memory-usage table instead of +# the PlatformIO summary. Only the FLASH and RAM regions are real memory +# (IDT_LIST is a build-time pseudo-region discarded from the final image). +# Each cell is humanized to the largest unit that divides evenly, so used +# sizes are not always plain bytes (zero prints as "0 GB"). +_ZEPHYR_RAM_PATTERN = re.compile( + r"^\s*RAM:\s+(\d+)\s*([KMG]?B)\s+\d+\s*[KMG]?B\s+\d+\.\d+%", re.MULTILINE +) +_ZEPHYR_FLASH_PATTERN = re.compile( + r"^\s*FLASH:\s+(\d+)\s*([KMG]?B)\s+\d+\s*[KMG]?B\s+\d+\.\d+%", re.MULTILINE +) +_ZEPHYR_UNIT_MULTIPLIERS = {"B": 1, "KB": 1024, "MB": 1024**2, "GB": 1024**3} + + +def _zephyr_bytes(matches: list[tuple[str, str]]) -> int: + """Sum humanized (value, unit) pairs from the Zephyr memory table.""" + return sum(int(value) * _ZEPHYR_UNIT_MULTIPLIERS[unit] for value, unit in matches) + def extract_from_compile_output( output_text: str, ) -> tuple[int | None, int | None, str | None]: - """Extract memory usage and build directory from PlatformIO compile output. + """Extract memory usage and build directory from compile output. Supports multiple builds (for component groups or isolated components). When test_build_components.py creates multiple builds, this sums the memory usage across all builds. - Looks for lines like: + Looks for PlatformIO lines like: RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) + and Zephyr (native west build) linker table rows like: + Memory region Used Size Region Size %age Used + FLASH: 90624 B 796 KB 11.12% + RAM: 22432 B 256 KB 8.56% + Also extracts build directory from lines like: INFO Compiling app... Build path: /path/to/build @@ -61,12 +91,20 @@ def extract_from_compile_output( ram_matches = _RAM_PATTERN.findall(output_text) flash_matches = _FLASH_PATTERN.findall(output_text) - if not ram_matches or not flash_matches: + # Zephyr native builds print the linker memory table instead + zephyr_ram_matches = _ZEPHYR_RAM_PATTERN.findall(output_text) + zephyr_flash_matches = _ZEPHYR_FLASH_PATTERN.findall(output_text) + + if not (ram_matches or zephyr_ram_matches) or not ( + flash_matches or zephyr_flash_matches + ): return None, None, None # Sum all builds (handles multiple component groups) total_ram = sum(int(match) for match in ram_matches) total_flash = sum(int(match) for match in flash_matches) + total_ram += _zephyr_bytes(zephyr_ram_matches) + total_flash += _zephyr_bytes(zephyr_flash_matches) # Extract build directory from ESPHome's explicit build path output # Look for: INFO Compiling app... Build path: /path/to/build @@ -202,20 +240,23 @@ def main() -> int: ) if ram_bytes is None or flash_bytes is None: - print("Failed to extract memory usage from compile output", file=sys.stderr) - print("Expected lines like:", file=sys.stderr) print( - " RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)", - file=sys.stderr, - ) - print( - " Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)", + "Failed to extract memory usage from compile output\n" + "Expected lines like:\n" + " RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)\n" + " Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)\n" + "or a Zephyr linker memory usage table like:\n" + " Memory region Used Size Region Size %age Used\n" + " FLASH: 90624 B 796 KB 11.12%\n" + " RAM: 22432 B 256 KB 8.56%", file=sys.stderr, ) return 1 # Count how many builds were found - num_builds = len(_RAM_PATTERN.findall(compile_output)) + num_builds = len(_RAM_PATTERN.findall(compile_output)) + len( + _ZEPHYR_RAM_PATTERN.findall(compile_output) + ) if num_builds > 1: print( diff --git a/tests/components/api/test.nrf52-adafruit.yaml b/tests/components/api/test.nrf52-adafruit.yaml index 18bf23d710..347480bab6 100644 --- a/tests/components/api/test.nrf52-adafruit.yaml +++ b/tests/components/api/test.nrf52-adafruit.yaml @@ -1,7 +1,7 @@ +<<: !include common.yaml + network: enable_ipv6: true openthread: tlv: 0E080000000000010000 - -api: diff --git a/tests/components/nrf52/test.nrf52-adafruit.yaml b/tests/components/nrf52/test.nrf52-adafruit.yaml index 3ae48b2a5f..5fa0d6e88f 100644 --- a/tests/components/nrf52/test.nrf52-adafruit.yaml +++ b/tests/components/nrf52/test.nrf52-adafruit.yaml @@ -19,5 +19,3 @@ nrf52: reg0: voltage: 2.1V uicr_erase: true - framework: - version: "2.6.1-b"