From 72da2b1ce24112193e13831b746c4e13516b5bf9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 12:53:50 -0500 Subject: [PATCH 1/6] Add the ESP8266 linker-script surgery and board build metadata --- esphome/components/esp8266/boards.py | 111 ++++++++++++++++++ esphome/components/esp8266/build_surgery.py | 92 +++++++++++++++ esphome/components/esp8266/const.py | 3 + .../components/esp8266/test_build_surgery.py | 87 ++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 esphome/components/esp8266/build_surgery.py create mode 100644 tests/unit_tests/components/esp8266/test_build_surgery.py diff --git a/esphome/components/esp8266/boards.py b/esphome/components/esp8266/boards.py index 02bfa9e662..e5a9628e90 100644 --- a/esphome/components/esp8266/boards.py +++ b/esphome/components/esp8266/boards.py @@ -360,3 +360,114 @@ BOARDS = { "flash_size": FLASH_SIZE_4_MB, }, } + +""" +ESP8266_BOARD_BUILD generate with: + +git clone https://github.com/platformio/platform-espressif8266 +python3 - <<'EOF' +import json, glob, os +for f in sorted(glob.glob("platform-espressif8266/boards/*.json")): + b = json.load(open(f))["build"] + extra = b["extra_flags"] + extra = extra.split() if isinstance(extra, str) else extra + defines = [ + e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266") + ] + entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "") + board = os.path.splitext(os.path.basename(f))[0] + print(f' "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},') +EOF +""" + +# Per-board Arduino core build metadata for the native (PlatformIO-free) +# toolchain: the variant directory (supplies pins_arduino.h) and the +# board-identity defines the PlatformIO builder passes via build.extra_flags. +# -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by +# the generator; only the per-board defines are listed here. +ESP8266_BOARD_BUILD = { + "agruminolemon": { + "variant": "agruminolemonv4", + "defines": ("ARDUINO_ESP8266_AGRUMINO_LEMON_V4",), + }, + "d1": {"variant": "d1", "defines": ("ARDUINO_ESP8266_WEMOS_D1R1",)}, + "d1_mini": {"variant": "d1_mini", "defines": ("ARDUINO_ESP8266_WEMOS_D1MINI",)}, + "d1_mini_lite": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINILITE",), + }, + "d1_mini_pro": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1MINIPRO",), + }, + "d1_wroom_02": { + "variant": "d1_mini", + "defines": ("ARDUINO_ESP8266_WEMOS_D1WROOM02",), + }, + "eduinowifi": { + "variant": "eduinowifi", + "defines": ("ARDUINO_ESP8266_SCHIRMILABS_EDUINO_WIFI",), + }, + "esp01": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp01_1m": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp07": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp07s": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP07",)}, + "esp12e": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "esp210": {"variant": "generic", "defines": ("ARDUINO_ESP8266_ESP210",)}, + "esp8285": {"variant": "esp8285", "defines": ("ARDUINO_ESP8266_ESP01",)}, + "esp_wroom_02": { + "variant": "nodemcu", + "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",), + }, + "espduino": {"variant": "ESPDuino", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espectro": {"variant": "espectro", "defines": ("ARDUINO_ESP8266_ESPECTRO_CORE",)}, + "espino": {"variant": "espino", "defines": ("ARDUINO_ESP8266_ESP12",)}, + "espinotee": {"variant": "espinotee", "defines": ("ARDUINO_ESP8266_ESP13",)}, + "espmxdevkit": { + "variant": "esp8285", + "defines": ("ARDUINO_ESP8266_ESP01", "LED_BUILTIN=16"), + }, + "espresso_lite_v1": { + "variant": "espresso_lite_v1", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V1",), + }, + "espresso_lite_v2": { + "variant": "espresso_lite_v2", + "defines": ("ARDUINO_ESP8266_ESPRESSO_LITE_V2",), + }, + "gen4iod": {"variant": "generic", "defines": ("ARDUINO_GEN4_IOD",)}, + "heltec_wifi_kit_8": { + "variant": "wifi_kit_8", + "defines": ("ARDUINO_wifi_kit_8",), + }, + "huzzah": {"variant": "adafruit", "defines": ("ARDUINO_ESP8266_ADAFRUIT_HUZZAH",)}, + "inventone": {"variant": "inventone", "defines": ("ARDUINO_ESP8266_INVENT_ONE",)}, + "modwifi": {"variant": "generic", "defines": ("ARDUINO_MOD_WIFI_ESP8266",)}, + "nodemcu": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU",)}, + "nodemcuv2": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_NODEMCU_ESP12E",)}, + "oak": {"variant": "oak", "defines": ("ARDUINO_ESP8266_OAK",)}, + "phoenix_v1": { + "variant": "phoenix_v1", + "defines": ("ARDUINO_ESP8266_PHOENIX_V1",), + }, + "phoenix_v2": { + "variant": "phoenix_v2", + "defines": ("ARDUINO_ESP8266_PHOENIX_V2",), + }, + "sonoff_basic": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_BASIC",)}, + "sonoff_s20": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_S20",)}, + "sonoff_sv": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_SV",)}, + "sonoff_th": {"variant": "itead", "defines": ("ARDUINO_ESP8266_SONOFF_TH",)}, + "sparkfunBlynk": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thing": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING",)}, + "thingdev": {"variant": "thing", "defines": ("ARDUINO_ESP8266_THING_DEV",)}, + "wifi_slot": {"variant": "wifi_slot", "defines": ("ARDUINO_AMPERKA_WIFI_SLOT",)}, + "wifiduino": {"variant": "wifiduino", "defines": ("ARDUINO_WIFIDUINO_ESP8266",)}, + "wifinfo": {"variant": "wifinfo", "defines": ("ARDUINO_WIFINFO",)}, + "wio_link": {"variant": "wiolink", "defines": ("ARDUINO_ESP8266_WIO_LINK",)}, + "wio_node": {"variant": "nodemcu", "defines": ("ARDUINO_ESP8266_ESP_WROOM_02",)}, + "xinabox_cw01": { + "variant": "xinabox", + "defines": ("ARDUINO_ESP8266_XINABOX_CW01",), + }, +} diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py new file mode 100644 index 0000000000..039ecaf0c6 --- /dev/null +++ b/esphome/components/esp8266/build_surgery.py @@ -0,0 +1,92 @@ +"""Linker-script surgery shared with the native (PlatformIO-free) toolchain. + +These mirror the PlatformIO extra scripts in this directory +(``relocate_ratetable.py.script`` and ``testing_mode.py.script``), which run +inside SCons and must stay self-contained. The native build generator applies +the same patches to the linker scripts it generates, so the logic lives here +as plain functions. Keep both in sync when changing either. +""" + +from __future__ import annotations + +from collections.abc import Collection +import re + +# Move the NONOS SDK wifi rate tables from flash to DRAM; see +# relocate_ratetable.py.script for the full background (NONOS SDK issue 320). +RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +# Match the whole line: "_data_start" is also a substring of the +# "_dport0_data_start" line in the earlier .dport0.data section +_RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) + +# Memory sizes for testing mode (allow larger builds for CI component grouping) +TESTING_IRAM_SIZE = "0x200000" # 2MB +TESTING_DRAM_SIZE = "0x200000" # 2MB +TESTING_FLASH_SIZE = "0x2000000" # 32MB + + +def relocate_ratetable(content: str) -> str: + """Insert the rate-table DRAM rule into a generated common linker script.""" + if RATETABLE_RULE in content: + return content + match = _RATETABLE_ANCHOR.search(content) + if match is None: + raise RuntimeError( + "'_data_start' anchor not found in the generated linker script; " + "cannot apply wifi rate table DRAM relocation " + "(has the Arduino core linker script changed?)" + ) + insert_pos = match.end() + return ( + content[:insert_pos] + + "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" + + f"\n {RATETABLE_RULE}" + + content[insert_pos:] + ) + + +_TESTING_SEGMENT_SIZES = ( + ("iram1_0_seg", TESTING_IRAM_SIZE), + ("dram0_0_seg", TESTING_DRAM_SIZE), + ("irom0_0_seg", TESTING_FLASH_SIZE), +) + + +def _patch_segment_size(content: str, segment_name: str, new_size: str) -> str: + pattern = ( + rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)" + r"0x[0-9a-fA-F]+" + ) + return re.sub(pattern, rf"\g<1>{new_size}", content) + + +def apply_testing_memory_patches(content: str, require: Collection[str]) -> str: + """Enlarge IRAM/DRAM/flash segments so grouped CI test builds can link. + + ``require`` names the segments this file must define; a silently + unpatched segment would keep the real memory limits and fail grouped + builds far from the cause. The segments are split across the two linker + scripts (iram1_0_seg in the generated common one, dram0_0_seg and + irom0_0_seg in the flash one), so each caller requires only its own. + """ + missing = set(require) + for segment, size in _TESTING_SEGMENT_SIZES: + patched = _patch_segment_size(content, segment, size) + if patched != content: + missing.discard(segment) + content = patched + if missing: + raise RuntimeError( + f"Testing-mode memory patch failed: segment(s) {', '.join(sorted(missing))} " + "not found (has the Arduino core linker script changed?)" + ) + return content + + +def segment_length(content: str, segment_name: str) -> int | None: + """Read a memory segment's length from linker script content.""" + match = re.search( + rf"{segment_name}\s*:.+len\s*=\s*(0x[\da-fA-F]+)", + content, + ) + return int(match.group(1), 16) if match else None diff --git a/esphome/components/esp8266/const.py b/esphome/components/esp8266/const.py index 3e89ab989f..8729007e3a 100644 --- a/esphome/components/esp8266/const.py +++ b/esphome/components/esp8266/const.py @@ -15,6 +15,9 @@ CONF_ENABLE_SERIAL1 = "enable_serial1" KEY_WAVEFORM_REQUIRED = "waveform_required" KEY_SERIAL_REQUIRED = "serial_required" KEY_SERIAL1_REQUIRED = "serial1_required" +# Set for the native (non-PlatformIO) toolchain's build generator +KEY_FLASH_MODE = "flash_mode" +KEY_SCANF_FLOAT = "scanf_float" # esp8266 namespace is already defined by arduino, manually prefix esphome esp8266_ns = cg.global_ns.namespace("esphome").namespace("esp8266") diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py new file mode 100644 index 0000000000..200954494b --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -0,0 +1,87 @@ +"""Tests for the linker-script surgery shared with the native toolchain.""" + +from __future__ import annotations + +import pytest + +from esphome.components.esp8266.build_surgery import ( + RATETABLE_RULE, + apply_testing_memory_patches, + relocate_ratetable, +) + +_COMMON_LD_SNIPPET = """\ + .dport0.data : ALIGN(4) + { + _dport0_data_start = ABSOLUTE(.); + } >dport0_0_seg :dport0_0_phdr + .data : ALIGN(4) + { + _data_start = ABSOLUTE(.); + *(.data) + } >dram0_0_seg :dram0_0_phdr +""" + +_FLASH_LD_SNIPPET = """\ +MEMORY +{ + dport0_0_seg : org = 0x3FF00000, len = 0x10 + dram0_0_seg : org = 0x3FFE8000, len = 0x14000 + iram1_0_seg : org = 0x40100000, len = 0x8000 + irom0_0_seg : org = 0x40201010, len = 0xfeff0 +} +""" + + +def test_relocate_ratetable_inserts_after_data_start() -> None: + patched = relocate_ratetable(_COMMON_LD_SNIPPET) + assert RATETABLE_RULE in patched + # Inserted after the .data section's anchor, not the .dport0.data one + assert patched.index("_data_start = ABSOLUTE(.);") < patched.index(RATETABLE_RULE) + assert patched.index(RATETABLE_RULE) < patched.index("*(.data)") + # Idempotent on an already-patched script + assert relocate_ratetable(patched) == patched + + +def test_relocate_ratetable_requires_anchor() -> None: + with pytest.raises(RuntimeError, match="_data_start"): + relocate_ratetable("SECTIONS { }") + + +def test_testing_memory_patches_enlarge_segments() -> None: + patched = apply_testing_memory_patches(_FLASH_LD_SNIPPET, require=()) + assert ( + "iram1_0_seg : org = 0x40100000, len = 0x200000" + in patched + ) + assert ( + "dram0_0_seg : org = 0x3FFE8000, len = 0x200000" + in patched + ) + assert ( + "irom0_0_seg : org = 0x40201010, len = 0x2000000" + in patched + ) + + +def test_segment_length() -> None: + from esphome.components.esp8266.build_surgery import segment_length + + assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0 + assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None + + +def test_testing_memory_patches_require() -> None: + """A required segment the patch could not find raises instead of + silently keeping the real memory limits.""" + patched = apply_testing_memory_patches( + _FLASH_LD_SNIPPET, require=("dram0_0_seg", "irom0_0_seg") + ) + assert "0x2000000" in patched + with pytest.raises(RuntimeError, match="dram0_0_seg, irom0_0_seg"): + apply_testing_memory_patches( + "MEMORY { }", require=("dram0_0_seg", "irom0_0_seg") + ) + # Segments a file does not require are patched opportunistically only + # With nothing required, unmatched content passes through unchanged + assert apply_testing_memory_patches("MEMORY { }", require=()) == "MEMORY { }" From 04ad9524f52644cc4ad95cc0c8d969f136915409 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 12:56:34 -0500 Subject: [PATCH 2/6] Add the arduino toolchain enum and validate --toolchain on every platform --- esphome/components/esp32/__init__.py | 5 ++++ esphome/components/esp8266/__init__.py | 3 ++ esphome/components/host/__init__.py | 1 + esphome/components/libretiny/__init__.py | 1 + esphome/components/nrf52/__init__.py | 5 ++++ esphome/components/rp2/__init__.py | 1 + esphome/config_validation.py | 22 +++++++++++++++ esphome/const.py | 2 ++ esphome/core/__init__.py | 4 +++ esphome/core/config.py | 13 +++++---- tests/component_tests/esp32/test_esp32.py | 14 +++++++++ tests/unit_tests/core/test_config.py | 33 ++++++++++++++++++++++ tests/unit_tests/test_config_validation.py | 16 +++++++++++ tests/unit_tests/test_nrf52_framework.py | 11 ++++++++ 14 files changed, 126 insertions(+), 5 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index d6ed6d9399..a8a5abab25 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1083,6 +1083,11 @@ def _resolve_toolchain(value: ConfigType) -> ConfigType: # CORE.toolchain instead of re-resolving it from the config dict. if CORE.toolchain is None: CORE.toolchain = value.get(CONF_TOOLCHAIN, Toolchain.ESP_IDF) + if CORE.toolchain not in (Toolchain.PLATFORMIO, Toolchain.ESP_IDF): + raise cv.Invalid( + f"Unsupported toolchain '{CORE.toolchain.value}' for ESP32. " + "Supported toolchains are 'platformio' and 'esp-idf'." + ) return value diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 2161a902cb..abc2f5dc91 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -244,6 +244,9 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean, } ), + # Until the native toolchain lands, PlatformIO is the only backend; + # reject a --toolchain this platform cannot serve yet. + cv.require_platformio_toolchain("ESP8266"), set_core_data, ) diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index b6a3b8b615..6f0fcb9d52 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -37,6 +37,7 @@ CONFIG_SCHEMA = cv.All( } ), set_core_data, + cv.require_platformio_toolchain("host"), ) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index c56cc48055..f83593269e 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -315,6 +315,7 @@ BASE_SCHEMA = cv.Schema( BASE_SCHEMA.add_extra(_detect_variant) BASE_SCHEMA.add_extra(_update_core_data) +BASE_SCHEMA.add_extra(cv.require_platformio_toolchain("LibreTiny")) def _configure_lwip(config: dict) -> None: diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 2d25558254..2328d444f7 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -128,6 +128,11 @@ def set_core_data(config: ConfigType) -> ConfigType: def _resolve_toolchain(config: ConfigType) -> ConfigType: if CORE.toolchain is None: CORE.toolchain = config.get(CONF_TOOLCHAIN, Toolchain.SDK_NRF) + if CORE.toolchain not in (Toolchain.PLATFORMIO, Toolchain.SDK_NRF): + raise cv.Invalid( + f"Unsupported toolchain '{CORE.toolchain.value}' for nRF52. " + "Supported toolchains are 'platformio' and 'sdk-nrf'." + ) return config diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index ed975ec01a..14806b26ce 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -313,6 +313,7 @@ CONFIG_SCHEMA = cv.All( cv.has_at_least_one_key(CONF_BOARD, CONF_VARIANT), _detect_variant, set_core_data, + cv.require_platformio_toolchain("RP2"), ) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index f455c7b8bf..c3ee760761 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -2532,6 +2532,28 @@ def platformio_version_constraint(value): return constraints +def require_platformio_toolchain(platform_name: str): + """Reject a CLI-selected toolchain other than PlatformIO. + + For platforms with only the PlatformIO backend; without this a + ``--toolchain`` they cannot serve would silently build with PlatformIO. + """ + + def validator(config): + from esphome.const import Toolchain + + if CORE.toolchain is None: + CORE.toolchain = Toolchain.PLATFORMIO + if CORE.toolchain != Toolchain.PLATFORMIO: + raise Invalid( + f"Unsupported toolchain '{CORE.toolchain.value}' for " + f"{platform_name}. The only supported toolchain is 'platformio'." + ) + return config + + return validator + + def require_framework_version( *, max_version=False, diff --git a/esphome/const.py b/esphome/const.py index 0dd948544f..6e9378ec9a 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -21,6 +21,8 @@ class Toolchain(StrEnum): PLATFORMIO = "platformio" ESP_IDF = "esp-idf" SDK_NRF = "sdk-nrf" + # ESP8266: the Arduino core built directly (no PlatformIO) + ARDUINO = "arduino" class Platform(StrEnum): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 0f1ac9213e..8e9f8e9751 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -982,6 +982,10 @@ class EsphomeCore: def using_toolchain_sdk_nrf(self): return self.toolchain == Toolchain.SDK_NRF + @property + def using_toolchain_arduino(self): + return self.toolchain == Toolchain.ARDUINO + @property def using_zephyr(self): return self.target_framework == "zephyr" diff --git a/esphome/core/config.py b/esphome/core/config.py index 1095a4886e..be577cccbb 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -557,10 +557,12 @@ def _add_library_str(lib: str) -> None: @coroutine_with_priority(CoroPriority.FINAL) async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None: - if CORE.using_toolchain_esp_idf: - # The native ESP-IDF build doesn't read platformio.ini; honor the - # options with a native equivalent and warn about the rest, which - # would otherwise be silently ignored. + if CORE.using_toolchain_esp_idf or ( + CORE.using_toolchain_arduino and CORE.is_esp8266 + ): + # The native builds don't read platformio.ini; honor the options + # with a native equivalent and warn about the rest, which would + # otherwise be silently ignored. for key, val in pio_options.items(): vals = [val] if isinstance(val, str) else val if key == CONF_BUILD_FLAGS: @@ -588,8 +590,9 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No # config at upload time (upload_using_esptool) _LOGGER.warning( "esphome->platformio_options->%s is ignored when building with " - "the native ESP-IDF toolchain", + "the native '%s' toolchain", key, + CORE.toolchain.value, ) return # Add includes at the very end, so that they override everything diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 7208318d3a..370fd2a2dc 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -131,6 +131,20 @@ def test_esp32_rejects_unsupported_toolchains( CONFIG_SCHEMA({"variant": VARIANT_ESP32, "toolchain": config_toolchain}) +def test_esp32_rejects_unsupported_cli_toolchain( + set_core_config: SetCoreConfigCallable, +) -> None: + """A --toolchain the platform cannot serve fails instead of silently + building with PlatformIO (the CLI path bypasses the YAML validator).""" + set_core_config(PlatformFramework.ESP32_IDF) + + from esphome.components.esp32 import CONFIG_SCHEMA + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + CONFIG_SCHEMA({"variant": VARIANT_ESP32}) + + @pytest.mark.parametrize( ("config", "error_match"), [ diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index e09edd7f26..80e7f3b5c2 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1389,3 +1389,36 @@ def test_esphome_build_internals_are_yaml_only() -> None: assert markers[field].visibility is cv.Visibility.ADVANCED, field # A regular device-config field stays on the main form. assert markers[CONF_NAME_ADD_MAC_SUFFIX].visibility is None + + +@pytest.mark.asyncio +async def test_add_platformio_options_native_arduino( + caplog: pytest.LogCaptureFixture, +) -> None: + """The native ESP8266 Arduino toolchain warns about ignored options the + same way the native IDF toolchain does.""" + CORE.toolchain = Toolchain.ARDUINO + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp8266", + KEY_TARGET_FRAMEWORK: "arduino", + } + + await config._add_platformio_options( + { + "board_build.f_cpu": "160000000L", + "upload_speed": "115200", + } + ) + + assert "esphome->platformio_options->board_build.f_cpu is ignored" in caplog.text + assert "'arduino' toolchain" in caplog.text + assert "upload_speed" not in caplog.text + + +def test_esp8266_rejects_unsupported_cli_toolchain() -> None: + """Until the native backend lands, ESP8266 serves only PlatformIO.""" + from esphome.components.esp8266 import CONFIG_SCHEMA + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + CONFIG_SCHEMA({"board": "nodemcuv2"}) diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 971c4e462d..a7ddb931b3 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -3165,3 +3165,19 @@ def test_file__remapped_path_is_directory_raises(setup_core: Path) -> None: with pytest.raises(Invalid, match="is not a file"): cv.file_("/original/config/headers") + + +def test_require_platformio_toolchain() -> None: + """Platforms with only the PlatformIO backend reject other toolchains.""" + from esphome.const import Toolchain + from esphome.core import CORE + + validator = cv.require_platformio_toolchain("RP2") + CORE.toolchain = None + config: dict = {} + assert validator(config) is config + assert CORE.toolchain == Toolchain.PLATFORMIO + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(Invalid, match="Unsupported toolchain 'arduino' for RP2"): + validator(config) diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index c2ee0c2a75..b5f6b5794f 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -619,3 +619,14 @@ def test_needs_venv_rebuild_on_dangling_interpreter_symlink(tmp_path: Path) -> N assert not python.exists() assert _needs_venv_rebuild(python, sentinel, "abc123") + + +def test_resolve_toolchain_rejects_unsupported() -> None: + """A --toolchain nRF52 cannot serve fails instead of degrading silently.""" + from esphome.components.nrf52 import _resolve_toolchain + import esphome.config_validation as cv + from esphome.const import Toolchain + + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"): + _resolve_toolchain({}) From 3e776547190d5fb2151cbde8156fb2b804ce91a3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 12:58:06 -0500 Subject: [PATCH 3/6] Add the ESP8266 Arduino framework and toolchain installer --- esphome/arduino8266/__init__.py | 9 + esphome/arduino8266/framework.py | 335 ++++++++++++++ esphome/writer.py | 8 +- requirements.txt | 1 + .../unit_tests/test_arduino8266_framework.py | 431 ++++++++++++++++++ 5 files changed, 783 insertions(+), 1 deletion(-) create mode 100644 esphome/arduino8266/__init__.py create mode 100644 esphome/arduino8266/framework.py create mode 100644 tests/unit_tests/test_arduino8266_framework.py diff --git a/esphome/arduino8266/__init__.py b/esphome/arduino8266/__init__.py new file mode 100644 index 0000000000..8f403a8553 --- /dev/null +++ b/esphome/arduino8266/__init__.py @@ -0,0 +1,9 @@ +"""Native (PlatformIO-free) build support for the ESP8266 Arduino core. + +This package downloads the Arduino ESP8266 core and the xtensa-lx106 +toolchain, generates a ninja build for them plus the ESPHome sources, and +drives the build directly — the ESP8266 equivalent of ``esphome.espidf``. + +Deliberately importable without the esp8266 component to avoid circular +imports; the component wires these modules in via lazy imports. +""" diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py new file mode 100644 index 0000000000..066667806c --- /dev/null +++ b/esphome/arduino8266/framework.py @@ -0,0 +1,335 @@ +"""Download and install the Arduino ESP8266 core, toolchain, and ninja. + +Artifacts land in a machine-global cache (shared across projects, like the +ESP-IDF install in ``esphome.espidf.framework``): + + /arduino8266/frameworks// framework-arduinoespressif8266 + /arduino8266/toolchains// toolchain-xtensa (gcc 10.3) + +ninja itself comes from PATH or the ninja PyPI wheel (a requirements.txt +dependency), so only the two packages above are downloaded here. + +Sources default to the PlatformIO registry (the exact packages the PlatformIO +toolchain has always used, so the bits are identical); the +``ESPHOME_ARDUINO8266_*_MIRRORS`` environment variables override the URLs with +``{VERSION}`` / ``{SYSTEM}`` substitution. +""" + +from __future__ import annotations + +from collections.abc import Collection +import functools +import logging +import os +from pathlib import Path +import platform +import shutil + +import platformdirs + +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import ( + archive_extract_all, + download_from_mirrors, + download_with_resume, + rmdir, + str_to_lst_of_str, +) +from esphome.helpers import get_bool_env, get_str_env +from esphome.platformio.library import ensure_list + +_LOGGER = logging.getLogger(__name__) + +FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +TOOLCHAIN_PACKAGE = "toolchain-xtensa" +# gcc 10.3, the toolchain Arduino core 3.x builds with. The compile flags in +# the build generator are tuned to it; treat version changes as a full +# reinstall (the install dir is keyed on the version). +TOOLCHAIN_VERSION = "2.100300.220621" + +_REGISTRY_URL = ( + "https://api.registry.platformio.org/v3/packages/platformio/tool/{package}" +) + +ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "") +) +ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS = str_to_lst_of_str( + os.environ.get("ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", "") +) + + +def get_arduino8266_tools_path() -> Path: + # Treat an empty/whitespace prefix as unset: Path("") resolves to the CWD, + # which clean-all would then delete. + if prefix := get_str_env("ESPHOME_ARDUINO8266_PREFIX", "").strip(): + path = Path(prefix).expanduser() + else: + # Machine-global so all projects share one install; see + # espidf.framework.get_idf_tools_path for the location rationale. + path = ( + Path(platformdirs.user_cache_dir("esphome", appauthor=False)) + / "arduino8266" + ) + return path.resolve() + + +# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0 +MIN_FRAMEWORK_VERSION = cv.Version(3, 1, 1) + + +def framework_package_version(ver: cv.Version) -> str: + """Map an Arduino core version (e.g. 3.1.2) to its package version. + + Same encoding as the PlatformIO package registry uses for core 3.x + releases (3.1.2 -> 3.30102.0). The native toolchain only supports core + >= MIN_FRAMEWORK_VERSION, so the 1.x/2.x encodings never apply here. + """ + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + +def get_framework_path(package_version: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / package_version + + +def get_toolchain_path() -> Path: + return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION + + +def _downloads_path() -> Path: + path = get_arduino8266_tools_path() / "downloads" + path.mkdir(parents=True, exist_ok=True) + return path + + +def _pio_system() -> str: + """The PlatformIO registry system tag for the current host. + + Hand-rolled instead of ``platformio.util.get_systype()`` so this backend + never imports the PlatformIO package. The windows-arm64 and darwin-arm64 + mappings are deliberate: the toolchain packages ship x86_64 binaries for + those hosts (Rosetta / x86 emulation). + """ + sysname = platform.system().lower() + machine = platform.machine().lower() + if sysname == "darwin": + return "darwin_arm64" if machine == "arm64" else "darwin_x86_64" + if sysname == "windows": + return "windows_amd64" if machine in ("amd64", "arm64") else "windows_x86" + if sysname == "linux": + if machine in ("arm64", "aarch64"): + return "linux_aarch64" + if machine in ("i686", "i386", "x86"): + return "linux_i686" + if machine.startswith("arm"): + return f"linux_{machine}" + if machine in ("x86_64", "amd64"): + return "linux_x86_64" + # Fail here, near the cause, rather than installing a toolchain whose + # binaries cannot execute on this host. + raise EsphomeError( + f"No {sysname}/{machine} build of the ESP8266 toolchain exists; " + "use 'toolchain: platformio'" + ) + + +def _registry_download(package: str, version: str) -> tuple[str, str, int | None]: + """Resolve a package's download URL, sha256, and size via the PIO registry.""" + import requests + + url = _REGISTRY_URL.format(package=package) + last_err: Exception | None = None + for _ in range(3): + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + data = resp.json() + break + except requests.RequestException as err: + last_err = err + else: + # A clean, retried error like the other download paths in the tree + raise EsphomeError( + f"Could not query the package registry for {package}: {last_err}" + ) from last_err + system = _pio_system() + for ver in data.get("versions", []): + if ver.get("name") != version: + continue + for file in ver.get("files", []): + # ensure_list: a bare string would make ``in`` a substring test + systems = ensure_list(file.get("system") or "*") + if "*" in systems or system in systems: + sha256 = (file.get("checksum") or {}).get("sha256") + if not sha256: + # Never extract an unverified archive; the registry + # publishes a checksum for every package file. + raise EsphomeError( + f"The package registry returned no sha256 for " + f"{package} {version}; refusing the unverified download" + ) + return (file["download_url"], sha256, file.get("size")) + raise EsphomeError(f"No {package} {version} build for this platform ({system})") + raise EsphomeError(f"{package} {version} not found in the package registry") + + +def _install_package( + name: str, + version: str, + dest: Path, + mirrors: list[str], + expect: Collection[str] = (), +) -> None: + """Download, verify, and extract one package if not already installed. + + The registry path is integrity-checked against the sha256 the registry + publishes; a mirror override is trusted as configured. + """ + marker = dest / ".esphome_extracted" + if marker.is_file(): + return + from filelock import FileLock + + # The cache is machine-global; serialize concurrent cold builds so one + # process cannot wipe the directory another is extracting into (same + # filelock pattern as platformio/toolchain.py and git.py). + dest.parent.mkdir(parents=True, exist_ok=True) + with FileLock(f"{dest}.lock"): + if marker.is_file(): + # Another process finished the install while we waited + return + rmdir(dest, msg=f"Clean up incomplete {name} install") + # A persistent download location (not a temp dir) so an interrupted + # download resumes across esphome runs via download_with_resume's + # .part file, mirroring the espidf dist/ convention. + archive = _downloads_path() / f"{name}-{version}" + _LOGGER.info("Downloading %s %s ...", name, version) + if mirrors: + _LOGGER.warning( + "Downloading %s from a mirror override; checksum verification " + "is skipped for mirrors", + name, + ) + download_from_mirrors( + mirrors, {"VERSION": version, "SYSTEM": _pio_system()}, archive + ) + else: + url, sha256, size = _registry_download(name, version) + download_with_resume(url, archive, sha256=sha256, size=size) + _LOGGER.info("Extracting %s ...", name) + archive_extract_all(archive, dest, progress_header="Extracting") + # Validate the layout before recording success, so an unexpected + # package is never cached as a working install. + for rel in expect: + if not (dest / rel).is_dir(): + raise EsphomeError( + f"{name} {version} extracted without the expected {rel} " + "directory; run 'esphome clean-all' and retry" + ) + marker.touch() + archive.unlink(missing_ok=True) + + +def _find_ninja() -> Path: + """Locate the ninja binary: PATH first, else the ninja PyPI wheel. + + The wheel is a requirements.txt dependency, so pip has already + integrity-checked it; no download logic is needed here. + """ + if binary := shutil.which("ninja"): + return Path(binary) + try: + import ninja + except ImportError as err: + raise EsphomeError( + "ninja not found on PATH or in the ninja package; reinstall the " + "esphome Python environment" + ) from err + + binary = Path(ninja.BIN_DIR) / ("ninja.exe" if os.name == "nt" else "ninja") + if not binary.is_file(): + raise EsphomeError( + "ninja not found on PATH or in the ninja package; reinstall the " + "esphome Python environment" + ) + return binary + + +def check_and_install(framework_version: cv.Version) -> dict[str, Path]: + """Ensure framework, toolchain, and ninja are installed; return their paths.""" + package_version = framework_package_version(framework_version) + framework_path = get_framework_path(package_version) + _install_package( + FRAMEWORK_PACKAGE, + package_version, + framework_path, + ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + expect=("cores/esp8266", "tools/sdk", "libraries"), + ) + toolchain_path = get_toolchain_path() + _install_package( + TOOLCHAIN_PACKAGE, + TOOLCHAIN_VERSION, + toolchain_path, + ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + expect=("bin",), + ) + return { + "framework_path": framework_path, + "toolchain_path": toolchain_path, + "ninja_path": _find_ninja(), + } + + +def get_build_env(toolchain_path: Path) -> dict[str, str]: + env = os.environ.copy() + env["PATH"] = str(toolchain_path / "bin") + os.pathsep + env.get("PATH", "") + env.update(ccache_env()) + return env + + +@functools.cache +def ccache_path() -> str | None: + """The ccache binary to prefix compiles with, or None when disabled. + + Same convention as the PlatformIO path: on by default when the binary is + on PATH, ``ESPHOME_CCACHE_ENABLE=0`` disables it, and an explicit ``=1`` + warns when no binary is found and skips the runnability probe. + """ + from esphome.platformio.toolchain import _ccache_runs, _strip_win_long_path_prefix + + explicit = "ESPHOME_CCACHE_ENABLE" in os.environ + if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"): + return None + ccache = shutil.which("ccache") + if ccache is None: + if explicit: + _LOGGER.warning( + "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; " + "compiling without ccache" + ) + return None + ccache = _strip_win_long_path_prefix(ccache) + if not explicit and not _ccache_runs(ccache): + return None + return ccache + + +def ccache_env() -> dict[str, str]: + """Return ccache settings for the build subprocess (not os.environ). + + Mirrors ``espidf.framework._ccache_env``: cache under the machine-global + tools dir, depend mode (gcc emits depfiles via -MMD), and CCACHE_BASEDIR + scoped to the build dir so devices share framework cache entries. Values + the user already set in the environment are respected. + """ + if ccache_path() is None: + return {} + defaults = { + "CCACHE_DIR": str(get_arduino8266_tools_path() / "ccache"), + "CCACHE_NOHASHDIR": "true", + "CCACHE_DEPEND": "1", + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + } + return {k: v for k, v in defaults.items() if k not in os.environ} diff --git a/esphome/writer.py b/esphome/writer.py index 866377d2f5..26d091a823 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -660,11 +660,17 @@ def clean_all(configuration: list[str]): # that live outside it. import platformdirs + from esphome.arduino8266.framework import get_arduino8266_tools_path from esphome.components.nrf52.framework import get_sdk_nrf_tools_path from esphome.espidf.framework import get_idf_tools_path cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve() - for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()): + for install_path in ( + cache_root, + get_idf_tools_path(), + get_sdk_nrf_tools_path(), + get_arduino8266_tools_path(), + ): if install_path.is_dir(): _LOGGER.info("Deleting %s", install_path) rmtree(install_path) diff --git a/requirements.txt b/requirements.txt index 740a8c1a79..04844f67dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.3 # native esp-idf toolchain global cache dir +ninja==1.13.0 # native esp8266 arduino toolchain build driver filelock==3.32.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py new file mode 100644 index 0000000000..2e72dfb26a --- /dev/null +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -0,0 +1,431 @@ +"""Tests for esphome.arduino8266.framework (downloads and environment).""" + +from __future__ import annotations + +from contextlib import contextmanager +import os +from pathlib import Path +import subprocess +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.arduino8266 import framework +import esphome.config_validation as cv +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _clear_caches(tmp_path: Path) -> None: + framework.ccache_path.cache_clear() + CORE.build_path = tmp_path + + +def test_framework_package_version() -> None: + assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" + assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" + + +def test_tools_path_default_and_prefix(tmp_path: Path) -> None: + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}): + assert framework.get_arduino8266_tools_path() == tmp_path.resolve() + # A blank prefix must be treated as unset, not as the CWD + with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": " "}): + path = framework.get_arduino8266_tools_path() + assert path.name == "arduino8266" + assert path != Path.cwd() + + +@pytest.mark.parametrize( + ("system", "machine", "expected"), + [ + ("Darwin", "arm64", "darwin_arm64"), + ("Darwin", "x86_64", "darwin_x86_64"), + ("Windows", "AMD64", "windows_amd64"), + ("Windows", "ARM64", "windows_amd64"), + ("Windows", "x86", "windows_x86"), + ("Linux", "x86_64", "linux_x86_64"), + ("Linux", "aarch64", "linux_aarch64"), + ("Linux", "i686", "linux_i686"), + ("Linux", "armv7l", "linux_armv7l"), + ], +) +def test_pio_system(system: str, machine: str, expected: str) -> None: + with ( + patch("platform.system", return_value=system), + patch("platform.machine", return_value=machine), + ): + assert framework._pio_system() == expected + + +@pytest.mark.parametrize( + ("system", "machine"), + [ + ("FreeBSD", "amd64"), + ("Linux", "ppc64le"), + ], +) +def test_pio_system_unsupported_host_raises(system: str, machine: str) -> None: + # Fails at resolution rather than installing a toolchain that can't run + with ( + patch("platform.system", return_value=system), + patch("platform.machine", return_value=machine), + pytest.raises(EsphomeError, match="use 'toolchain: platformio'"), + ): + framework._pio_system() + + +def _registry_response(files: list[dict]) -> MagicMock: + resp = MagicMock() + resp.json.return_value = {"versions": [{"name": "1.0.0", "files": files}]} + return resp + + +def test_registry_download_network_error_is_clean_and_retried() -> None: + """Registry failures raise EsphomeError after retries, not a traceback.""" + import requests + + with ( + patch("requests.get", side_effect=requests.ConnectionError("boom")) as mock_get, + pytest.raises(EsphomeError, match="Could not query the package registry"), + ): + framework._registry_download("pkg", "1.0.0") + assert mock_get.call_count == 3 + + +def test_registry_download_retries_transient_error() -> None: + import requests + + resp = _registry_response( + [ + { + "system": ["linux_x86_64"], + "download_url": "http://x/linux", + "checksum": {"sha256": "abc123"}, + "size": 42, + } + ] + ) + with ( + patch("requests.get", side_effect=[requests.ConnectionError("boom"), resp]), + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + assert framework._registry_download("pkg", "1.0.0") == ( + "http://x/linux", + "abc123", + 42, + ) + + +def test_registry_download_matches_system() -> None: + resp = _registry_response( + [ + {"system": ["windows_amd64"], "download_url": "http://x/win"}, + { + "system": ["linux_x86_64"], + "download_url": "http://x/linux", + "checksum": {"sha256": "abc123"}, + "size": 42, + }, + ] + ) + with ( + patch("requests.get", return_value=resp), + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + assert framework._registry_download("pkg", "1.0.0") == ( + "http://x/linux", + "abc123", + 42, + ) + + +def test_registry_download_bare_string_system() -> None: + """A bare-string system tag is an exact match, not a substring test.""" + resp = _registry_response( + [ + {"system": "linux_x86", "download_url": "http://x/x86"}, + { + "system": "linux_x86_64", + "download_url": "http://x/x86_64", + "checksum": {"sha256": "abc"}, + }, + ] + ) + with ( + patch("requests.get", return_value=resp), + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + assert framework._registry_download("pkg", "1.0.0")[0] == "http://x/x86_64" + + +def test_registry_download_wildcard_system() -> None: + resp = _registry_response( + [ + { + "system": "*", + "download_url": "http://x/any", + "checksum": {"sha256": "abc"}, + "size": 7, + } + ] + ) + with patch("requests.get", return_value=resp): + assert framework._registry_download("pkg", "1.0.0") == ( + "http://x/any", + "abc", + 7, + ) + + +def test_registry_download_missing_checksum_raises() -> None: + """An unverifiable archive is refused, never silently extracted.""" + resp = _registry_response([{"system": "*", "download_url": "http://x/any"}]) + with ( + patch("requests.get", return_value=resp), + pytest.raises(EsphomeError, match="no sha256"), + ): + framework._registry_download("pkg", "1.0.0") + + +def test_registry_download_no_system_match() -> None: + resp = _registry_response( + [{"system": ["windows_amd64"], "download_url": "http://x/win"}] + ) + with ( + patch("requests.get", return_value=resp), + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + pytest.raises(EsphomeError, match="No pkg 1.0.0 build"), + ): + framework._registry_download("pkg", "1.0.0") + + +def test_registry_download_version_not_found() -> None: + resp = _registry_response([]) + resp.json.return_value = {"versions": [{"name": "2.0.0", "files": []}]} + with ( + patch("requests.get", return_value=resp), + pytest.raises(EsphomeError, match="not found"), + ): + framework._registry_download("pkg", "1.0.0") + + +def test_install_package_skips_when_marker_exists(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + dest.mkdir() + (dest / ".esphome_extracted").touch() + with patch.object(framework, "download_from_mirrors") as mock_download: + framework._install_package("pkg", "1.0.0", dest, []) + mock_download.assert_not_called() + + +def test_install_package_downloads_via_mirrors(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + mirrors = ["http://mirror/{VERSION}/{SYSTEM}.tar.gz"] + with ( + patch.object(framework, "download_from_mirrors") as mock_download, + patch.object(framework, "archive_extract_all") as mock_extract, + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + # Extraction is expected to create the directory + mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir() + framework._install_package("pkg", "1.0.0", dest, mirrors) + assert mock_download.call_args[0][0] is mirrors + assert mock_download.call_args[0][1] == { + "VERSION": "1.0.0", + "SYSTEM": "linux_x86_64", + } + assert (dest / ".esphome_extracted").is_file() + + +def test_install_package_downloads_via_registry(tmp_path: Path) -> None: + """The registry path downloads with the registry's sha256 and size.""" + dest = tmp_path / "pkg" + with ( + patch.object(framework, "download_with_resume") as mock_download, + patch.object(framework, "archive_extract_all") as mock_extract, + patch.object( + framework, + "_registry_download", + return_value=("http://x/pkg.tar.gz", "abc123", 42), + ), + ): + mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir() + framework._install_package("pkg", "1.0.0", dest, []) + assert mock_download.call_args[0][0] == "http://x/pkg.tar.gz" + assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42} + + +def test_find_ninja_prefers_path(tmp_path: Path) -> None: + with patch("shutil.which", return_value=str(tmp_path / "ninja")): + assert framework._find_ninja() == tmp_path / "ninja" + + +def test_find_ninja_falls_back_to_wheel(tmp_path: Path) -> None: + """Without a PATH entry, the ninja PyPI wheel's binary is used.""" + binary_name = "ninja.exe" if os.name == "nt" else "ninja" + (tmp_path / binary_name).touch() + wheel = MagicMock(BIN_DIR=str(tmp_path)) + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": wheel}), + ): + assert framework._find_ninja() == tmp_path / binary_name + + +def test_find_ninja_package_not_installed() -> None: + """A missing ninja package raises the actionable message, not ImportError.""" + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": None}), + pytest.raises(EsphomeError, match="ninja not found"), + ): + framework._find_ninja() + + +def test_find_ninja_missing_everywhere(tmp_path: Path) -> None: + wheel = MagicMock(BIN_DIR=str(tmp_path)) + with ( + patch("shutil.which", return_value=None), + patch.dict(sys.modules, {"ninja": wheel}), + pytest.raises(EsphomeError, match="ninja not found"), + ): + framework._find_ninja() + + +def test_check_and_install_returns_paths(tmp_path: Path) -> None: + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "_install_package") as mock_install, + patch.object(framework, "_find_ninja", return_value=tmp_path / "ninja"), + ): + paths = framework.check_and_install(cv.Version(3, 1, 2)) + assert paths["framework_path"] == tmp_path / "frameworks" / "3.30102.0" + assert ( + paths["toolchain_path"] == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION + ) + assert paths["ninja_path"] == tmp_path / "ninja" + assert mock_install.call_count == 2 + # The layout checks cover the directories write_project needs, including + # the bundled libraries/ tree + fw_expect = mock_install.call_args_list[0].kwargs["expect"] + assert fw_expect == ("cores/esp8266", "tools/sdk", "libraries") + assert mock_install.call_args_list[1].kwargs["expect"] == ("bin",) + + +def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: + with patch.object(framework, "ccache_env", return_value={"CCACHE_DIR": "x"}): + env = framework.get_build_env(tmp_path) + assert env["PATH"].startswith(str(tmp_path / "bin") + os.pathsep) + assert env["CCACHE_DIR"] == "x" + + +def test_ccache_path_disabled_by_env() -> None: + with patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}): + assert framework.ccache_path() is None + + +def test_ccache_path_no_binary(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False) + with patch("shutil.which", return_value=None): + assert framework.ccache_path() is None + + +def test_ccache_path_probe_failure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False) + with ( + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("subprocess.run", side_effect=subprocess.SubprocessError), + ): + assert framework.ccache_path() is None + + +def test_ccache_path_ok(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False) + with ( + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("subprocess.run"), + ): + assert framework.ccache_path() == "/usr/bin/ccache" + + +def test_ccache_path_explicit_missing_binary_warns( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1") + with patch("shutil.which", return_value=None): + assert framework.ccache_path() is None + assert "no ccache binary is on PATH" in caplog.text + + +def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> None: + """An explicit opt-in trusts the binary without the runnability probe.""" + monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1") + with ( + patch("shutil.which", return_value="/usr/bin/ccache"), + patch("esphome.platformio.toolchain._ccache_runs", side_effect=AssertionError), + ): + assert framework.ccache_path() == "/usr/bin/ccache" + + +def test_ccache_env(tmp_path: Path) -> None: + with patch.object(framework, "ccache_path", return_value=None): + assert framework.ccache_env() == {} + with ( + patch.object(framework, "ccache_path", return_value="/usr/bin/ccache"), + patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}), + ): + env = framework.ccache_env() + # User-set values are respected; the rest get defaults + assert "CCACHE_NOHASHDIR" not in env + assert env["CCACHE_DEPEND"] == "1" + assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve()) + assert env["CCACHE_DIR"].endswith("ccache") + + +def test_install_package_validates_expected_layout(tmp_path: Path) -> None: + """The success marker is only written when the extracted tree is usable.""" + dest = tmp_path / "pkg" + with ( + patch.object(framework, "download_from_mirrors"), + patch.object(framework, "archive_extract_all") as mock_extract, + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "bin").mkdir(parents=True) + framework._install_package("pkg", "1.0.0", dest, ["http://m"], expect=("bin",)) + assert (dest / ".esphome_extracted").is_file() + + +def test_install_package_unexpected_layout_raises(tmp_path: Path) -> None: + dest = tmp_path / "pkg" + with ( + patch.object(framework, "download_from_mirrors"), + patch.object(framework, "archive_extract_all") as mock_extract, + patch.object(framework, "_pio_system", return_value="linux_x86_64"), + pytest.raises(EsphomeError, match="without the expected bin"), + ): + mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir() + framework._install_package("pkg", "1.0.0", dest, ["http://m"], expect=("bin",)) + assert not (dest / ".esphome_extracted").exists() + + +def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None: + """A concurrent install finishing while we wait for the lock is detected.""" + dest = tmp_path / "pkg" + marker = dest / ".esphome_extracted" + + @contextmanager + def _fake_lock(*_a, **_kw): + dest.mkdir(parents=True, exist_ok=True) + marker.touch() + yield + + with ( + patch("filelock.FileLock", _fake_lock), + patch.object(framework, "download_from_mirrors") as mock_download, + patch.object(framework, "rmdir") as mock_rmdir, + ): + framework._install_package("pkg", "1.0.0", dest, ["http://m"]) + mock_download.assert_not_called() + mock_rmdir.assert_not_called() From 82c3c1f50adca2e8532b79becf3107eb721722c6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 12:59:10 -0500 Subject: [PATCH 4/6] Add the ESP8266 native library backend --- esphome/arduino8266/build_tool.py | 36 ++ esphome/arduino8266/component.py | 232 +++++++++++++ .../unit_tests/test_arduino8266_build_tool.py | 63 ++++ .../unit_tests/test_arduino8266_component.py | 310 ++++++++++++++++++ 4 files changed, 641 insertions(+) create mode 100644 esphome/arduino8266/build_tool.py create mode 100644 esphome/arduino8266/component.py create mode 100644 tests/unit_tests/test_arduino8266_build_tool.py create mode 100644 tests/unit_tests/test_arduino8266_component.py diff --git a/esphome/arduino8266/build_tool.py b/esphome/arduino8266/build_tool.py new file mode 100644 index 0000000000..bfcf4128eb --- /dev/null +++ b/esphome/arduino8266/build_tool.py @@ -0,0 +1,36 @@ +"""Tiny cross-platform build steps invoked from the generated ninja file. + +Plain script (not ``python -m``): it runs from ninja with whatever Python +started esphome and must not depend on the package being importable. + +Subcommands: + ar remove stale archive, then ``ar rc`` + copy copy a file +""" + +from pathlib import Path +import shutil +import subprocess +import sys + + +def main() -> int: + mode = sys.argv[1] + if mode == "ar": + ar, archive, rspfile = sys.argv[2:5] + # Remove first: ``ar rc`` replaces members but never drops ones whose + # source was removed from the build, which would leak stale objects. + Path(archive).unlink(missing_ok=True) + return subprocess.run( + [ar, "rc", archive, f"@{rspfile}"], check=False + ).returncode + if mode == "copy": + src, dst = sys.argv[2:4] + shutil.copyfile(src, dst) + return 0 + print(f"unknown build_tool mode: {mode}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/esphome/arduino8266/component.py b/esphome/arduino8266/component.py new file mode 100644 index 0000000000..17fd5471b4 --- /dev/null +++ b/esphome/arduino8266/component.py @@ -0,0 +1,232 @@ +"""Arduino ESP8266 backend for the shared PlatformIO library converter. + +Turns the libraries registered via ``cg.add_library()`` into build inputs for +the ninja generator. Bare names that exist under the framework's bundled +``libraries/`` directory (ESP8266WiFi, Wire, SPI, ...) are read straight from +the framework tree; everything else goes through the shared +resolution/download pipeline in ``esphome.platformio.library``. + +Mirrors PlatformIO's ``lib_ldf_mode=off`` behavior: each library builds into +its own static archive and every library's include dir joins one global +include path. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path + +from esphome.build_helpers.extra_script import apply_extra_script +from esphome.core import CORE, EsphomeError, Library +from esphome.platformio.library import ( + DEFAULT_BUILD_INCLUDE_DIR, + DEFAULT_BUILD_SRC_FILTER, + SRC_FILE_EXTENSIONS, + ConvertedLibrary, + InvalidLibrary, + LibraryBackend, + check_library_data, + collect_filtered_files, + convert_libraries, + ensure_list, + is_lib_ignored, + join_flag_args, + lib_ignore_set, + normalize_dependencies, + parse_library_properties, + split_flag_entry, +) + +_LOGGER = logging.getLogger(__name__) + +ESP8266_PLATFORM = "espressif8266" + + +@dataclass +class ArduinoLibrary: + """One resolved library, ready for the ninja generator.""" + + name: str + sources: list[Path] = field(default_factory=list) + include_dirs: list[Path] = field(default_factory=list) + # Extra compile flags private to this library's own sources + flags: list[str] = field(default_factory=list) + # Link inputs the library contributes (-L dirs / -l libs, e.g. from + # precompiled vendor blobs) and -Wl, options for the firmware link + link_dirs: list[Path] = field(default_factory=list) + link_libs: list[str] = field(default_factory=list) + link_flags: list[str] = field(default_factory=list) + + +def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: + """Resolve one library's sources, include dirs, and flags (PIO semantics).""" + build = data.get("build", {}) + + # PIO's source-dir resolution: manifest srcDir, else src/Src, else the root + src_dir = build.get("srcDir") or next( + (d for d in ("src", "Src") if (read_path / d).is_dir()), "." + ) + if "srcDir" in build and not (read_path / src_dir).is_dir(): + # Unlike the default probes, an explicitly declared srcDir that does + # not resolve is unambiguously a manifest/tree error; a silently + # empty source set would surface as link errors far from the cause + raise EsphomeError( + f"Library {name} declares srcDir {src_dir} which does not exist" + ) + + src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) + # PlatformIO shell-lexes each build.flags entry + flag_tokens = join_flag_args( + ( + token + for entry in ensure_list(build.get("flags", [])) + for token in split_flag_entry(entry, f"library {name}") + ), + f"library {name}", + ) + + lib = ArduinoLibrary(name=name) + include_flags: list[str] = [] + for tok in flag_tokens: + if tok.startswith("-I"): + include_flags.append(tok[2:]) + elif tok.startswith("-L"): + link_dir = (read_path / tok[2:]).resolve() + if not link_dir.is_dir(): + # Kept anyway (the linker ignores missing -L dirs); the + # warning names the culprit before a bare "cannot find -lfoo" + _LOGGER.warning( + "Library %s declares library dir %s which does not exist", + name, + tok[2:], + ) + lib.link_dirs.append(link_dir) + elif tok.startswith("-l"): + lib.link_libs.append(tok[2:]) + elif tok.startswith("-Wl,"): + lib.link_flags.append(tok) + else: + lib.flags.append(tok) + + include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR) + for d in [include_dir, src_dir, *include_flags]: + if (path := (read_path / d)).is_dir(): + lib.include_dirs.append(path.resolve()) + elif d in include_flags or (d == include_dir and "includeDir" in build): + # The includeDir/srcDir defaults are probes; an explicitly + # declared path that does not resolve is a manifest error + _LOGGER.warning( + "Library %s declares include dir %s which does not exist", name, d + ) + + lib.sources = sorted( + path.resolve() + for f in collect_filtered_files(read_path / src_dir, src_filter) + if (path := Path(f)).suffix in SRC_FILE_EXTENSIONS + ) + if not lib.sources and ("srcFilter" in build or "srcDir" in build): + # A default probe finding nothing is a header-only library; a + # declared filter matching nothing is a manifest/tree problem. + _LOGGER.warning( + "Library %s declares srcFilter/srcDir but no source files matched", + name, + ) + return lib + + +def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: + """A library bundled with the Arduino core, read from the framework tree.""" + lib_dir = framework_path / "libraries" / name + manifest = lib_dir / "library.properties" + data = parse_library_properties(manifest) if manifest.is_file() else {} + return _library_info(name, lib_dir, {"name": name, **data}) + + +def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]: + """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`.""" + bundled: list[ArduinoLibrary] = [] + external: list[Library] = [] + # PlatformIO's lib_ignore covers framework-bundled libraries too; the + # shared converter only filters the registry/git ones. + lib_ignore = lib_ignore_set() + for library in CORE.platformio_libraries.values(): + if is_lib_ignored(library.name, lib_ignore): + continue + # A version pin means a registry package ("pngle@1.1.0"), never a + # framework-bundled library. + if ( + library.repository + or library.version + or not library.name + or "/" in library.name + ): + external.append(library) + elif (framework_path / "libraries" / library.name).is_dir(): + bundled.append(_bundled_library(framework_path, library.name)) + else: + # A bare registry name; resolved at the latest version, matching + # PlatformIO (a typo fails loudly as a registry lookup error). + external.append(library) + + converted: list[ArduinoLibrary] = [] + bundled_names = {lib.name for lib in bundled} + + def _add_bundled_dependencies(component: ConvertedLibrary) -> None: + # A version-less bare-name dependency ("Hash" in ESPAsyncWebServer) + # is a core-bundled library; the shared converter skips it because + # it cannot be resolved from the registry. + for dep in normalize_dependencies(component.data.get("dependencies")): + name = dep.get("name") + if ( + not name + or dep.get("owner") + or "version" in dep + or name in bundled_names + or is_lib_ignored(name, lib_ignore) + ): + continue + if not (framework_path / "libraries" / name).is_dir(): + # The shared converter skips version-less deps too, so this + # is the only place the drop can be made visible before the + # missing sources surface as link errors. + _LOGGER.warning( + "Dependency %s of library %s is not bundled with the " + "framework and has no version to resolve; skipping", + name, + component.name, + ) + continue + try: + check_library_data(dep, ESP8266_PLATFORM, "arduino") + except InvalidLibrary as err: + # check_library_data's only raise is the platform filter, and + # rejecting another platform's dependency of a cross-platform + # manifest is routine (every ESPAsyncWebServer build hits it); + # a warning here would be noise, and the reason is in the log. + _LOGGER.debug("Skipping bundled dependency %s: %s", name, err) + continue + bundled_names.add(name) + bundled.append(_bundled_library(framework_path, name)) + + def _emit(component: ConvertedLibrary) -> None: + apply_extra_script(component, "esp8266", pio_platform=ESP8266_PLATFORM) + converted.append( + _library_info( + component.get_require_name(), component.source_dir, component.data + ) + ) + _add_bundled_dependencies(component) + + if external: + convert_libraries( + external, + LibraryBackend( + platform=ESP8266_PLATFORM, + framework="arduino", + emit=_emit, + cache_key="arduino8266", + ), + ) + + return bundled + converted diff --git a/tests/unit_tests/test_arduino8266_build_tool.py b/tests/unit_tests/test_arduino8266_build_tool.py new file mode 100644 index 0000000000..66d11a6700 --- /dev/null +++ b/tests/unit_tests/test_arduino8266_build_tool.py @@ -0,0 +1,63 @@ +"""Tests for the ninja build-tool helper script.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.arduino8266 import build_tool + + +def test_ar_removes_stale_archive(tmp_path: Path) -> None: + archive = tmp_path / "lib.a" + archive.write_text("stale") + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert not archive.exists() + assert mock_run.call_args[0][0] == ["ar-bin", "rc", str(archive), f"@{rsp}"] + + +def test_copy(tmp_path: Path) -> None: + src = tmp_path / "firmware.bin" + src.write_text("data") + dst = tmp_path / "firmware.factory.bin" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(dst)] + ): + assert build_tool.main() == 0 + assert dst.read_text() == "data" + + +def test_unknown_mode(capsys: pytest.CaptureFixture[str]) -> None: + with patch.object(build_tool.sys, "argv", ["build_tool", "bogus"]): + assert build_tool.main() == 1 + assert "unknown build_tool mode" in capsys.readouterr().err + + +def test_runs_as_script(tmp_path: Path) -> None: + """The ninja rules invoke the file as a plain script.""" + import subprocess + import sys + + src = tmp_path / "a.bin" + src.write_text("x") + dst = tmp_path / "b.bin" + result = subprocess.run( + [sys.executable, build_tool.__file__, "copy", str(src), str(dst)], + check=False, + ) + assert result.returncode == 0 + assert dst.read_text() == "x" diff --git a/tests/unit_tests/test_arduino8266_component.py b/tests/unit_tests/test_arduino8266_component.py new file mode 100644 index 0000000000..efe38a68dc --- /dev/null +++ b/tests/unit_tests/test_arduino8266_component.py @@ -0,0 +1,310 @@ +"""Tests for esphome.arduino8266.component (library resolution).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino8266 import component +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_ESP8266 +from esphome.core import CORE, EsphomeError, Library +from esphome.platformio.library import ConvertedLibrary, LibraryBackend + + +@pytest.fixture(autouse=True) +def _reset_libraries() -> None: + CORE.platformio_libraries = {} + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266} + + +def _add_library(name: str, version: str | None, repository: str | None = None) -> None: + CORE.add_library(Library(name=name, version=version, repository=repository)) + + +def _make_framework(tmp_path: Path) -> Path: + framework = tmp_path / "framework" + lib = framework / "libraries" / "ESP8266WiFi" / "src" + lib.mkdir(parents=True) + (lib / "ESP8266WiFi.cpp").write_text("") + (lib / "ESP8266WiFi.h").write_text("") + (lib.parent / "library.properties").write_text("name=ESP8266WiFi\nversion=1.0\n") + root_lib = framework / "libraries" / "Wire" + root_lib.mkdir(parents=True) + (root_lib / "Wire.cpp").write_text("") + (root_lib / "examples").mkdir() + (root_lib / "examples" / "scan.ino").write_text("") + return framework + + +def test_library_info_src_layout(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "ESP8266WiFi") + assert lib.name == "ESP8266WiFi" + assert [p.name for p in lib.sources] == ["ESP8266WiFi.cpp"] + assert lib.include_dirs == [(framework / "libraries/ESP8266WiFi/src").resolve()] + + +def test_library_info_root_layout_excludes_examples(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "Wire") + assert [p.name for p in lib.sources] == ["Wire.cpp"] + assert lib.include_dirs == [(framework / "libraries/Wire").resolve()] + + +def test_library_info_flags_parsing(tmp_path: Path) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "a.cpp").write_text("") + (read_path / "inc").mkdir() + (read_path / "blobs").mkdir() + data = { + "build": { + "flags": [ + "-DFOO=1 -I inc", + "-lalgobsec", + "-fno-lto", + "-Wl,--wrap=malloc", + "-l", + "m", + "-L", + "blobs", + ], + } + } + lib = component._library_info("x", read_path, data) + assert lib.flags == ["-DFOO=1", "-fno-lto"] + assert lib.include_dirs == [ + (read_path / "src").resolve(), + (read_path / "inc").resolve(), + ] + assert lib.link_dirs == [(read_path / "blobs").resolve()] + assert lib.link_libs == ["algobsec", "m"] + assert lib.link_flags == ["-Wl,--wrap=malloc"] + + +def test_library_info_missing_link_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + data = {"build": {"flags": ["-Lmissing_blobs"]}} + lib = component._library_info("x", read_path, data) + assert "declares library dir missing_blobs which does not exist" in caplog.text + # Kept anyway: the linker ignores missing -L dirs + assert lib.link_dirs == [(read_path / "missing_blobs").resolve()] + + +def test_library_info_declared_filter_matches_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + data = {"build": {"srcFilter": ["+"]}} + lib = component._library_info("x", read_path, data) + assert not lib.sources + assert "declares srcFilter/srcDir but no source files matched" in caplog.text + + +def test_library_info_header_only_does_not_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {}) + assert not lib.sources + assert "no source files matched" not in caplog.text + + +def test_library_info_no_src_dir(tmp_path: Path) -> None: + read_path = tmp_path / "empty" + read_path.mkdir() + lib = component._library_info("x", read_path, {}) + # With no manifest hints the source dir falls back to the library root + assert lib.sources == [] + assert lib.include_dirs == [read_path.resolve()] + + +def test_resolve_libraries_bundled(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + libs = component.resolve_libraries(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_bare_registry_name_is_external(tmp_path: Path) -> None: + """A bare name that is not bundled resolves from the registry at the + latest version, matching PlatformIO and the documented libraries: key.""" + framework = _make_framework(tmp_path) + _add_library("pngle", None) + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + component.resolve_libraries(framework) + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def _converted(name: str, source_dir: Path, data: dict) -> ConvertedLibrary: + converted = ConvertedLibrary(name, "1.0.0", source=None) + converted.path = source_dir + converted.data = data + return converted + + +def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + converted = _converted( + "esp32async__ESPAsyncWebServer", + lib_dir, + { + "build": {}, + "dependencies": [ + # Version-less bundled dependency: resolved from the framework + {"name": "Wire", "platforms": "espressif8266"}, + # Wrong platform: skipped + {"name": "ESP8266WiFi", "platforms": "espressif32"}, + # Registry dependency with a version: handled by the converter + {"name": "ESPAsyncTCP", "owner": "ESP32Async", "version": "^2.0.0"}, + # Not bundled: skipped + {"name": "NotBundled"}, + ], + }, + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + libs = component.resolve_libraries(framework) + + mock_extra.assert_called_once_with( + converted, "esp8266", pio_platform="espressif8266" + ) + assert [lib.name for lib in libs] == [ + "Wire", + "esp32async__ESPAsyncWebServer", + ] + + +def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("Wire", None) + _add_library("Some/External", "1.0.0") + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script"), + ): + libs = component.resolve_libraries(framework) + + # Wire appears once (from the explicit registration), not twice + assert [lib.name for lib in libs] == ["Wire", "some__External"] + + +def test_resolve_libraries_versioned_bare_name_is_external(tmp_path: Path) -> None: + """A bare name with a version pin ("pngle@1.1.0") is a registry package, + not a bundled library, and must reach the converter.""" + framework = _make_framework(tmp_path) + _add_library("pngle", "1.1.0") + + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + component.resolve_libraries(framework) + + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def test_library_info_trailing_bare_flag_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {"build": {"flags": ["-DA=1 -l"]}}) + assert lib.flags == ["-DA=1"] + assert lib.link_libs == [] + assert "Ignoring trailing '-l'" in caplog.text + + +def test_library_info_missing_explicit_include_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + lib = component._library_info("x", read_path, {"build": {"flags": ["-Inope"]}}) + assert lib.include_dirs == [(read_path / "src").resolve()] + assert "include dir nope which does not exist" in caplog.text + + +def test_library_info_missing_declared_src_dir_raises(tmp_path: Path) -> None: + """An explicitly declared srcDir that does not exist is a manifest error.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="srcDir nosrc which does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": "nosrc"}}) + + +def test_library_info_missing_declared_include_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + component._library_info("x", read_path, {"build": {"includeDir": "noinc"}}) + assert "include dir noinc which does not exist" in caplog.text + + +def test_resolve_libraries_lib_ignore_covers_bundled(tmp_path: Path) -> None: + """lib_ignore applies to framework-bundled libraries, as under PlatformIO.""" + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + _add_library("Wire", None) + CORE.platformio_options = {"lib_ignore": ["Wire"]} + libs = component.resolve_libraries(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( + tmp_path: Path, +) -> None: + framework = _make_framework(tmp_path) + _add_library("Some/External", "1.0.0") + CORE.platformio_options = {"lib_ignore": ["Wire"]} + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + backend.emit(converted) + return [converted] + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script"), + ): + libs = component.resolve_libraries(framework) + + assert [lib.name for lib in libs] == ["some__External"] From 61e32230d9210136c2f6bd8e820c5b148fcec542 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 13:02:52 -0500 Subject: [PATCH 5/6] Add the ESP8266 native build spec (knobs, defines, flags, linker scripts) --- esphome/build_gen/arduino8266.py | 428 ++++++++++++++++ .../unit_tests/build_gen/test_arduino8266.py | 460 ++++++++++++++++++ 2 files changed, 888 insertions(+) create mode 100644 esphome/build_gen/arduino8266.py create mode 100644 tests/unit_tests/build_gen/test_arduino8266.py diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py new file mode 100644 index 0000000000..fb46cfd7f1 --- /dev/null +++ b/esphome/build_gen/arduino8266.py @@ -0,0 +1,428 @@ +"""Native ninja build generator for the ESP8266 Arduino core. + +Transliterates the PlatformIO build spec for the Arduino ESP8266 framework +(``framework-arduinoespressif8266/tools/platformio-build.py`` plus +``platform-espressif8266/builder/main.py``) into a ``build.ninja`` under +``.pioenvs//``. The flag sets, defines, link line, linker-script +generation, and ``elf2bin`` invocation deliberately match what PlatformIO +produces so the binaries stay near-identical between the two toolchains. + +The ``PIO_FRAMEWORK_ARDUINO_*`` knob defines (lwIP variant, NONOS SDK +version, MMU layout, exceptions, waveform phase) keep working: they are read +from the build flags with the same precedence as the PlatformIO builder. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path +import re +import subprocess + +from esphome.components.esp8266 import build_surgery +from esphome.components.esp8266.boards import BOARDS, ESP8266_LD_SCRIPTS +from esphome.components.esp8266.const import KEY_BOARD, KEY_ESP8266, KEY_FLASH_SIZE +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION +from esphome.core import CORE, EsphomeError +from esphome.helpers import mkdir_p, write_file_if_changed +from esphome.platformio.library import join_flag_args, split_flag_entry + +_LOGGER = logging.getLogger(__name__) + +# From platformio-build.py. The first entry is the default; with multiple SDK +# knobs set (a pathological config) ties break by table order, since +# upstream's tie-break depends on define order and is not reproducible here. +_NONOSDK_VERSIONS = ( + ("SDK22x_190703", "NONOSDK22x_190703"), + ("SDK221", "NONOSDK221"), + ("SDK22x_190313", "NONOSDK22x_190313"), + ("SDK22x_191024", "NONOSDK22x_191024"), + ("SDK22x_191105", "NONOSDK22x_191105"), + ("SDK22x_191122", "NONOSDK22x_191122"), + ("SDK305", "NONOSDK305"), +) + +# knob define -> (TCP_MSS, LWIP_FEATURES, LWIP_IPV6, library name) +_LWIP_VARIANTS = ( + ("PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY", (536, 1, 1, "lwip6-536-feat")), + ( + "PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_HIGHER_BANDWIDTH", + (1460, 1, 1, "lwip6-1460-feat"), + ), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH", (1460, 1, 0, "lwip2-1460-feat")), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY_LOW_FLASH", (536, 0, 0, "lwip2-536")), + ( + "PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", + (1460, 0, 0, "lwip2-1460"), + ), +) +_LWIP_DEFAULT = (536, 1, 0, "lwip2-536-feat") + +_ASFLAGS = ["-mlongcalls", "-mtext-section-literals"] +_CFLAGS = [ + "-std=gnu17", + "-Wpointer-arith", + "-Wno-implicit-function-declaration", + "-Wl,-EL", + "-fno-inline-functions", + "-nostdlib", +] +_CCFLAGS = [ + "-Os", + "-mlongcalls", + "-mtext-section-literals", + "-falign-functions=4", + "-U__STRICT_ANSI__", + "-ffunction-sections", + "-fdata-sections", + "-Wall", + "-Werror=return-type", + "-free", + "-fipa-pta", +] +_LINKFLAGS = [ + "-Os", + "-nostdlib", + "-Wl,--no-check-sections", + "-Wl,-static", + "-Wl,--gc-sections", + "-Wl,-wrap,system_restart_local", + "-Wl,-wrap,spi_flash_read", + "-u", + "app_entry", + "-u", + "_printf_float", + "-u", + "_DebugExceptionVector", + "-u", + "_DoubleExceptionVector", + "-u", + "_KernelExceptionVector", + "-u", + "_NMIExceptionVector", + "-u", + "_UserExceptionVector", +] +_SYSTEM_LIBS_PRE_LWIP = ["hal", "phy", "pp", "net80211"] +_SYSTEM_LIBS_POST_LWIP = [ + "wpa", + "crypto", + "main", + "wps", + "bearssl", + "espnow", + "smartconfig", + "airkiss", + "wpa2", +] + + +@dataclass +class _BuildConfig: + """Knob-derived build configuration (PIO_FRAMEWORK_ARDUINO_* defines).""" + + nonosdk: str + lwip_lib: str + exceptions: bool + vtables: str + fp_in_irom: bool + knob_defines: list[str] = field(default_factory=list) + mmu_defines: list[str] = field(default_factory=list) + + +def _flag_defines() -> dict[str, str]: + """Map define name -> full ``NAME[=VALUE]`` for every -D build flag.""" + defines: dict[str, str] = {} + for flag in CORE.build_flags: + # Shell-lex multi-token entries the way PlatformIO does, so a knob + # in "-DKNOB -DOTHER" or a spaced "-D KNOB" is still detected; + # single tokens pass verbatim to keep quoting in their bodies intact. + tokens = ( + join_flag_args(split_flag_entry(flag, "esphome"), "esphome") + if " " in flag + else (flag,) + ) + for tok in tokens: + if tok.startswith("-D") and len(tok) > 2: + body = tok[2:] + defines[body.split("=", 1)[0]] = body + return defines + + +def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig: + nonosdk = _NONOSDK_VERSIONS[0][1] + for name, define in _NONOSDK_VERSIONS: + if f"PIO_FRAMEWORK_ARDUINO_ESPRESSIF_{name}" in defines: + nonosdk = define + break + + tcp_mss, features, ipv6, lwip_lib = _LWIP_DEFAULT + for knob, variant in _LWIP_VARIANTS: + if knob in defines: + tcp_mss, features, ipv6, lwip_lib = variant + break + + knob_defines = [ + f"{nonosdk}=1", + f"TCP_MSS={tcp_mss}", + f"LWIP_FEATURES={features}", + f"LWIP_IPV6={ipv6}", + ] + if "PIO_FRAMEWORK_ARDUINO_WAVEFORM_LOCKED_PHASE" in defines: + knob_defines.append("WAVEFORM_LOCKED_PHASE=1") + + # Sorted so the pick is deterministic: the dict is built from a set of + # build flags, whose iteration order varies between processes. + vtables = next( + (name for name in sorted(defines) if name.startswith("VTABLES_IN_")), + "VTABLES_IN_FLASH", + ) + + if "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48" in defines: + mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED" in defines: + mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x4000", + "MMU_SEC_HEAP_SIZE=0x4000", + "MMU_SEC_HEAP=0x40108000", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x8000", + "MMU_EXTERNAL_HEAP=128", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K" in defines: + mmu = [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x8000", + "MMU_EXTERNAL_HEAP=256", + ] + elif "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM" in defines: + if "MMU_IRAM_SIZE" not in defines or "MMU_ICACHE_SIZE" not in defines: + raise EsphomeError( + "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM requires MMU_IRAM_SIZE and " + "MMU_ICACHE_SIZE build flags" + ) + # Sorted so build.ninja and the linker-script stamp stay byte-stable + # across runs (the flag set has no deterministic iteration order). + mmu = sorted(body for name, body in defines.items() if name.startswith("MMU_")) + else: + mmu = ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + + return _BuildConfig( + nonosdk=nonosdk, + lwip_lib=lwip_lib, + exceptions="PIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS" in defines, + vtables=vtables, + fp_in_irom="FP_IN_IROM" in defines, + knob_defines=knob_defines, + mmu_defines=mmu, + ) + + +def _flash_ld_name(board: str) -> str: + return ESP8266_LD_SCRIPTS[BOARDS[board][KEY_FLASH_SIZE]][1] + + +def _e(value) -> str: + """Escape a path or token for a ninja file.""" + return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + + +def _quote_arg(tok: str) -> str: + """Wrap a token in double quotes with the Windows argv rule. + + Same escaping rule as ``subprocess.list2cmdline``: a backslash run + doubles only immediately before a quote (or the closing quote), and the + quote itself is escaped. POSIX sh parses the result identically for + backslashes and quotes. ``$`` must already be doubled for ninja. + """ + quoted = re.sub(r'(\\*)"', lambda m: m.group(1) * 2 + '\\"', tok) + quoted = re.sub(r"(\\+)\Z", lambda m: m.group(1) * 2, quoted) + return f'"{quoted}"' + + +def _q(value) -> str: + """Force-quote a path for the ninja command line (shell/CreateProcess).""" + return _quote_arg(str(value).replace("$", "$$")) + + +_NEEDS_QUOTE = re.compile(r'[\s"\']') + + +def _shell_token(tok: str) -> str: + """Quote a lexed token only when needed; ``_q`` force-quotes paths. + + Lexing strips the quoting a user wrote (``-DX="a b"`` becomes the single + token ``-DX=a b``); re-quote on the way out so the compiler receives the + same argv element SCons would pass under PlatformIO. After ninja + un-doubles ``$$``, sh still expands ``$VAR`` while CreateProcess passes + it literally -- the same divergence SCons-under-sh has, so this stays + PlatformIO parity. + """ + tok = tok.replace("$", "$$") # ninja would expand a bare $ to nothing + if not _NEEDS_QUOTE.search(tok): + return tok + return _quote_arg(tok) + + +def _defines_flags( + config: _BuildConfig, flash_mode: str, board: str, board_defines: tuple[str, ...] +) -> list[str]: + return [ + f"-D{d}" + for d in ( + "F_CPU=80000000L", + "__ets__", + "ICACHE_FLASH", + "_GNU_SOURCE", + "ARDUINO=10805", + f'ARDUINO_BOARD=\\"PLATFORMIO_{board.upper()}\\"', + f'ARDUINO_BOARD_ID=\\"{board}\\"', + f"FLASHMODE_{flash_mode.upper()}", + "LWIP_OPEN_SRC", + *config.knob_defines, + config.vtables, + *config.mmu_defines, + "ESP8266", + "ARDUINO_ARCH_ESP8266", + *board_defines, + ) + ] + + +def _unflag_tokens() -> set[str]: + """``build_unflags`` entries shell-lexed to tokens, as PlatformIO matches.""" + return { + tok + for entry in CORE.build_unflags + for tok in split_flag_entry(entry, "esphome build_unflags") + } + + +def _project_flags( + unflags: set[str] | None = None, +) -> tuple[list[str], list[str], list[Path], list[str]]: + """Split the ESPHome build flags into compile, linker, -L, and -l lists. + + Every entry is shell-lexed the way PlatformIO's ``ParseFlags`` does, so a + linker flag anywhere in an entry reaches the link line and + ``build_unflags`` matches individual tokens (``-Os`` inside ``-Os -g3``). + Lexed tokens are re-quoted at emission via ``_shell_token``. + """ + if unflags is None: + unflags = _unflag_tokens() + compile_flags: list[str] = [] + link_flags: list[str] = [] + lib_dirs: list[Path] = [] + libs: list[str] = [] + for flag in sorted(CORE.build_flags): + for tok in join_flag_args(split_flag_entry(flag, "esphome"), "esphome"): + if tok in unflags: + continue + if tok.startswith("-Wl,"): + link_flags.append(_shell_token(tok)) + elif tok.startswith("-L"): + lib_dirs.append(Path(tok[2:])) + elif tok.startswith("-l"): + libs.append(tok[2:]) + else: + compile_flags.append(_shell_token(tok)) + return compile_flags, link_flags, lib_dirs, libs + + +def generate_ld_scripts( + paths: dict[str, Path], config: _BuildConfig, flash_ld_name: str +) -> None: + """Generate the common linker script (and testing-mode flash ld copy). + + Runs the same preprocessor invocation as the PlatformIO builder over + ``eagle.app.v6.common.ld.h``, then applies ESPHome's surgeries: the wifi + rate-table DRAM relocation, and enlarged memory segments in testing mode. + """ + framework = paths["framework_path"] + gcc = paths["toolchain_path"] / "bin" / "xtensa-lx106-elf-gcc" + ld_dir = CORE.relative_pioenvs_path(CORE.name, "ld") + mkdir_p(ld_dir) + + cmd = [str(gcc), "-CC", "-E", "-P", f"-D{config.vtables}"] + cmd += [f"-D{d}" for d in config.mmu_defines] + if config.fp_in_irom: + cmd.append("-DFP_IN_IROM") + cmd += [ + str(framework / "tools" / "sdk" / "ld" / "eagle.app.v6.common.ld.h"), + "-o", + "-", + ] + + # The inputs are the command line (defines + framework version, which is + # baked into the paths) plus testing mode; skip the preprocessor spawn on + # incremental builds when nothing changed. + output = ld_dir / "local.eagle.app.v6.common.ld" + stamp = ld_dir / ".local.eagle.app.v6.common.ld.stamp" + # The surgery constants are inputs too: an edit to build_surgery.py must + # invalidate existing build dirs, not wait for an esphome clean. + stamp_content = ( + " ".join(cmd) + + f" testing={CORE.testing_mode}" + + f" {build_surgery.RATETABLE_RULE}" + + f" {build_surgery.TESTING_IRAM_SIZE}" + + f" {build_surgery.TESTING_DRAM_SIZE}" + + f" {build_surgery.TESTING_FLASH_SIZE}" + ) + if not ( + output.is_file() + and stamp.is_file() + and stamp.read_text(encoding="utf-8") == stamp_content + ): + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, close_fds=False + ) + if result.returncode != 0: + raise EsphomeError(f"Generating the linker script failed:\n{result.stderr}") + content = build_surgery.relocate_ratetable(result.stdout) + if CORE.testing_mode: + content = build_surgery.apply_testing_memory_patches( + content, require=("iram1_0_seg",) + ) + write_file_if_changed(output, content) + stamp.write_text(stamp_content, encoding="utf-8") + + if CORE.testing_mode: + # A patched copy of the flash ld in the build dir; resolved through + # the same -L path as the SDK original it shadows. + flash_ld = framework / "tools" / "sdk" / "ld" / flash_ld_name + write_file_if_changed( + ld_dir / f"testing_{flash_ld_name}", + build_surgery.apply_testing_memory_patches( + flash_ld.read_text(encoding="utf-8"), + require=("dram0_0_seg", "irom0_0_seg"), + ), + ) + + +def get_flash_ld_path(build_dir: Path) -> Path: + """The flash linker script the link actually uses (for size reporting).""" + from esphome.arduino8266.framework import ( + framework_package_version, + get_framework_path, + ) + + name = _flash_ld_name(CORE.data[KEY_ESP8266][KEY_BOARD]) + if CORE.testing_mode: + return build_dir / "ld" / f"testing_{name}" + version = framework_package_version(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]) + return get_framework_path(version) / "tools" / "sdk" / "ld" / name + + +def _flash_size_str(flash_ld_name: str) -> str: + """Flash size for elf2bin, derived from the ld script name (PIO logic).""" + match = re.search(r"\.flash\.(\d+)([mk])", flash_ld_name) + if not match: + raise EsphomeError(f"Cannot parse flash size from {flash_ld_name}") + return f"{match.group(1)}{match.group(2).upper()}" diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py new file mode 100644 index 0000000000..b214e45e49 --- /dev/null +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -0,0 +1,460 @@ +"""Drift tests for the native ESP8266 Arduino build generator. + +These pin the build spec transliterated from the PlatformIO builder +(framework-arduinoespressif8266/tools/platformio-build.py and +platform-espressif8266/builder/main.py) so a change on either side of the +toolchain seam is caught: the knob-define precedence, the define/flag sets, +the link line, and the core source exclusions must keep matching what the +PlatformIO toolchain produces for the same configuration. +""" + +from __future__ import annotations + +from collections.abc import Generator +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.build_gen import arduino8266 +from esphome.build_gen.arduino8266 import ( + _defines_flags, + _flag_defines, + _flash_size_str, + _resolve_build_config, + get_flash_ld_path, +) +from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD +from esphome.components.esp8266.build_surgery import RATETABLE_RULE +from esphome.components.esp8266.const import ( + KEY_BOARD, + KEY_ESP8266, + KEY_FLASH_MODE, + KEY_SCANF_FLOAT, +) +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _setup_core(tmp_path: Path) -> Generator[None]: + CORE.name = "test8266" + CORE.build_path = tmp_path + CORE.testing_mode = False + CORE.cpp_standard = "gnu++20" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 1, 2)} + CORE.data[KEY_ESP8266] = { + KEY_BOARD: "nodemcuv2", + KEY_FLASH_MODE: "dout", + KEY_SCANF_FLOAT: False, + } + yield + # CORE.reset() (the suite-wide autouse fixture) does not clear this flag + CORE.testing_mode = False + + +def _set_flags(*flags: str) -> None: + CORE.build_flags = set(flags) + + +def test_board_build_covers_every_board() -> None: + """Every supported board must have variant/define metadata.""" + assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) + + +def test_build_config_defaults() -> None: + + _set_flags() + config = _resolve_build_config(_flag_defines()) + assert config.nonosdk == "NONOSDK22x_190703" + assert config.lwip_lib == "lwip2-536-feat" + assert not config.exceptions + assert config.vtables == "VTABLES_IN_FLASH" + assert config.knob_defines == [ + "NONOSDK22x_190703=1", + "TCP_MSS=536", + "LWIP_FEATURES=1", + "LWIP_IPV6=0", + ] + assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + + +def test_build_config_esphome_lwip_knob() -> None: + """The lwIP variant ESPHome selects maps to the same defines and library + as the PlatformIO builder.""" + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + config = _resolve_build_config(_flag_defines()) + assert config.lwip_lib == "lwip2-1460" + assert "TCP_MSS=1460" in config.knob_defines + assert "LWIP_FEATURES=0" in config.knob_defines + assert "LWIP_IPV6=0" in config.knob_defines + + +def test_build_config_knobs() -> None: + + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_ESPRESSIF_SDK305", + "-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS", + "-DPIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48", + "-DVTABLES_IN_DRAM", + ) + config = _resolve_build_config(_flag_defines()) + assert config.nonosdk == "NONOSDK305" + assert config.exceptions + assert config.vtables == "VTABLES_IN_DRAM" + assert config.mmu_defines == ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"] + + +def test_build_config_mmu_custom_requires_sizes() -> None: + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM") + with pytest.raises(EsphomeError, match="MMU_IRAM_SIZE"): + _resolve_build_config(_flag_defines()) + + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM", + "-DMMU_IRAM_SIZE=0xC000", + "-DMMU_ICACHE_SIZE=0x4000", + ) + config = _resolve_build_config(_flag_defines()) + # Emitted pre-sorted so build.ninja stays byte-stable across runs + assert config.mmu_defines == [ + "MMU_ICACHE_SIZE=0x4000", + "MMU_IRAM_SIZE=0xC000", + ] + + +def test_defines_match_platformio_builder() -> None: + """The exact define set the PlatformIO builder passes for nodemcuv2/dout.""" + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + assert _defines_flags( + _resolve_build_config(_flag_defines()), + "dout", + "nodemcuv2", + ESP8266_BOARD_BUILD["nodemcuv2"]["defines"], + ) == [ + "-DF_CPU=80000000L", + "-D__ets__", + "-DICACHE_FLASH", + "-D_GNU_SOURCE", + "-DARDUINO=10805", + '-DARDUINO_BOARD=\\"PLATFORMIO_NODEMCUV2\\"', + '-DARDUINO_BOARD_ID=\\"nodemcuv2\\"', + "-DFLASHMODE_DOUT", + "-DLWIP_OPEN_SRC", + "-DNONOSDK22x_190703=1", + "-DTCP_MSS=1460", + "-DLWIP_FEATURES=0", + "-DLWIP_IPV6=0", + "-DVTABLES_IN_FLASH", + "-DMMU_IRAM_SIZE=0x8000", + "-DMMU_ICACHE_SIZE=0x8000", + "-DESP8266", + "-DARDUINO_ARCH_ESP8266", + "-DARDUINO_ESP8266_NODEMCU_ESP12E", + ] + + +def _make_framework(tmp_path: Path) -> dict[str, Path]: + framework = tmp_path / "framework" + core = framework / "cores" / "esp8266" + core.mkdir(parents=True) + for name in ( + "core_esp8266_main.cpp", + "Updater.cpp", + "core_esp8266_waveform_pwm.cpp", + "core_esp8266_waveform_phase.cpp", + "cont.S", + "abi.c", + ): + (core / name).write_text("") + (framework / "variants" / "nodemcu").mkdir(parents=True) + for sub in ("include", "ld", "lwip2/include", "lib"): + (framework / "tools" / "sdk" / sub).mkdir(parents=True) + (framework / "libraries").mkdir() + toolchain = tmp_path / "toolchain" + (toolchain / "bin").mkdir(parents=True) + (toolchain / "include").mkdir() + return { + "framework_path": framework, + "toolchain_path": toolchain, + "ninja_path": Path("ninja"), + } + + +@pytest.mark.parametrize( + ("knob", "lib", "mss", "features", "ipv6"), + [ + ("PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY", "lwip6-536-feat", 536, 1, 1), + ( + "PIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_HIGHER_BANDWIDTH", + "lwip6-1460-feat", + 1460, + 1, + 1, + ), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH", "lwip2-1460-feat", 1460, 1, 0), + ("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY_LOW_FLASH", "lwip2-536", 536, 0, 0), + ], +) +def test_build_config_lwip_variants( + knob: str, lib: str, mss: int, features: int, ipv6: int +) -> None: + """Every lwIP knob maps to the same defines and library as the PIO builder.""" + + _set_flags(f"-D{knob}") + config = _resolve_build_config(_flag_defines()) + assert config.lwip_lib == lib + assert f"TCP_MSS={mss}" in config.knob_defines + assert f"LWIP_FEATURES={features}" in config.knob_defines + assert f"LWIP_IPV6={ipv6}" in config.knob_defines + + +@pytest.mark.parametrize( + ("knob", "expected"), + [ + ( + "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED", + ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED", + [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x4000", + "MMU_SEC_HEAP_SIZE=0x4000", + "MMU_SEC_HEAP=0x40108000", + ], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K", + ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000", "MMU_EXTERNAL_HEAP=128"], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K", + ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000", "MMU_EXTERNAL_HEAP=256"], + ), + ], +) +def test_build_config_mmu_variants(knob: str, expected: list[str]) -> None: + + _set_flags(f"-D{knob}") + assert _resolve_build_config(_flag_defines()).mmu_defines == expected + + +def test_build_config_waveform_locked_phase() -> None: + + _set_flags("-DPIO_FRAMEWORK_ARDUINO_WAVEFORM_LOCKED_PHASE", "-DFP_IN_IROM") + config = _resolve_build_config(_flag_defines()) + assert "WAVEFORM_LOCKED_PHASE=1" in config.knob_defines + assert config.fp_in_irom + + +_COMMON_LD_H_OUTPUT = """\ +MEMORY +{ + iram1_0_seg : org = 0x40100000, len = 0x8000 +} +SECTIONS +{ + .data : ALIGN(4) + { + _data_start = ABSOLUTE(.); + } >dram0_0_seg :dram0_0_phdr +} +""" + + +def _run_generate_ld_scripts(paths: dict[str, Path]) -> Path: + + config = _resolve_build_config(_flag_defines()) + arduino8266.generate_ld_scripts(paths, config, "eagle.flash.4m.ld") + return CORE.relative_pioenvs_path(CORE.name, "ld") + + +def test_generate_ld_scripts(tmp_path: Path) -> None: + + paths = _make_framework(tmp_path) + _set_flags("-DFP_IN_IROM") + result = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT) + with patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run: + ld_dir = _run_generate_ld_scripts(paths) + content = (ld_dir / "local.eagle.app.v6.common.ld").read_text() + assert RATETABLE_RULE in content + cmd = mock_run.call_args[0][0] + assert "-DVTABLES_IN_FLASH" in cmd + assert "-DMMU_IRAM_SIZE=0x8000" in cmd + assert "-DFP_IN_IROM" in cmd + + # Unchanged inputs skip the preprocessor spawn on the next run + with patch.object(arduino8266.subprocess, "run") as mock_run: + _run_generate_ld_scripts(paths) + mock_run.assert_not_called() + + # An edit to the surgery constants invalidates the stamp (a stale linker + # script would otherwise persist until an esphome clean) + with ( + patch.object(arduino8266.build_surgery, "TESTING_FLASH_SIZE", "0x3000000"), + patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run, + ): + _run_generate_ld_scripts(paths) + mock_run.assert_called_once() + + +def test_generate_ld_scripts_failure(tmp_path: Path) -> None: + + paths = _make_framework(tmp_path) + result = MagicMock(returncode=1, stderr="nope") + with ( + patch.object(arduino8266.subprocess, "run", return_value=result), + pytest.raises(EsphomeError, match="linker script failed"), + ): + _run_generate_ld_scripts(paths) + + +def test_generate_ld_scripts_testing_mode(tmp_path: Path) -> None: + + paths = _make_framework(tmp_path) + (paths["framework_path"] / "tools" / "sdk" / "ld" / "eagle.flash.4m.ld").write_text( + "MEMORY\n{\n" + " dram0_0_seg : org = 0x3FFE8000, len = 0x14000\n" + " iram1_0_seg : org = 0x40100000, len = 0x8000\n" + " irom0_0_seg : org = 0x40201010, len = 0xfeff0\n" + "}\n" + ) + CORE.testing_mode = True + result = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT) + with patch.object(arduino8266.subprocess, "run", return_value=result): + ld_dir = _run_generate_ld_scripts(paths) + patched = (ld_dir / "testing_eagle.flash.4m.ld").read_text() + assert "len = 0x2000000" in patched + + +def test_get_flash_ld_path(tmp_path: Path) -> None: + + CORE.testing_mode = True + assert get_flash_ld_path(tmp_path) == ( + tmp_path / "ld" / "testing_eagle.flash.4m.ld" + ) + + CORE.testing_mode = False + with ( + patch( + "esphome.arduino8266.framework.get_framework_path", + return_value=tmp_path / "framework", + ), + patch( + "esphome.arduino8266.framework.framework_package_version", + return_value="3.30102.0", + ), + ): + assert get_flash_ld_path(tmp_path) == ( + tmp_path / "framework" / "tools" / "sdk" / "ld" / "eagle.flash.4m.ld" + ) + + +def test_flash_size_str() -> None: + + assert _flash_size_str("eagle.flash.4m.ld") == "4M" + assert _flash_size_str("eagle.flash.512k.ld") == "512K" + with pytest.raises(EsphomeError, match="Cannot parse flash size"): + _flash_size_str("bogus.ld") + + +def test_build_config_nonosdk_precedence() -> None: + """With two SDK knobs set (a pathological config), ties break + deterministically by table order.""" + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_ESPRESSIF_SDK305", + "-DPIO_FRAMEWORK_ARDUINO_ESPRESSIF_SDK221", + ) + assert _resolve_build_config(_flag_defines()).nonosdk == "NONOSDK221" + + +def test_project_flags_trailing_bare_linker_flag_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + _set_flags("-l") + compile_flags, link_flags, lib_dirs, libs = arduino8266._project_flags() + assert "Ignoring trailing '-l'" in caplog.text + assert not libs + assert not lib_dirs + assert "-l" not in compile_flags + assert "-l" not in link_flags + + +def test_project_flags_lexed_entry_scatters_non_linker_tokens() -> None: + _set_flags("-L /d -Wl,-Map=m stray") + compile_flags, link_flags, lib_dirs, libs = arduino8266._project_flags() + assert lib_dirs == [Path("/d")] + assert link_flags == ["-Wl,-Map=m"] + assert "stray" in compile_flags + assert not libs + + +def test_flag_defines_lexes_multi_token_entries() -> None: + """A knob inside a multi-token entry is detected like PlatformIO does.""" + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH -DFOO=1 -Os") + defines = _flag_defines() + assert "PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH" in defines + assert defines["FOO"] == "FOO=1" + config = _resolve_build_config(defines) + assert config.lwip_lib == "lwip2-1460" + + +def test_project_flags_lexes_every_entry() -> None: + """A linker flag anywhere in an entry reaches the link line (PIO parity).""" + _set_flags("-DFOO=1 -lbar") + compile_flags, _link, _dirs, libs = arduino8266._project_flags() + assert libs == ["bar"] + assert "-DFOO=1" in compile_flags + + +def test_project_flags_unflags_match_tokens() -> None: + """build_unflags removes a token embedded in a multi-token entry.""" + _set_flags("-Os -g3") + CORE.build_unflags = {"-Os"} + compile_flags, _link, _dirs, _libs = arduino8266._project_flags( + arduino8266._unflag_tokens() + ) + assert "-g3" in compile_flags + assert "-Os" not in compile_flags + + +def test_project_flags_requotes_lexed_defines() -> None: + """A quoted spaced value stays one compiler argument after lex/emit.""" + _set_flags('-DGREETING="hello world"') + compile_flags, _link, _dirs, _libs = arduino8266._project_flags() + # shlex folds the quotes (as PIO's ParseFlags does); _shell_token + # re-quotes the spaced token so the shell passes one argv element + assert compile_flags == ['"-DGREETING=hello world"'] + + +def test_shell_token_escaping() -> None: + """Tokens survive both POSIX sh and the Windows CRT argv parser.""" + assert arduino8266._shell_token("-Os") == "-Os" + # $ would be expanded (to nothing) by ninja itself + assert arduino8266._shell_token("-DX=$HOME") == "-DX=$$HOME" + # Backslashes not before a quote stay single (Windows path in a define) + assert arduino8266._shell_token("-DP=C:\\x y") == '"-DP=C:\\x y"' + # A quote is escaped and the preceding backslash run doubles + assert arduino8266._shell_token('-DX=a\\"b c') == '"-DX=a\\\\\\"b c"' + # A trailing backslash run doubles before the closing quote + assert arduino8266._shell_token("a b\\") == '"a b\\\\"' + + +def test_flag_defines_joins_spaced_define() -> None: + """A spaced "-D KNOB" entry is detected exactly as PlatformIO detects it.""" + _set_flags("-D PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + defines = _flag_defines() + assert "PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH" in defines + assert "" not in defines + + +def test_ninja_path_escaping() -> None: + """Build-statement paths and command-line paths escape differently.""" + assert arduino8266._e("a b:$c") == "a$ b$:$$c" + assert arduino8266._q("/a b/$x") == '"/a b/$$x"' From 207968f301a33c1c23f85b99bcdb2413990c327a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 13:03:50 -0500 Subject: [PATCH 6/6] Add the ESP8266 native ninja emission --- esphome/build_gen/arduino8266.py | 302 +++++++++++++++++- .../unit_tests/build_gen/test_arduino8266.py | 227 ++++++++++++- 2 files changed, 524 insertions(+), 5 deletions(-) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index fb46cfd7f1..bccdb23d0d 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -16,20 +16,60 @@ from __future__ import annotations from dataclasses import dataclass, field import logging +import os from pathlib import Path import re import subprocess +import sys from esphome.components.esp8266 import build_surgery -from esphome.components.esp8266.boards import BOARDS, ESP8266_LD_SCRIPTS -from esphome.components.esp8266.const import KEY_BOARD, KEY_ESP8266, KEY_FLASH_SIZE +from esphome.components.esp8266.boards import ( + BOARDS, + ESP8266_BOARD_BUILD, + ESP8266_LD_SCRIPTS, +) +from esphome.components.esp8266.const import ( + KEY_BOARD, + KEY_ESP8266, + KEY_FLASH_MODE, + KEY_FLASH_SIZE, + KEY_SCANF_FLOAT, +) from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import get_project_cxx_compile_flags from esphome.helpers import mkdir_p, write_file_if_changed from esphome.platformio.library import join_flag_args, split_flag_entry _LOGGER = logging.getLogger(__name__) +# Compile rule per source suffix; keys must cover SRC_FILE_EXTENSIONS so any +# source a library manifest selects has a rule (pinned by a drift test). +_RULE_FOR_SUFFIX = { + ".c": "cc", + ".cpp": "cxx", + ".cc": "cxx", + ".cxx": "cxx", + ".c++": "cxx", + ".S": "asm", + ".spp": "asm", + ".SPP": "asm", + ".sx": "asm", + ".s": "asm", + ".asm": "asm", + ".ASM": "asm", +} + +# Always excluded from the core build: ESPHome uses its own native OTA +# backend, so the Arduino Updater (and its 228-byte global) never links. +_CORE_EXCLUDE_ALWAYS = {"Updater.cpp"} +# Excluded when no component called require_waveform(); waveform_stubs.cpp +# supplies the stopWaveform()/_stopPWM() stubs digitalWrite needs. +_CORE_EXCLUDE_WAVEFORM = { + "core_esp8266_waveform_pwm.cpp", + "core_esp8266_waveform_phase.cpp", +} + # From platformio-build.py. The first entry is the default; with multiple SDK # knobs set (a pathological config) ties break by table order, since # upstream's tie-break depends on define order and is not reproducible here. @@ -336,6 +376,14 @@ def _project_flags( return compile_flags, link_flags, lib_dirs, libs +def _collect_sources(root: Path, exclude: set[str] = frozenset()) -> list[Path]: + return sorted( + p + for p in root.rglob("*") + if p.suffix in _RULE_FOR_SUFFIX and p.name not in exclude + ) + + def generate_ld_scripts( paths: dict[str, Path], config: _BuildConfig, flash_ld_name: str ) -> None: @@ -406,6 +454,256 @@ def generate_ld_scripts( ) +def _ninja_compile_edges( + lines: list[str], + sources: list[Path], + root: Path, + group: str, + flags: str = "", +) -> list[str]: + """Emit compile edges for ``sources``; return the object paths.""" + objects = [] + for src in sources: + rel = src.relative_to(root).as_posix() + obj = f"obj/{group}/{rel}.o" + lines.append(f"build {_e(obj)}: {_RULE_FOR_SUFFIX[src.suffix]} {_e(src)}") + if flags: + lines.append(f" flags = {flags}") + # Escaped once here: the returned paths only ever appear in build + # statements (archive/link inputs), which use ninja escaping. + objects.append(_e(obj)) + return objects + + +def _common_parent(paths: list[Path]) -> Path: + return Path(os.path.commonpath([str(p.parent) for p in paths])) + + +def write_project(paths: dict[str, Path]) -> bool: + """Write the ninja build for the current configuration. + + Returns True when ``build.ninja`` changed, so the caller can skip work + derived purely from it (the compile database) on unchanged builds. + """ + from esphome.arduino8266.component import resolve_libraries + from esphome.arduino8266.framework import ccache_path + + framework = paths["framework_path"] + toolchain_bin = paths["toolchain_path"] / "bin" + build_dir = CORE.relative_pioenvs_path(CORE.name) + mkdir_p(build_dir) + + flag_defines = _flag_defines() + config = _resolve_build_config(flag_defines) + esp8266_data = CORE.data[KEY_ESP8266] + # Board support was validated at config time (_validate_native_toolchain). + board = esp8266_data[KEY_BOARD] + board_build = ESP8266_BOARD_BUILD[board] + flash_ld_name = _flash_ld_name(board) + + generate_ld_scripts(paths, config, flash_ld_name) + + sdk = framework / "tools" / "sdk" + core_dir = framework / "cores" / "esp8266" + variant_dir = framework / "variants" / board_build["variant"] + src_dir = CORE.relative_src_path() + + libraries = resolve_libraries(framework) + + # A missing install directory would otherwise surface as a wall of + # include errors; failing here names the path instead. + include_dirs = [ + src_dir, + sdk / "include", + core_dir, + paths["toolchain_path"] / "include", + sdk / "lwip2" / "include", + variant_dir, + ] + for required in include_dirs: + if not required.is_dir(): + raise EsphomeError( + f"Arduino toolchain install is incomplete: missing {required}; " + "run 'esphome clean-all' and retry" + ) + for lib in libraries: + include_dirs += lib.include_dirs + + unflags = _unflag_tokens() + ( + project_compile_flags, + project_link_flags, + project_lib_dirs, + project_libs, + ) = _project_flags(unflags) + defines = _defines_flags( + config, esp8266_data[KEY_FLASH_MODE], board, board_build["defines"] + ) + includes = [f"-I{_q(d)}" for d in include_dirs] + + common = _CCFLAGS + defines + includes + project_compile_flags + cflags = _CFLAGS + common + cpp_standard = CORE.cpp_standard or "gnu++17" + cxxflags = ( + ["-fno-rtti", f"-std={cpp_standard}"] + + ["-fexceptions" if config.exceptions else "-fno-exceptions"] + + common + + get_project_cxx_compile_flags() + ) + asflags = _ASFLAGS + defines + includes + project_compile_flags + + # build_unflags applies to the framework flag sets too (compile and link), + # as under PlatformIO (a silently ignored ``build_unflags: -Os`` would + # diverge between the toolchains). + cflags = [f for f in cflags if f not in unflags] + cxxflags = [f for f in cxxflags if f not in unflags] + asflags = [f for f in asflags if f not in unflags] + + link_flags = [f for f in _LINKFLAGS if f not in unflags] + if esp8266_data[KEY_SCANF_FLOAT]: + link_flags += ["-u", "_scanf_float"] + link_flags += project_link_flags + link_flags += [flag for lib in libraries for flag in lib.link_flags] + flash_ld = f"testing_{flash_ld_name}" if CORE.testing_mode else flash_ld_name + link_flags += ["-T", flash_ld] + + lib_dirs = [Path("ld"), sdk / "lib", sdk / "ld", sdk / "lib" / config.nonosdk] + lib_dirs += project_lib_dirs + for lib in libraries: + lib_dirs += lib.link_dirs + system_libs = ( + _SYSTEM_LIBS_PRE_LWIP + + [config.lwip_lib] + + _SYSTEM_LIBS_POST_LWIP + + project_libs + + [lib_name for lib in libraries for lib_name in lib.link_libs] + + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"] + ) + + build_tool = Path(__file__).parent.parent / "arduino8266" / "build_tool.py" + ccache = ccache_path() + + # $in/$out stay unquoted in the rule commands: ninja shell-escapes its + # built-in path variables itself when expanding a command (POSIX and + # Windows), so adding quotes would wrap ninja's own quoting and break + # space-containing paths. Only literal paths need _q(). + lines = [ + "# Auto-generated by ESPHome", + "ninja_required_version = 1.5", + f"cc = {_q(toolchain_bin / 'xtensa-lx106-elf-gcc')}", + f"cxx = {_q(toolchain_bin / 'xtensa-lx106-elf-g++')}", + f"python = {_q(sys.executable)}", + f"buildtool = {_q(build_tool)}", + f"ccache = {_q(ccache) if ccache else ''}", + "", + "rule cc", + " command = $ccache $cc -MMD -MF $out.d $cflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CC $out", + "rule cxx", + " command = $ccache $cxx -MMD -MF $out.d $cxxflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CXX $out", + "rule asm", + " command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $asflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = AS $out", + "rule ar", + f" command = $python $buildtool ar {_q(toolchain_bin / 'xtensa-lx106-elf-ar')} $out $out.rsp", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = AR $out", + "rule link", + " command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = LINK $out", + "rule elf2bin", + f" command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {esp8266_data[KEY_FLASH_MODE]} --flash_freq 40 --flash_size {_flash_size_str(flash_ld_name)} --path {_q(toolchain_bin)} --out $out", + " description = BIN $out", + "rule copy", + " command = $python $buildtool copy $in $out", + " description = COPY $out", + "", + f"cflags = {' '.join(cflags)}", + f"cxxflags = {' '.join(cxxflags)}", + f"asflags = {' '.join(asflags)}", + f"linkflags = {' '.join(link_flags)}", + f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}", + f"libflags = {' '.join(f'-l{lib}' for lib in system_libs)}", + "", + ] + + core_exclude = set(_CORE_EXCLUDE_ALWAYS) + if "USE_ESP8266_WAVEFORM_STUBS" in flag_defines: + core_exclude |= _CORE_EXCLUDE_WAVEFORM + + archives = [] + variant_sources = _collect_sources(variant_dir) if variant_dir.is_dir() else [] + if variant_sources: + objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant") + lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}") + archives.append("libFrameworkArduinoVariant.a") + + core_objs = _ninja_compile_edges( + lines, _collect_sources(core_dir, core_exclude), core_dir, "core" + ) + if not core_objs: + # An empty archive would link into a wall of undefined references + # (app_entry, the exception vectors) far from the cause + raise EsphomeError( + f"Arduino toolchain install is incomplete: no core sources in " + f"{core_dir}; run 'esphome clean-all' and retry" + ) + lines.append(f"build libFrameworkArduino.a: ar {' '.join(core_objs)}") + archives.append("libFrameworkArduino.a") + + for lib in libraries: + if not lib.sources: + # Header-only libraries are legitimate; the log makes an empty + # srcFilter or broken tree traceable before link errors do. + _LOGGER.debug( + "Library %s has no source files; contributing includes only", + lib.name, + ) + continue + lib_root = _common_parent(lib.sources) + objs = _ninja_compile_edges( + lines, + lib.sources, + lib_root, + f"lib/{lib.name}", + flags=" ".join(_shell_token(f) for f in lib.flags), + ) + archive = f"lib{lib.name}.a" + lines.append(f"build {_e(archive)}: ar {' '.join(objs)}") + archives.append(archive) + + src_extra = f"-include {_q(src_dir / 'esphome' / 'components' / 'esp8266' / 'throw_stubs.h')}" + src_objs = _ninja_compile_edges( + lines, _collect_sources(src_dir), src_dir, "src", flags=src_extra + ) + + ld_deps = ["ld/local.eagle.app.v6.common.ld"] + if CORE.testing_mode: + ld_deps.append(f"ld/{flash_ld}") + lines.append( + f"build firmware.elf: link {' '.join(src_objs)} | " + f"{' '.join(_e(a) for a in archives)} {' '.join(_e(d) for d in ld_deps)}" + ) + lines.append(f" archives = {' '.join(archives)}") + lines.append("build firmware.bin: elf2bin firmware.elf") + lines.append("build firmware.factory.bin: copy firmware.bin") + lines.append("build firmware.ota.bin: copy firmware.bin") + lines.append("default firmware.factory.bin firmware.ota.bin") + lines.append("") + + return write_file_if_changed(build_dir / "build.ninja", "\n".join(lines)) + + def get_flash_ld_path(build_dir: Path) -> Path: """The flash linker script the link actually uses (for size reporting).""" from esphome.arduino8266.framework import ( diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index b214e45e49..a89bd145ea 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -11,6 +11,7 @@ PlatformIO toolchain produces for the same configuration. from __future__ import annotations from collections.abc import Generator +import logging from pathlib import Path from unittest.mock import MagicMock, patch @@ -63,6 +64,13 @@ def test_board_build_covers_every_board() -> None: assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) +def test_rule_map_covers_all_source_suffixes() -> None: + """Every suffix a library manifest can select must map to a ninja rule.""" + from esphome.platformio.library import SRC_FILE_EXTENSIONS + + assert set(arduino8266._RULE_FOR_SUFFIX) == set(SRC_FILE_EXTENSIONS) + + def test_build_config_defaults() -> None: _set_flags() @@ -185,6 +193,120 @@ def _make_framework(tmp_path: Path) -> dict[str, Path]: } +def _write_ninja( + paths: dict[str, Path], + libraries: list | None = None, + ccache: str | None = None, +) -> str: + src = CORE.relative_src_path() + (src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True) + (src / "main.cpp").write_text("") + (src / "esphome" / "vendor.c").write_text("") + + with ( + patch.object(arduino8266, "generate_ld_scripts"), + patch( + "esphome.arduino8266.component.resolve_libraries", + return_value=libraries or [], + ), + patch("esphome.arduino8266.framework.ccache_path", return_value=ccache), + ): + arduino8266.write_project(paths) + return (CORE.relative_pioenvs_path(CORE.name) / "build.ninja").read_text() + + +def test_write_project_link_line_and_exclusions(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + _set_flags( + "-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH", + "-DUSE_ESP8266_WAVEFORM_STUBS", + "-Wl,--wrap=millis", + "-Wl,--wrap=printf", + "-Wno-nonnull-compare", + "-L/opt/blobs", + "-luser_blob", + "-L /spc/blobs -l spaced_blob", + ) + content = _write_ninja(paths) + + # Base link flags from the PlatformIO builder + for flag in ( + "-Wl,--no-check-sections", + "-Wl,-static", + "-Wl,--gc-sections", + "-Wl,-wrap,system_restart_local", + "-Wl,-wrap,spi_flash_read", + "-u app_entry", + "-u _printf_float", + "-u _DebugExceptionVector", + "-u _DoubleExceptionVector", + "-u _KernelExceptionVector", + "-u _NMIExceptionVector", + "-u _UserExceptionVector", + ): + assert flag in content + # ESPHome's link flags and the board linker script + assert "-Wl,--wrap=millis" in content + assert "-Wl,--wrap=printf" in content + assert "-T eagle.flash.4m.ld" in content + # scanf float disabled: the forced-link flag must not appear + assert "_scanf_float" not in content + # $in/$out must stay UNQUOTED: ninja shell-escapes its built-in path + # variables itself, so added quotes would wrap ninja's quoting and break + # space-containing paths. + assert "-c $in -o $out" in content + assert "--app $in --flash_mode" in content + assert '"$in"' not in content + assert '"$out"' not in content + # -L/-l from esphome build_flags reach the link line, not the compiles; + # spaced forms ("-L /path") are shell-lexed the way PlatformIO does. + # str(Path(...)) so the separator matches the host platform. + opt_blobs = str(Path("/opt/blobs")) + spc_blobs = str(Path("/spc/blobs")) + assert f'-L"{opt_blobs}"' in content + assert "-luser_blob" in content + assert f'-L"{spc_blobs}"' in content + assert "-lspaced_blob" in content + for line in content.splitlines(): + if line.split(" = ")[0] in ("cflags", "cxxflags", "asflags"): + assert "user_blob" not in line + assert opt_blobs not in line + assert "spaced_blob" not in line + assert spc_blobs not in line + # System libraries with the selected lwIP variant, in the builder's order + assert ( + "-lhal -lphy -lpp -lnet80211 -llwip2-1460 -lwpa -lcrypto -lmain -lwps " + "-lbearssl -lespnow -lsmartconfig -lairkiss -lwpa2 -lspaced_blob " + "-luser_blob " + "-lstdc++ -lm -lc -lgcc" in content + ) + # Core exclusions: native OTA backend and waveform stubs + assert "Updater.cpp" not in content + assert "core_esp8266_waveform_pwm.cpp" not in content + assert "core_esp8266_waveform_phase.cpp" not in content + assert "core_esp8266_main.cpp.o" in content + # Assembly and C sources compile through their own rules + assert "cont.S.o: asm" in content + assert "abi.c.o: cc" in content + # throw_stubs is force-included for ESPHome sources only + src_lines = [line for line in content.splitlines() if "obj/src/" in line] + assert any("main.cpp.o: cxx" in line for line in src_lines) + assert content.count("throw_stubs.h") == len( + [line for line in content.splitlines() if line.startswith(" flags = ")] + ) + + +def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = True + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + content = _write_ninja(paths) + assert "-u _scanf_float" in content + # Waveform not stubbed out: both implementations stay in the archive + assert "core_esp8266_waveform_pwm.cpp.o" in content + assert "core_esp8266_waveform_phase.cpp.o" in content + + @pytest.mark.parametrize( ("knob", "lib", "mss", "features", "ipv6"), [ @@ -333,6 +455,54 @@ def test_generate_ld_scripts_testing_mode(tmp_path: Path) -> None: assert "len = 0x2000000" in patched +def test_write_project_libraries_and_variant( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + from esphome.arduino8266.component import ArduinoLibrary + + paths = _make_framework(tmp_path) + variant_src = paths["framework_path"] / "variants" / "nodemcu" / "variant.cpp" + variant_src.write_text("") + + lib_dir = tmp_path / "libsrc" + lib_dir.mkdir() + (lib_dir / "lib.cpp").write_text("") + (lib_dir / "impl.cc").write_text("") + headers_only = ArduinoLibrary(name="HeadersOnly", include_dirs=[lib_dir]) + library = ArduinoLibrary( + name="MyLib", + sources=[lib_dir / "impl.cc", lib_dir / "lib.cpp"], + include_dirs=[lib_dir], + flags=["-DMYLIB=1"], + link_dirs=[lib_dir / "blobs"], + link_libs=["algobsec"], + link_flags=["-Wl,--wrap=malloc"], + ) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS") + + with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.arduino8266"): + content = _write_ninja( + paths, libraries=[library, headers_only], ccache="/cc/ccache" + ) + + assert "build libFrameworkArduinoVariant.a: ar" in content + assert "build libMyLib.a: ar" in content + # A headers-only library contributes includes but no archive, with a + # debug log distinguishing it from a resolution failure + assert "libHeadersOnly.a" not in content + assert "Library HeadersOnly has no source files" in caplog.text + assert " flags = -DMYLIB=1" in content + assert "-lalgobsec" in content + # Library link flags reach the firmware link line; .cc compiles as C++ + assert "-Wl,--wrap=malloc" in content + assert "impl.cc.o: cxx" in content + assert f'-L"{lib_dir / "blobs"}"' in content + # Exceptions knob: -fexceptions and the exception-enabled stdc++ + assert "-fexceptions" in content + assert "-lstdc++-exc" in content + assert 'ccache = "/cc/ccache"' in content + + def test_get_flash_ld_path(tmp_path: Path) -> None: CORE.testing_mode = True @@ -364,6 +534,26 @@ def test_flash_size_str() -> None: _flash_size_str("bogus.ld") +def test_write_project_testing_mode(tmp_path: Path) -> None: + paths = _make_framework(tmp_path) + CORE.testing_mode = True + _set_flags() + content = _write_ninja(paths) + assert "-T testing_eagle.flash.4m.ld" in content + assert "ld/testing_eagle.flash.4m.ld" in content + + +def test_write_project_missing_framework_dir_raises(tmp_path: Path) -> None: + """An incomplete framework install fails naming the missing path.""" + import shutil + + paths = _make_framework(tmp_path) + shutil.rmtree(paths["framework_path"] / "tools" / "sdk" / "lwip2") + _set_flags() + with pytest.raises(EsphomeError, match="incomplete.*lwip2"): + _write_ninja(paths) + + def test_build_config_nonosdk_precedence() -> None: """With two SDK knobs set (a pathological config), ties break deterministically by table order.""" @@ -374,6 +564,20 @@ def test_build_config_nonosdk_precedence() -> None: assert _resolve_build_config(_flag_defines()).nonosdk == "NONOSDK221" +def test_write_project_build_unflags_apply_to_framework_flags(tmp_path: Path) -> None: + """build_unflags removes flags from the framework sets, as PlatformIO does.""" + paths = _make_framework(tmp_path) + _set_flags() + CORE.build_unflags = {"-fipa-pta", "-Wl,--gc-sections"} + content = _write_ninja(paths) + for line in content.splitlines(): + key = line.split(" = ")[0] + if key in ("cflags", "cxxflags", "asflags"): + assert "-fipa-pta" not in line + if key == "linkflags": + assert "-Wl,--gc-sections" not in line + + def test_project_flags_trailing_bare_linker_flag_warns( caplog: pytest.LogCaptureFixture, ) -> None: @@ -417,9 +621,7 @@ def test_project_flags_unflags_match_tokens() -> None: """build_unflags removes a token embedded in a multi-token entry.""" _set_flags("-Os -g3") CORE.build_unflags = {"-Os"} - compile_flags, _link, _dirs, _libs = arduino8266._project_flags( - arduino8266._unflag_tokens() - ) + compile_flags, _link, _dirs, _libs = arduino8266._project_flags() assert "-g3" in compile_flags assert "-Os" not in compile_flags @@ -446,6 +648,25 @@ def test_shell_token_escaping() -> None: assert arduino8266._shell_token("a b\\") == '"a b\\\\"' +def test_write_project_empty_core_raises(tmp_path: Path) -> None: + """A framework tree with no core sources fails at generation, not link.""" + paths = _make_framework(tmp_path) + core = paths["framework_path"] / "cores" / "esp8266" + for f in core.iterdir(): + f.unlink() + _set_flags() + src = CORE.relative_src_path() + (src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True) + (src / "main.cpp").write_text("") + with ( + patch.object(arduino8266, "generate_ld_scripts"), + patch("esphome.arduino8266.component.resolve_libraries", return_value=[]), + patch("esphome.arduino8266.framework.ccache_path", return_value=None), + pytest.raises(EsphomeError, match="no core sources"), + ): + arduino8266.write_project(paths) + + def test_flag_defines_joins_spaced_define() -> None: """A spaced "-D KNOB" entry is detected exactly as PlatformIO detects it.""" _set_flags("-D PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH")