Merge branch 'esp8266-native-toolchain-plumbing' into esp8266-native-framework-installer

esphome/components/esp8266/boards.py

@@ -362,9 +362,10 @@ BOARDS = {
 }
 
 """
-ESP8266_BOARD_BUILD generate with:
+ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the
+native toolchain mirrors; regenerate against the tag when bumping it):
 
-git clone https://github.com/platformio/platform-espressif8266
+git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266
 python3 - <<'EOF'
 import json, glob, os
 for f in sorted(glob.glob("platform-espressif8266/boards/*.json")):

esphome/components/esp8266/build_surgery.py

@@ -58,21 +58,33 @@ _TESTING_SEGMENT_SIZES = {
 
 
 def _segment_line_re(segment_name: str) -> re.Pattern[str]:
-    """The MEMORY line for one segment: ``<seg> : org = 0x..., len = 0x...``."""
+    """The MEMORY line for one segment: ``<seg> : org = 0x..., len = 0x...``.
+
+    Anchored to the start of the line so a name never matches inside a
+    longer one (``ram0_0_seg`` must not read ``dram0_0_seg``). The size
+    group stops at the hex digits, leaving any ``ul`` suffix (from the
+    preprocessed ``MMU_IRAM_SIZE``) in place.
+    """
     return re.compile(
-        rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
-        r"(0x[0-9a-fA-F]+)"
+        rf"(^[ \t]*{re.escape(segment_name)}"
+        r"\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
+        r"(0x[0-9a-fA-F]+)",
+        re.MULTILINE,
     )
 
 
 def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str:
     """Enlarge the named memory segments so grouped CI test builds can link.
 
-    Each caller passes the segments its linker script defines; a segment
-    that fails to match raises, since a silently kept real memory limit
-    would fail grouped builds far from the cause.
+    Each caller passes the segments its linker script defines: the
+    generated common ld carries ``iram1_0_seg``; the flash ld carries
+    ``dram0_0_seg`` and ``irom0_0_seg``. A segment that fails to match
+    raises, since a silently kept real memory limit would fail grouped
+    builds far from the cause.
     """
     for segment in segments:
+        if segment not in _TESTING_SEGMENT_SIZES:
+            raise RuntimeError(f"Unknown testing-mode segment {segment!r}")
         content, count = _segment_line_re(segment).subn(
             rf"\g<1>{_TESTING_SEGMENT_SIZES[segment]}", content
         )

esphome/components/libretiny/__init__.py

@@ -314,8 +314,8 @@ BASE_SCHEMA = cv.Schema(
 )
 
 BASE_SCHEMA.add_extra(_detect_variant)
-BASE_SCHEMA.add_extra(_update_core_data)
 BASE_SCHEMA.add_extra(cv.require_platformio_toolchain("LibreTiny"))
+BASE_SCHEMA.add_extra(_update_core_data)
 
 
 def _configure_lwip(config: dict) -> None:

esphome/config_validation.py

@@ -107,6 +107,9 @@ from esphome.util import parse_esphome_version  # noqa: F401
 from esphome.voluptuous_schema import _Schema
 from esphome.yaml_util import SensitiveStr, make_data_base
 
+if typing.TYPE_CHECKING:
+    from esphome.types import ConfigType
+
 _LOGGER = logging.getLogger(__name__)
 
 # pylint: disable=invalid-name
@@ -2533,17 +2536,21 @@ def platformio_version_constraint(value):
     return constraints
 
 
-def check_supported_toolchain(platform_name: str, supported: tuple) -> None:
+def check_supported_toolchain(
+    platform_name: str, supported: tuple[Toolchain, ...]
+) -> None:
     """Raise when the resolved ``CORE.toolchain`` is not in ``supported``.
 
     One message shape for every platform, so a ``--toolchain`` a platform
     cannot serve always fails by name instead of silently building with a
     different backend.
     """
-    if CORE.toolchain not in supported:
+    toolchain = CORE.toolchain
+    if toolchain is None or toolchain not in supported:
         names = ", ".join(f"'{tc.value}'" for tc in supported)
         raise Invalid(
-            f"Unsupported toolchain '{CORE.toolchain.value}' for "
+            f"Unsupported toolchain "
+            f"'{toolchain.value if toolchain else 'unresolved'}' for "
             f"{platform_name}. Supported: {names}."
         )
 
@@ -2555,7 +2562,7 @@ def require_platformio_toolchain(platform_name: str):
     ``--toolchain`` they cannot serve would silently build with PlatformIO.
     """
 
-    def validator(config):
+    def validator(config: ConfigType) -> ConfigType:
         if CORE.toolchain is None:
             CORE.toolchain = Toolchain.PLATFORMIO
         check_supported_toolchain(platform_name, (Toolchain.PLATFORMIO,))

esphome/core/__init__.py

@@ -984,6 +984,12 @@ class EsphomeCore:
 
     @property
     def using_toolchain_arduino(self):
+        """The native (PlatformIO-free) ESP8266 Arduino build backend.
+
+        Unlike ``using_arduino`` (the target *framework*, true for any
+        platform compiling Arduino code), this is a build *toolchain*
+        choice, like its ``using_toolchain_*`` siblings.
+        """
         return self.toolchain == Toolchain.ARDUINO
 
     @property
@@ -1099,6 +1105,8 @@ class EsphomeCore:
         return build_flag
 
     def add_build_unflag(self, build_unflag: str) -> None:
+        # No warning for using_toolchain_arduino: the native ESP8266 build
+        # honors build_unflags (token-level, matching PlatformIO).
         if self.using_toolchain_esp_idf:
             # The native ESP-IDF build generator does not consume build_unflags
             _LOGGER.warning(

esphome/core/config.py

@@ -576,14 +576,15 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No
                 for flag in vals:
                     cg.add_build_flag(flag)
             elif key == "lib_deps":
-                # Routed through the regular library mechanism so the libraries
-                # are converted to IDF components like any other PIO library
+                # Routed through the regular library mechanism so the
+                # libraries reach the native backend's converter (IDF
+                # components, or the ESP8266 native library resolution)
                 for lib in vals:
                     _add_library_str(lib)
             elif key == "lib_ignore":
-                # Read by the PIO-library-to-IDF-component conversion
-                # (generate_idf_components); filters both top-level libraries
-                # and dependencies discovered during conversion
+                # Read by the shared library conversion (lib_ignore_set in
+                # platformio/library.py) on both native backends; filters
+                # top-level libraries and discovered dependencies
                 cg.add_platformio_option(key, vals)
             elif key != "upload_speed":
                 # upload_speed needs no handling: it is read from the raw

tests/unit_tests/components/esp8266/test_build_surgery.py

@@ -24,16 +24,26 @@ _COMMON_LD_SNIPPET = """\
   } >dram0_0_seg :dram0_0_phdr
 """
 
+# Shaped like the real SDK flash ld scripts: no iram1_0_seg (that lives in
+# the generated common ld only)
 _FLASH_LD_SNIPPET = """\
 MEMORY
 {
   dport0_0_seg :                        org = 0x3FF00000, len = 0x10
   dram0_0_seg :                         org = 0x3FFE8000, len = 0x14000
-  iram1_0_seg :                         org = 0x40100000, len = 0x8000
   irom0_0_seg :                         org = 0x40201010, len = 0xfeff0
 }
 """
 
+# Shaped like the preprocessed common ld: MMU_IRAM_SIZE expands with a ul
+# suffix the patcher must leave in place
+_COMMON_LD_MEMORY_SNIPPET = """\
+MEMORY
+{
+  iram1_0_seg :                         org = 0x40100000, len = 0x8000ul
+}
+"""
+
 
 def test_relocate_ratetable_inserts_after_data_start() -> None:
     patched = relocate_ratetable(_COMMON_LD_SNIPPET)
@@ -52,15 +62,32 @@ def test_relocate_ratetable_requires_anchor() -> None:
 
 def test_testing_memory_patches_enlarge_segments() -> None:
     patched = apply_testing_memory_patches(
-        _FLASH_LD_SNIPPET, ("iram1_0_seg", "dram0_0_seg", "irom0_0_seg")
+        _FLASH_LD_SNIPPET, ("dram0_0_seg", "irom0_0_seg")
     )
-    assert segment_length(patched, "iram1_0_seg") == 0x200000
     assert segment_length(patched, "dram0_0_seg") == 0x200000
     assert segment_length(patched, "irom0_0_seg") == 0x2000000
     # Untouched segments keep their sizes
     assert segment_length(patched, "dport0_0_seg") == 0x10
 
 
+def test_testing_memory_patches_keep_ul_suffix() -> None:
+    """The common ld's preprocessed sizes carry a ul suffix; the patch must
+    replace only the hex digits, as testing_mode.py.script does."""
+    patched = apply_testing_memory_patches(_COMMON_LD_MEMORY_SNIPPET, ("iram1_0_seg",))
+    assert "len = 0x200000ul" in patched
+    assert segment_length(patched, "iram1_0_seg") == 0x200000
+
+
+def test_segment_length_requires_whole_name() -> None:
+    """A name must match its own line, never inside a longer segment name."""
+    assert segment_length(_FLASH_LD_SNIPPET, "ram0_0_seg") is None
+
+
+def test_testing_memory_patches_unknown_segment_raises() -> None:
+    with pytest.raises(RuntimeError, match="Unknown testing-mode segment"):
+        apply_testing_memory_patches("MEMORY { }", ("bogus_seg",))
+
+
 def test_segment_length() -> None:
     assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0
     assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None

tests/unit_tests/test_config_validation.py

@@ -3181,3 +3181,28 @@ def test_require_platformio_toolchain() -> None:
     CORE.toolchain = Toolchain.ARDUINO
     with pytest.raises(Invalid, match="Unsupported toolchain 'arduino' for RP2"):
         validator(config)
+
+
+@pytest.mark.parametrize(
+    ("platform", "minimal_config"),
+    [
+        ("host", {}),
+        ("rp2", {"board": "rpipicow"}),
+        ("bk72xx", {"board": "generic-bk7231n-qfn32-tuya"}),
+    ],
+)
+def test_every_platformio_only_platform_rejects_arduino_toolchain(
+    platform: str, minimal_config: dict
+) -> None:
+    """The invariant every native-toolchain gate relies on: a platform that
+    cannot serve a CLI toolchain rejects it at validation (esp32, esp8266,
+    and nrf52 pin this in their own suites)."""
+    import importlib
+
+    from esphome.const import Toolchain
+    from esphome.core import CORE
+
+    module = importlib.import_module(f"esphome.components.{platform}")
+    CORE.toolchain = Toolchain.ARDUINO
+    with pytest.raises(Invalid, match="Unsupported toolchain 'arduino'"):
+        module.CONFIG_SCHEMA(dict(minimal_config))