Merge branch 'esp8266-native-framework-installer' into esp8266-native-library-backend

esphome/arduino8266/framework.py

@@ -24,6 +24,7 @@ import os
 from pathlib import Path
 import platform
 import shutil
+import time
 
 from esphome.core import EsphomeError, Version
 from esphome.framework_helpers import (
@@ -103,9 +104,15 @@ def _pio_system() -> str:
     sysname = platform.system().lower()
     machine = platform.machine().lower()
     if sysname == "darwin":
-        return "darwin_arm64" if machine == "arm64" else "darwin_x86_64"
+        if machine == "arm64":
+            return "darwin_arm64"
+        if machine == "x86_64":
+            return "darwin_x86_64"
     if sysname == "windows":
-        return "windows_amd64" if machine in ("amd64", "arm64") else "windows_x86"
+        if machine in ("amd64", "arm64"):
+            return "windows_amd64"
+        if machine in ("x86", "i686", "i386"):
+            return "windows_x86"
     if sysname == "linux":
         if machine in ("arm64", "aarch64"):
             return "linux_aarch64"
@@ -129,7 +136,7 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None
 
     url = _REGISTRY_URL.format(package=package)
     last_err: Exception | None = None
-    for _ in range(3):
+    for attempt in range(3):
         try:
             resp = requests.get(url, timeout=30)
             resp.raise_for_status()
@@ -137,6 +144,8 @@ def _registry_download(package: str, version: str) -> tuple[str, str, int | None
             break
         except requests.RequestException as err:
             last_err = err
+            # Back off so the retries are not one burst against a hiccup
+            time.sleep(2**attempt)
     else:
         # A clean, retried error like the other download paths in the tree
         raise EsphomeError(
@@ -186,7 +195,10 @@ def _install_package(
     # process cannot wipe the directory another is extracting into (same
     # filelock pattern as platformio/toolchain.py and git.py).
     dest.parent.mkdir(parents=True, exist_ok=True)
-    with FileLock(f"{dest}.lock"):
+    # fallback_to_soft would silently degrade to an existence lock on a
+    # flock-less filesystem; a hard-killed run would then hang every later
+    # build forever (same hazard git.py documents).
+    with FileLock(f"{dest}.lock", fallback_to_soft=False):
         if marker.is_file():
             # Another process finished the install while we waited
             return
@@ -248,6 +260,15 @@ def _find_ninja() -> Path:
 
 def check_and_install(framework_version: Version) -> dict[str, Path]:
     """Ensure framework, toolchain, and ninja are installed; return their paths."""
+    if framework_version < MIN_FRAMEWORK_VERSION:
+        # Config validation enforces this too; keep the module honest when
+        # called directly.
+        raise EsphomeError(
+            f"The native toolchain requires the Arduino core "
+            f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}"
+        )
+    # Probe the cheap local dependency before ~110 MB of downloads
+    ninja_path = _find_ninja()
     package_version = framework_package_version(framework_version)
     framework_path = get_framework_path(package_version)
     _install_package(
@@ -268,7 +289,7 @@ def check_and_install(framework_version: Version) -> dict[str, Path]:
     return {
         "framework_path": framework_path,
         "toolchain_path": toolchain_path,
-        "ninja_path": _find_ninja(),
+        "ninja_path": ninja_path,
     }
 
 

esphome/components/esp8266/boards.py

@@ -362,9 +362,10 @@ BOARDS = {
 }
 
 """
-ESP8266_BOARD_BUILD generate with:
+ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the
+native toolchain mirrors; regenerate against the tag when bumping it):
 
-git clone https://github.com/platformio/platform-espressif8266
+git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266
 python3 - <<'EOF'
 import json, glob, os
 for f in sorted(glob.glob("platform-espressif8266/boards/*.json")):

esphome/components/esp8266/build_surgery.py

@@ -58,21 +58,33 @@ _TESTING_SEGMENT_SIZES = {
 
 
 def _segment_line_re(segment_name: str) -> re.Pattern[str]:
-    """The MEMORY line for one segment: ``<seg> : org = 0x..., len = 0x...``."""
+    """The MEMORY line for one segment: ``<seg> : org = 0x..., len = 0x...``.
+
+    Anchored to the start of the line so a name never matches inside a
+    longer one (``ram0_0_seg`` must not read ``dram0_0_seg``). The size
+    group stops at the hex digits, leaving any ``ul`` suffix (from the
+    preprocessed ``MMU_IRAM_SIZE``) in place.
+    """
     return re.compile(
-        rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
-        r"(0x[0-9a-fA-F]+)"
+        rf"(^[ \t]*{re.escape(segment_name)}"
+        r"\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
+        r"(0x[0-9a-fA-F]+)",
+        re.MULTILINE,
     )
 
 
 def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str:
     """Enlarge the named memory segments so grouped CI test builds can link.
 
-    Each caller passes the segments its linker script defines; a segment
-    that fails to match raises, since a silently kept real memory limit
-    would fail grouped builds far from the cause.
+    Each caller passes the segments its linker script defines: the
+    generated common ld carries ``iram1_0_seg``; the flash ld carries
+    ``dram0_0_seg`` and ``irom0_0_seg``. A segment that fails to match
+    raises, since a silently kept real memory limit would fail grouped
+    builds far from the cause.
     """
     for segment in segments:
+        if segment not in _TESTING_SEGMENT_SIZES:
+            raise RuntimeError(f"Unknown testing-mode segment {segment!r}")
         content, count = _segment_line_re(segment).subn(
             rf"\g<1>{_TESTING_SEGMENT_SIZES[segment]}", content
         )

esphome/components/libretiny/__init__.py

@@ -314,8 +314,8 @@ BASE_SCHEMA = cv.Schema(
 )
 
 BASE_SCHEMA.add_extra(_detect_variant)
-BASE_SCHEMA.add_extra(_update_core_data)
 BASE_SCHEMA.add_extra(cv.require_platformio_toolchain("LibreTiny"))
+BASE_SCHEMA.add_extra(_update_core_data)
 
 
 def _configure_lwip(config: dict) -> None:

esphome/config_validation.py

@@ -107,6 +107,9 @@ from esphome.util import parse_esphome_version  # noqa: F401
 from esphome.voluptuous_schema import _Schema
 from esphome.yaml_util import SensitiveStr, make_data_base
 
+if typing.TYPE_CHECKING:
+    from esphome.types import ConfigType
+
 _LOGGER = logging.getLogger(__name__)
 
 # pylint: disable=invalid-name
@@ -2533,17 +2536,21 @@ def platformio_version_constraint(value):
     return constraints
 
 
-def check_supported_toolchain(platform_name: str, supported: tuple) -> None:
+def check_supported_toolchain(
+    platform_name: str, supported: tuple[Toolchain, ...]
+) -> None:
     """Raise when the resolved ``CORE.toolchain`` is not in ``supported``.
 
     One message shape for every platform, so a ``--toolchain`` a platform
     cannot serve always fails by name instead of silently building with a
     different backend.
     """
-    if CORE.toolchain not in supported:
+    toolchain = CORE.toolchain
+    if toolchain is None or toolchain not in supported:
         names = ", ".join(f"'{tc.value}'" for tc in supported)
         raise Invalid(
-            f"Unsupported toolchain '{CORE.toolchain.value}' for "
+            f"Unsupported toolchain "
+            f"'{toolchain.value if toolchain else 'unresolved'}' for "
             f"{platform_name}. Supported: {names}."
         )
 
@@ -2555,7 +2562,7 @@ def require_platformio_toolchain(platform_name: str):
     ``--toolchain`` they cannot serve would silently build with PlatformIO.
     """
 
-    def validator(config):
+    def validator(config: ConfigType) -> ConfigType:
         if CORE.toolchain is None:
             CORE.toolchain = Toolchain.PLATFORMIO
         check_supported_toolchain(platform_name, (Toolchain.PLATFORMIO,))

esphome/core/__init__.py

@@ -984,6 +984,12 @@ class EsphomeCore:
 
     @property
     def using_toolchain_arduino(self):
+        """The native (PlatformIO-free) ESP8266 Arduino build backend.
+
+        Unlike ``using_arduino`` (the target *framework*, true for any
+        platform compiling Arduino code), this is a build *toolchain*
+        choice, like its ``using_toolchain_*`` siblings.
+        """
         return self.toolchain == Toolchain.ARDUINO
 
     @property
@@ -1099,6 +1105,8 @@ class EsphomeCore:
         return build_flag
 
     def add_build_unflag(self, build_unflag: str) -> None:
+        # No warning for using_toolchain_arduino: the native ESP8266 build
+        # honors build_unflags (token-level, matching PlatformIO).
         if self.using_toolchain_esp_idf:
             # The native ESP-IDF build generator does not consume build_unflags
             _LOGGER.warning(

esphome/core/config.py

@@ -576,14 +576,15 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No
                 for flag in vals:
                     cg.add_build_flag(flag)
             elif key == "lib_deps":
-                # Routed through the regular library mechanism so the libraries
-                # are converted to IDF components like any other PIO library
+                # Routed through the regular library mechanism so the
+                # libraries reach the native backend's converter (IDF
+                # components, or the ESP8266 native library resolution)
                 for lib in vals:
                     _add_library_str(lib)
             elif key == "lib_ignore":
-                # Read by the PIO-library-to-IDF-component conversion
-                # (generate_idf_components); filters both top-level libraries
-                # and dependencies discovered during conversion
+                # Read by the shared library conversion (lib_ignore_set in
+                # platformio/library.py) on both native backends; filters
+                # top-level libraries and discovered dependencies
                 cg.add_platformio_option(key, vals)
             elif key != "upload_speed":
                 # upload_speed needs no handling: it is read from the raw

tests/unit_tests/components/esp8266/test_build_surgery.py

@@ -24,16 +24,26 @@ _COMMON_LD_SNIPPET = """\
   } >dram0_0_seg :dram0_0_phdr
 """
 
+# Shaped like the real SDK flash ld scripts: no iram1_0_seg (that lives in
+# the generated common ld only)
 _FLASH_LD_SNIPPET = """\
 MEMORY
 {
   dport0_0_seg :                        org = 0x3FF00000, len = 0x10
   dram0_0_seg :                         org = 0x3FFE8000, len = 0x14000
-  iram1_0_seg :                         org = 0x40100000, len = 0x8000
   irom0_0_seg :                         org = 0x40201010, len = 0xfeff0
 }
 """
 
+# Shaped like the preprocessed common ld: MMU_IRAM_SIZE expands with a ul
+# suffix the patcher must leave in place
+_COMMON_LD_MEMORY_SNIPPET = """\
+MEMORY
+{
+  iram1_0_seg :                         org = 0x40100000, len = 0x8000ul
+}
+"""
+
 
 def test_relocate_ratetable_inserts_after_data_start() -> None:
     patched = relocate_ratetable(_COMMON_LD_SNIPPET)
@@ -52,15 +62,32 @@ def test_relocate_ratetable_requires_anchor() -> None:
 
 def test_testing_memory_patches_enlarge_segments() -> None:
     patched = apply_testing_memory_patches(
-        _FLASH_LD_SNIPPET, ("iram1_0_seg", "dram0_0_seg", "irom0_0_seg")
+        _FLASH_LD_SNIPPET, ("dram0_0_seg", "irom0_0_seg")
     )
-    assert segment_length(patched, "iram1_0_seg") == 0x200000
     assert segment_length(patched, "dram0_0_seg") == 0x200000
     assert segment_length(patched, "irom0_0_seg") == 0x2000000
     # Untouched segments keep their sizes
     assert segment_length(patched, "dport0_0_seg") == 0x10
 
 
+def test_testing_memory_patches_keep_ul_suffix() -> None:
+    """The common ld's preprocessed sizes carry a ul suffix; the patch must
+    replace only the hex digits, as testing_mode.py.script does."""
+    patched = apply_testing_memory_patches(_COMMON_LD_MEMORY_SNIPPET, ("iram1_0_seg",))
+    assert "len = 0x200000ul" in patched
+    assert segment_length(patched, "iram1_0_seg") == 0x200000
+
+
+def test_segment_length_requires_whole_name() -> None:
+    """A name must match its own line, never inside a longer segment name."""
+    assert segment_length(_FLASH_LD_SNIPPET, "ram0_0_seg") is None
+
+
+def test_testing_memory_patches_unknown_segment_raises() -> None:
+    with pytest.raises(RuntimeError, match="Unknown testing-mode segment"):
+        apply_testing_memory_patches("MEMORY { }", ("bogus_seg",))
+
+
 def test_segment_length() -> None:
     assert segment_length(_FLASH_LD_SNIPPET, "irom0_0_seg") == 0xFEFF0
     assert segment_length(_FLASH_LD_SNIPPET, "missing_seg") is None

tests/unit_tests/test_arduino8266_framework.py

@@ -64,6 +64,9 @@ def test_pio_system(system: str, machine: str, expected: str) -> None:
     [
         ("FreeBSD", "amd64"),
         ("Linux", "ppc64le"),
+        ("Darwin", "ppc"),
+        ("Darwin", ""),
+        ("Windows", "ia64"),
     ],
 )
 def test_pio_system_unsupported_host_raises(system: str, machine: str) -> None:
@@ -88,10 +91,13 @@ def test_registry_download_network_error_is_clean_and_retried() -> None:
 
     with (
         patch("requests.get", side_effect=requests.ConnectionError("boom")) as mock_get,
+        patch.object(framework.time, "sleep") as mock_sleep,
         pytest.raises(EsphomeError, match="Could not query the package registry"),
     ):
         framework._registry_download("pkg", "1.0.0")
     assert mock_get.call_count == 3
+    # Backed-off retries, not one burst
+    assert mock_sleep.call_count == 3
 
 
 def test_registry_download_retries_transient_error() -> None:
@@ -109,6 +115,7 @@ def test_registry_download_retries_transient_error() -> None:
     )
     with (
         patch("requests.get", side_effect=[requests.ConnectionError("boom"), resp]),
+        patch.object(framework.time, "sleep"),
         patch.object(framework, "_pio_system", return_value="linux_x86_64"),
     ):
         assert framework._registry_download("pkg", "1.0.0") == (
@@ -439,3 +446,23 @@ def test_ccache_env_requires_build_path() -> None:
         pytest.raises(ValueError, match="build_path"),
     ):
         framework.ccache_env()
+
+
+def test_check_and_install_rejects_old_core(tmp_path: Path) -> None:
+    """Calling the installer below the floor fails before any download."""
+    with pytest.raises(EsphomeError, match=">= 3.1.1"):
+        framework.check_and_install(cv.Version(3, 0, 2))
+
+
+def test_install_package_uses_hard_lock(tmp_path: Path) -> None:
+    """The install lock must never degrade to a soft (existence) lock."""
+    dest = tmp_path / "pkg"
+    with (
+        patch("filelock.FileLock") as mock_lock,
+        patch.object(framework, "download_from_mirrors"),
+        patch.object(framework, "archive_extract_all") as mock_extract,
+        patch.object(framework, "_pio_system", return_value="linux_x86_64"),
+    ):
+        mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir(exist_ok=True)
+        framework._install_package("pkg", "1.0.0", dest, ["http://m"])
+    assert mock_lock.call_args.kwargs["fallback_to_soft"] is False

tests/unit_tests/test_config_validation.py

@@ -3181,3 +3181,28 @@ def test_require_platformio_toolchain() -> None:
     CORE.toolchain = Toolchain.ARDUINO
     with pytest.raises(Invalid, match="Unsupported toolchain 'arduino' for RP2"):
         validator(config)
+
+
+@pytest.mark.parametrize(
+    ("platform", "minimal_config"),
+    [
+        ("host", {}),
+        ("rp2", {"board": "rpipicow"}),
+        ("bk72xx", {"board": "generic-bk7231n-qfn32-tuya"}),
+    ],
+)
+def test_every_platformio_only_platform_rejects_arduino_toolchain(
+    platform: str, minimal_config: dict
+) -> None:
+    """The invariant every native-toolchain gate relies on: a platform that
+    cannot serve a CLI toolchain rejects it at validation (esp32, esp8266,
+    and nrf52 pin this in their own suites)."""
+    import importlib
+
+    from esphome.const import Toolchain
+    from esphome.core import CORE
+
+    module = importlib.import_module(f"esphome.components.{platform}")
+    CORE.toolchain = Toolchain.ARDUINO
+    with pytest.raises(Invalid, match="Unsupported toolchain 'arduino'"):
+        module.CONFIG_SCHEMA(dict(minimal_config))