Address review: wire analyze-memory and idedata for the native toolchain, warn on ignored platformio_options, verify downloads, surface silent failures

esphome/__main__.py

@@ -1935,6 +1935,21 @@ def command_idedata(args: ArgsProtocol, config: ConfigType) -> int:
         print(json.dumps(idedata, indent=2) + "\n")
         return 0
 
+    if CORE.is_esp8266 and CORE.using_toolchain_arduino:
+        # Same contract as the ESP-IDF branch: idedata is derived from the
+        # build's compile_commands.json, so a compile must have run.
+        from esphome.arduino8266 import toolchain as arduino8266_toolchain
+
+        idedata = arduino8266_toolchain.get_idedata()
+        if idedata is None:
+            _LOGGER.error(
+                "No idedata available; compile the configuration first",
+            )
+            return 1
+
+        print(json.dumps(idedata, indent=2) + "\n")
+        return 0
+
     if not CORE.using_toolchain_platformio:
         _LOGGER.error(
             "The idedata command is not compatible with %s toolchain",
@@ -1981,6 +1996,13 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int:
         objdump_path = str(toolchain.get_objdump_path())
         readelf_path = str(toolchain.get_readelf_path())
 
+        firmware_elf = toolchain.get_elf_path()
+    elif CORE.is_esp8266 and CORE.using_toolchain_arduino:
+        from esphome.arduino8266 import toolchain
+
+        objdump_path = str(toolchain.get_objdump_path())
+        readelf_path = str(toolchain.get_readelf_path())
+
         firmware_elf = toolchain.get_elf_path()
     else:
         from esphome.platformio import toolchain

esphome/arduino8266/component.py

@@ -39,6 +39,12 @@ _LOGGER = logging.getLogger(__name__)
 
 ESP8266_PLATFORM = "espressif8266"
 
+# Bare names components register that intentionally have no bundled library
+# ("Updater" is replaced by ESPHome's native OTA backend). Anything else
+# missing from the framework tree is worth a warning: it is likely a typo or
+# a library the build genuinely needs.
+_KNOWN_ABSENT_BUNDLED = frozenset({"Updater"})
+
 
 @dataclass
 class ArduinoLibrary:
@@ -133,11 +139,14 @@ def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]:
             external.append(library)
         elif (framework_path / "libraries" / library.name).is_dir():
             bundled.append(_bundled_library(framework_path, library.name))
+        elif library.name in _KNOWN_ABSENT_BUNDLED:
+            _LOGGER.debug("Skipping known-absent bundled library %s", library.name)
         else:
-            # A bare name that is not a bundled library ("Updater" from the
-            # ota component) has nothing to build; PlatformIO's LDF-off mode
-            # ignores it the same way.
-            _LOGGER.debug("Skipping unknown bundled library %s", library.name)
+            _LOGGER.warning(
+                "Library %s is not bundled with the Arduino framework and has "
+                "no version or repository to download it from; skipping",
+                library.name,
+            )
 
     converted: list[ArduinoLibrary] = []
     bundled_names = {lib.name for lib in bundled}
@@ -158,7 +167,8 @@ def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]:
                 continue
             try:
                 check_library_data(dep, ESP8266_PLATFORM, "arduino")
-            except InvalidLibrary:
+            except InvalidLibrary as err:
+                _LOGGER.debug("Skipping bundled dependency %s: %s", name, err)
                 continue
             bundled_names.add(name)
             bundled.append(_bundled_library(framework_path, name))

esphome/arduino8266/framework.py

@@ -31,6 +31,7 @@ from esphome.core import CORE, EsphomeError
 from esphome.framework_helpers import (
     archive_extract_all,
     download_from_mirrors,
+    download_with_resume,
     rmdir,
     str_to_lst_of_str,
 )
@@ -47,6 +48,17 @@ TOOLCHAIN_VERSION = "2.100300.220621"
 
 NINJA_VERSION = "1.12.1"
 
+# sha256 of the ninja release archives, so the binary we chmod and execute is
+# integrity-checked. Only applies to the default download source; a mirror
+# override via ESPHOME_ARDUINO8266_NINJA_MIRRORS is trusted as configured.
+_NINJA_SHA256 = {
+    "ninja-mac.zip": "89a287444b5b3e98f88a945afa50ce937b8ffd1dcc59c555ad9b1baf855298c9",
+    "ninja-win.zip": "f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a",
+    "ninja-winarm64.zip": "79c96a50e0deafec212cfa85aa57c6b74003f52d9d1673ddcd1eab1c958c5900",
+    "ninja-linux.zip": "6f98805688d19672bd699fbbfa2c2cf0fc054ac3df1f0e6a47664d963d530255",
+    "ninja-linux-aarch64.zip": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1",
+}
+
 _REGISTRY_URL = (
     "https://api.registry.platformio.org/v3/packages/platformio/tool/{package}"
 )
@@ -121,8 +133,10 @@ def _pio_system() -> str:
     return "linux_x86_64"
 
 
-def _registry_download_url(package: str, version: str) -> str:
-    """Resolve a package's download URL for this host via the PIO registry."""
+def _registry_download(
+    package: str, version: str
+) -> tuple[str, str | None, int | None]:
+    """Resolve a package's download URL, sha256, and size via the PIO registry."""
     import requests
 
     url = _REGISTRY_URL.format(package=package)
@@ -136,7 +150,11 @@ def _registry_download_url(package: str, version: str) -> str:
         for file in ver.get("files", []):
             systems = file.get("system") or "*"
             if systems == "*" or system in systems:
-                return file["download_url"]
+                return (
+                    file["download_url"],
+                    (file.get("checksum") or {}).get("sha256"),
+                    file.get("size"),
+                )
         raise EsphomeError(f"No {package} {version} build for this platform ({system})")
     raise EsphomeError(f"{package} {version} not found in the package registry")
 
@@ -147,21 +165,28 @@ def _install_package(
     dest: Path,
     mirrors: list[str],
 ) -> None:
-    """Download and extract one package if not already installed."""
+    """Download, verify, and extract one package if not already installed.
+
+    The registry path is integrity-checked against the sha256 the registry
+    publishes; a mirror override is trusted as configured.
+    """
     marker = dest / ".esphome_extracted"
     if marker.is_file():
         return
     rmdir(dest, msg=f"Clean up incomplete {name} install")
-    with tempfile.NamedTemporaryFile() as tmp:
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        archive = Path(tmp_dir) / "package"
         _LOGGER.info("Downloading %s %s ...", name, version)
         if mirrors:
-            download_from_mirrors(
-                mirrors, {"VERSION": version, "SYSTEM": _pio_system()}, tmp.file
-            )
+            with archive.open("wb") as file:
+                download_from_mirrors(
+                    mirrors, {"VERSION": version, "SYSTEM": _pio_system()}, file
+                )
         else:
-            download_from_mirrors([_registry_download_url(name, version)], {}, tmp.file)
+            url, sha256, size = _registry_download(name, version)
+            download_with_resume(url, archive, sha256=sha256, size=size)
         _LOGGER.info("Extracting %s ...", name)
-        archive_extract_all(tmp.file, dest, progress_header="Extracting")
+        archive_extract_all(archive, dest, progress_header="Extracting")
     marker.touch()
 
 
@@ -186,14 +211,23 @@ def _check_ninja_install() -> Path:
     if binary.is_file():
         return binary
     rmdir(ninja_dir, msg="Clean up incomplete ninja install")
-    with tempfile.NamedTemporaryFile() as tmp:
+    archive_name = _ninja_archive_name()
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        archive = Path(tmp_dir) / archive_name
         _LOGGER.info("Downloading ninja %s ...", NINJA_VERSION)
-        download_from_mirrors(
-            ESPHOME_ARDUINO8266_NINJA_MIRRORS,
-            {"VERSION": NINJA_VERSION, "ARCHIVE": _ninja_archive_name()},
-            tmp.file,
-        )
-        archive_extract_all(tmp.file, ninja_dir)
+        if "ESPHOME_ARDUINO8266_NINJA_MIRRORS" in os.environ:
+            with archive.open("wb") as file:
+                download_from_mirrors(
+                    ESPHOME_ARDUINO8266_NINJA_MIRRORS,
+                    {"VERSION": NINJA_VERSION, "ARCHIVE": archive_name},
+                    file,
+                )
+        else:
+            url = ESPHOME_ARDUINO8266_NINJA_MIRRORS[0].format(
+                VERSION=NINJA_VERSION, ARCHIVE=archive_name
+            )
+            download_with_resume(url, archive, sha256=_NINJA_SHA256[archive_name])
+        archive_extract_all(archive, ninja_dir)
     if not binary.is_file():
         raise EsphomeError(f"ninja binary missing after extraction in {ninja_dir}")
     binary.chmod(binary.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

esphome/arduino8266/toolchain.py

@@ -13,7 +13,7 @@ from esphome.const import (
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
 )
-from esphome.core import CORE
+from esphome.core import CORE, EsphomeError
 from esphome.helpers import write_file_if_changed
 from esphome.types import ConfigType
 
@@ -34,8 +34,20 @@ def get_elf_path() -> Path:
     return get_build_dir() / "firmware.elf"
 
 
+def _toolchain_tool(name: str) -> Path:
+    return framework.get_toolchain_path() / "bin" / f"xtensa-lx106-elf-{name}"
+
+
 def get_addr2line_path() -> Path:
-    return framework.get_toolchain_path() / "bin" / "xtensa-lx106-elf-addr2line"
+    return _toolchain_tool("addr2line")
+
+
+def get_objdump_path() -> Path:
+    return _toolchain_tool("objdump")
+
+
+def get_readelf_path() -> Path:
+    return _toolchain_tool("readelf")
 
 
 def run_compile(config: ConfigType, verbose: bool) -> int:
@@ -77,12 +89,14 @@ def _write_compile_commands(
         check=False,
         close_fds=False,
     )
-    if result.returncode == 0:
-        # write_file_if_changed keeps the mtime stable on no-op builds so the
-        # idedata cache in get_idedata() stays valid.
-        write_file_if_changed(build_dir / "compile_commands.json", result.stdout)
-    else:
-        _LOGGER.warning("Could not generate compile_commands.json: %s", result.stderr)
+    if result.returncode != 0:
+        # Drop any stale database so consumers (IDE integration, clang-tidy,
+        # the memory analyzer) can't silently read outdated data.
+        (build_dir / "compile_commands.json").unlink(missing_ok=True)
+        raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}")
+    # write_file_if_changed keeps the mtime stable on no-op builds so the
+    # idedata cache in get_idedata() stays valid.
+    write_file_if_changed(build_dir / "compile_commands.json", result.stdout)
 
 
 def _parse_app_size(build_dir: Path) -> int | None:
@@ -90,12 +104,18 @@ def _parse_app_size(build_dir: Path) -> int | None:
     from esphome.build_gen.arduino8266 import get_flash_ld_path
     from esphome.components.esp8266.build_surgery import segment_length
 
+    # Warnings, not debug: without the app size the Flash summary line is
+    # dropped and CI's memory-impact extraction loses its flash metric.
+    ld_path = get_flash_ld_path(build_dir)
     try:
-        ld_text = get_flash_ld_path(build_dir).read_text(encoding="utf-8")
+        ld_text = ld_path.read_text(encoding="utf-8")
     except OSError as err:
-        _LOGGER.debug("Cannot read linker script for the Flash summary: %s", err)
+        _LOGGER.warning("Cannot read linker script for the Flash summary: %s", err)
         return None
-    return segment_length(ld_text, "irom0_0_seg")
+    app_size = segment_length(ld_text, "irom0_0_seg")
+    if app_size is None:
+        _LOGGER.warning("irom0_0_seg not found in %s; skipping Flash summary", ld_path)
+    return app_size
 
 
 def _print_size_summary(build_dir: Path, toolchain_path: Path) -> None:
@@ -124,6 +144,8 @@ def _print_size_summary(build_dir: Path, toolchain_path: Path) -> None:
             try:
                 sections[parts[0]] = int(parts[1])
             except ValueError:
+                # An omitted section would silently skew the reported totals
+                _LOGGER.warning("Unparsable size output for section %s", parts[0])
                 continue
     ram = sum(sections.get(s, 0) for s in _RAM_SECTIONS)
     flash = sum(sections.get(s, 0) for s in _FLASH_SECTIONS)

esphome/build_gen/arduino8266.py

@@ -365,7 +365,9 @@ def generate_ld_scripts(
         flash_ld = framework / "tools" / "sdk" / "ld" / flash_ld_name
         write_file_if_changed(
             ld_dir / f"testing_{flash_ld_name}",
-            apply_testing_memory_patches(flash_ld.read_text(encoding="utf-8")),
+            apply_testing_memory_patches(
+                flash_ld.read_text(encoding="utf-8"), require=True
+            ),
         )
 
 
@@ -423,13 +425,8 @@ def write_project(paths: dict[str, Path]) -> bool:
 
     libraries = resolve_libraries(framework)
 
-    # A missing framework-owned directory is a broken install; failing here
-    # names the path instead of producing a wall of include errors.
-    for required in (sdk / "include", core_dir, sdk / "lwip2" / "include", variant_dir):
-        if not required.is_dir():
-            raise EsphomeError(
-                f"Arduino framework install is incomplete: missing {required}"
-            )
+    # A missing install directory would otherwise surface as a wall of
+    # include errors; failing here names the path instead.
     include_dirs = [
         src_dir,
         sdk / "include",
@@ -438,7 +435,11 @@ def write_project(paths: dict[str, Path]) -> bool:
         sdk / "lwip2" / "include",
         variant_dir,
     ]
-    include_dirs = [d for d in include_dirs if d.is_dir()]
+    for required in include_dirs:
+        if not required.is_dir():
+            raise EsphomeError(
+                f"Arduino toolchain install is incomplete: missing {required}"
+            )
     for lib in libraries:
         include_dirs += lib.include_dirs
 

esphome/components/esp8266/build_surgery.py

@@ -44,6 +44,13 @@ def relocate_ratetable(content: str) -> str:
     )
 
 
+_TESTING_SEGMENT_SIZES = (
+    ("iram1_0_seg", TESTING_IRAM_SIZE),
+    ("dram0_0_seg", TESTING_DRAM_SIZE),
+    ("irom0_0_seg", TESTING_FLASH_SIZE),
+)
+
+
 def _patch_segment_size(content: str, segment_name: str, new_size: str) -> str:
     pattern = (
         rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)"
@@ -52,11 +59,26 @@ def _patch_segment_size(content: str, segment_name: str, new_size: str) -> str:
     return re.sub(pattern, rf"\g<1>{new_size}", content)
 
 
-def apply_testing_memory_patches(content: str) -> str:
-    """Enlarge IRAM/DRAM/flash segments so grouped CI test builds can link."""
-    content = _patch_segment_size(content, "iram1_0_seg", TESTING_IRAM_SIZE)
-    content = _patch_segment_size(content, "dram0_0_seg", TESTING_DRAM_SIZE)
-    return _patch_segment_size(content, "irom0_0_seg", TESTING_FLASH_SIZE)
+def apply_testing_memory_patches(content: str, require: bool = False) -> str:
+    """Enlarge IRAM/DRAM/flash segments so grouped CI test builds can link.
+
+    With ``require``, raise when a segment was not found: a silently
+    unpatched flash linker script would keep the real 32KB IRAM limits and
+    fail grouped builds far from the cause. The common linker script has no
+    MEMORY block, so its caller leaves ``require`` off.
+    """
+    missing: list[str] = []
+    for segment, size in _TESTING_SEGMENT_SIZES:
+        patched = _patch_segment_size(content, segment, size)
+        if patched == content:
+            missing.append(segment)
+        content = patched
+    if require and missing:
+        raise RuntimeError(
+            f"Testing-mode memory patch failed: segment(s) {', '.join(missing)} "
+            "not found (has the Arduino core linker script changed?)"
+        )
+    return content
 
 
 def segment_length(content: str, segment_name: str) -> int | None:

esphome/core/config.py

@@ -557,10 +557,12 @@ def _add_library_str(lib: str) -> None:
 
 @coroutine_with_priority(CoroPriority.FINAL)
 async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None:
-    if CORE.using_toolchain_esp_idf:
-        # The native ESP-IDF build doesn't read platformio.ini; honor the
-        # options with a native equivalent and warn about the rest, which
-        # would otherwise be silently ignored.
+    if CORE.using_toolchain_esp_idf or (
+        CORE.using_toolchain_arduino and CORE.is_esp8266
+    ):
+        # The native builds don't read platformio.ini; honor the options
+        # with a native equivalent and warn about the rest, which would
+        # otherwise be silently ignored.
         for key, val in pio_options.items():
             vals = [val] if isinstance(val, str) else val
             if key == CONF_BUILD_FLAGS:
@@ -588,8 +590,9 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No
                 # config at upload time (upload_using_esptool)
                 _LOGGER.warning(
                     "esphome->platformio_options->%s is ignored when building with "
-                    "the native ESP-IDF toolchain",
+                    "the native '%s' toolchain",
                     key,
+                    CORE.toolchain.value,
                 )
         return
     # Add includes at the very end, so that they override everything