Derive the exact image size from the ELF when json2 lacks total_size

esphome/espidf/size_summary.py

@@ -11,11 +11,10 @@ byte-identical to PlatformIO's output:
 The format matches ``script/ci_memory_impact_extract.py`` so CI memory
 analysis works unchanged on native ESP-IDF builds. RAM usage comes from
 the DRAM (or unified DIRAM) region of the linker map. Flash used is the
-json2 ``total_size`` field when present (esp-idf-size >= 2.1, the exact
-map-derived figure matching the ``Total image size`` line); older 1.x
-json2 lacks it, and the fallback is the size of the app ``.bin`` on
-disk, which includes esptool's 16-byte image padding and appended
-SHA-256 so it reads slightly high and moves in 16-byte steps.
+exact image size matching the ``Total image size`` line: the json2
+``total_size`` field when present (esp-idf-size >= 2.1); older 1.x
+json2 lacks it, so we derive the same figure from the ELF by summing
+loadable PROGBITS sections, the rule esp_idf_size itself uses.
 Flash total is taken from
 ``partitions.csv`` using PlatformIO's rule (first app partition whose
 subtype is ``factory`` or ``ota_0``; see
@@ -35,6 +34,7 @@ import csv
 import json
 import logging
 from pathlib import Path
+import struct
 
 from esphome.build_helpers.size_summary import print_size_line
 
@@ -77,7 +77,30 @@ def _find_app_partition_size(partitions_csv: Path) -> int:
     raise ValueError(f"No app+factory or app+ota_0 partition in {partitions_csv}")
 
 
-def print_summary(size_json: Path, partitions_csv: Path, firmware_bin: Path) -> None:
+def _image_size_from_elf(elf: Path) -> int:
+    """Sum the loadable PROGBITS section sizes from an ELF32 file.
+
+    This is the rule ``esp_idf_size.ng.memorymap._get_image_size`` uses for
+    its image size figure, so the result is byte-identical to the tool's.
+    Raises ``ValueError`` if the file is not a 32-bit little-endian ELF.
+    """
+    data = elf.read_bytes()
+    if len(data) < 52 or data[:4] != b"\x7fELF" or data[4] != 1 or data[5] != 1:
+        raise ValueError(f"{elf} is not a 32-bit little-endian ELF")
+    (e_shoff,) = struct.unpack_from("<I", data, 0x20)
+    e_shentsize, e_shnum = struct.unpack_from("<HH", data, 0x2E)
+    total = 0
+    for i in range(e_shnum):
+        off = e_shoff + i * e_shentsize
+        sh_type, sh_flags = struct.unpack_from("<II", data, off + 4)
+        (sh_size,) = struct.unpack_from("<I", data, off + 20)
+        # SHT_PROGBITS with SHF_ALLOC: sections with data that occupy memory
+        if sh_size and sh_type == 1 and sh_flags & 0x2:
+            total += sh_size
+    return total
+
+
+def print_summary(size_json: Path, partitions_csv: Path, firmware_elf: Path) -> None:
     """Print PlatformIO-shaped RAM and Flash one-liners.
 
     Failures are non-fatal: the build has already succeeded, we just couldn't
@@ -103,15 +126,14 @@ def print_summary(size_json: Path, partitions_csv: Path, firmware_bin: Path) ->
     if ram_total and ram_used is not None:
         print_size_line("RAM", ram_used, ram_total)
 
-    # esp-idf-size >= 2.1 (IDF >= 6.0) reports the exact map-derived image
-    # size in json2; older 1.x omits it, so fall back to the padded on-disk
-    # .bin size.
+    # esp-idf-size >= 2.1 (IDF >= 6.0) reports the exact image size in
+    # json2; older 1.x omits it, so derive the same figure from the ELF.
     flash_used = data.get("total_size")
     try:
         if flash_used is None:
-            flash_used = firmware_bin.stat().st_size
+            flash_used = _image_size_from_elf(firmware_elf)
         app_size = _find_app_partition_size(partitions_csv)
-    except (OSError, ValueError) as e:
+    except (OSError, ValueError, struct.error) as e:
         _LOGGER.debug("Skipping Flash summary: %s", e)
         return
     print_size_line("Flash", flash_used, app_size)

esphome/espidf/toolchain.py

@@ -542,7 +542,7 @@ def run_compile(config, verbose: bool) -> int:
     if rc == 0:
         size_json = CORE.relative_build_path("build", "esp_idf_size.json")
         partitions = CORE.relative_build_path("partitions.csv")
-        print_summary(size_json, partitions, get_firmware_path())
+        print_summary(size_json, partitions, get_built_elf_path())
     return rc
 
 
@@ -579,6 +579,16 @@ def get_ota_firmware_path() -> Path:
     return build_dir / "firmware.ota.bin"
 
 
+def get_built_elf_path() -> Path:
+    """Get the path to the ELF that idf.py writes directly, ``<build>/<name>.elf``.
+
+    Unlike ``get_elf_path``, this file exists as soon as the build finishes,
+    before ``create_elf_copy`` produces the ``firmware.elf`` copy.
+    """
+    build_dir = CORE.relative_build_path("build")
+    return build_dir / f"{CORE.name}.elf"
+
+
 def get_elf_path() -> Path:
     """Get the path to the firmware ELF file.
 
@@ -706,8 +716,7 @@ def create_elf_copy() -> bool:
     "download ELF" link requests the literal filename ``firmware.elf``
     (PlatformIO convention), so copy it to that name.
     """
-    build_dir = CORE.relative_build_path("build")
-    src_elf = build_dir / f"{CORE.name}.elf"
+    src_elf = get_built_elf_path()
     dst_elf = get_elf_path()
 
     if not src_elf.is_file():

tests/unit_tests/test_espidf_toolchain.py

@@ -639,8 +639,8 @@ def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None:
 
 
 def test_run_compile_passes_size_summary_paths(setup_core: Path) -> None:
-    """print_summary receives the size json, partitions.csv, and the firmware
-    bin from get_firmware_path, which must stay in lockstep with the
+    """print_summary receives the size json, partitions.csv, and the built
+    ELF from get_built_elf_path, which must stay in lockstep with the
     project() name in the generated CMakeLists."""
     _setup_build(setup_core)
     config = {CONF_ESPHOME: {}}
@@ -655,7 +655,7 @@ def test_run_compile_passes_size_summary_paths(setup_core: Path) -> None:
     mock_summary.assert_called_once_with(
         CORE.relative_build_path("build", "esp_idf_size.json"),
         CORE.relative_build_path("partitions.csv"),
-        toolchain.get_firmware_path(),
+        toolchain.get_built_elf_path(),
     )
 
 

tests/unit_tests/test_size_summary.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
+import struct
 
 import pytest
 
@@ -27,6 +28,25 @@ def _write_partitions(tmp_path: Path) -> Path:
     return out
 
 
+def _write_elf(tmp_path: Path, sections: list[tuple[int, int, int]]) -> Path:
+    """Write a minimal ELF32 LE whose section headers carry the given
+    (sh_type, sh_flags, sh_size) triples."""
+    header = bytearray(52)
+    header[0:4] = b"\x7fELF"
+    header[4] = header[5] = 1  # 32-bit, little-endian
+    struct.pack_into("<I", header, 0x20, 52)  # e_shoff
+    struct.pack_into("<HH", header, 0x2E, 40, len(sections))
+    out = bytearray(header)
+    for sh_type, sh_flags, sh_size in sections:
+        shdr = bytearray(40)
+        struct.pack_into("<II", shdr, 4, sh_type, sh_flags)
+        struct.pack_into("<I", shdr, 20, sh_size)
+        out += shdr
+    elf = tmp_path / "firmware.elf"
+    elf.write_bytes(bytes(out))
+    return elf
+
+
 def _esp32_size_data() -> dict:
     """Synthetic json2 for the original ESP32 (split IRAM/DRAM), in the
     esp-idf-size >= 2.1 shape that carries ``total_size``."""
@@ -152,43 +172,51 @@ def test_print_summary_handles_no_layout(
 def test_print_summary_flash_line_prefers_total_size(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:
-    """With ``total_size`` in the json, the exact figure wins over the padded
-    bin size, in the exact shape script/ci_memory_impact_extract.py greps."""
+    """With ``total_size`` in the json, that figure wins without reading the
+    ELF, in the exact shape script/ci_memory_impact_extract.py greps."""
     size_json = _write_size_json(tmp_path, _esp32_size_data())
     partitions = _write_partitions(tmp_path)
-    firmware_bin = tmp_path / "firmware.bin"
-    firmware_bin.write_bytes(b"\x00" * 999999)
-    print_summary(size_json, partitions, firmware_bin)
+    print_summary(size_json, partitions, tmp_path / "firmware.elf")
     out = capsys.readouterr().out
     assert "Flash: " in out
     assert "(used 827455 bytes from 1835008 bytes)" in out
 
 
-def test_print_summary_flash_line_falls_back_to_bin_size(
+def test_print_summary_flash_line_derives_from_elf(
     tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:
-    """A 1.x json without ``total_size`` uses the on-disk bin size."""
+    """A 1.x json without ``total_size`` sums the ELF's loadable PROGBITS
+    sections; NOBITS and non-alloc sections are excluded."""
     size_json = _write_size_json(tmp_path, _s3_size_data())
     partitions = _write_partitions(tmp_path)
-    firmware_bin = tmp_path / "firmware.bin"
-    firmware_bin.write_bytes(b"\x00" * 724224)
-    print_summary(size_json, partitions, firmware_bin)
+    firmware_elf = _write_elf(
+        tmp_path,
+        [
+            (1, 0x6, 700000),  # PROGBITS, alloc+exec: counted
+            (1, 0x2, 24215),  # PROGBITS, alloc: counted
+            (8, 0x2, 50000),  # NOBITS (.bss): excluded
+            (1, 0x0, 12345),  # PROGBITS, no alloc (.debug_*): excluded
+        ],
+    )
+    print_summary(size_json, partitions, firmware_elf)
     out = capsys.readouterr().out
-    assert "(used 724224 bytes from 1835008 bytes)" in out
+    assert "(used 724215 bytes from 1835008 bytes)" in out
 
 
-@pytest.mark.parametrize("missing", ["bin", "partitions"])
-def test_print_summary_skips_flash_on_missing_input(
-    missing: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+@pytest.mark.parametrize("problem", ["missing_elf", "not_an_elf", "missing_partitions"])
+def test_print_summary_skips_flash_on_bad_input(
+    problem: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]
 ) -> None:
-    """A missing firmware bin or partitions.csv skips the Flash line, not the RAM line."""
+    """An unusable ELF or missing partitions.csv skips the Flash line, not the RAM line."""
     size_json = _write_size_json(tmp_path, _s3_size_data())
-    firmware_bin = tmp_path / "firmware.bin"
-    if missing == "bin":
-        _write_partitions(tmp_path)
+    firmware_elf = tmp_path / "firmware.elf"
+    if problem == "missing_partitions":
+        _write_elf(tmp_path, [(1, 0x2, 1024)])
     else:
-        firmware_bin.write_bytes(b"\x00" * 16)
-    print_summary(size_json, tmp_path / "partitions.csv", firmware_bin)
+        _write_partitions(tmp_path)
+        if problem == "not_an_elf":
+            firmware_elf.write_bytes(b"junk")
+    print_summary(size_json, tmp_path / "partitions.csv", firmware_elf)
     out = capsys.readouterr().out
     assert "RAM:" in out
     assert "Flash:" not in out