Compare commits

...
Author SHA1 Message Date
J. Nick Koston 472c7335fc Merge branch 'analyze-memory-esp8266-rodata' into analyze-memory-esp8266-strings
# Conflicts:
#	esphome/analyze_memory/__init__.py
2026-11-15 03:35:45 -06:00
J. Nick Koston 4ba019395b [analyze_memory] Skip constants whose linked copy is unknown, detect jump tables from unnamed bytes only 2026-11-15 03:35:15 -06:00
J. Nick Koston 96562837f1 Merge branch 'analyze-memory-esp8266-rodata' into analyze-memory-esp8266-strings
# Conflicts:
#	esphome/analyze_memory/__init__.py
2026-11-15 03:22:16 -06:00
J. Nick Koston 2f984a48b9 [analyze_memory] Count literals after other data, attribute literals reached through unnamed tables 2026-11-15 03:21:54 -06:00
J. Nick Koston d3dd10da12 [analyze_memory] Share relocation and dump parsing, name native library objects 2026-11-15 03:21:45 -06:00
J. Nick Koston 60ae039690 [analyze_memory] Share relocation and dump parsing, track string users per component 2026-11-15 03:05:53 -06:00
J. Nick Koston 6959075536 Merge branch 'analyze-memory-esp8266-rodata' into analyze-memory-esp8266-strings
# Conflicts:
#	esphome/analyze_memory/__init__.py
2026-11-15 03:00:11 -06:00
J. Nick Koston da34aa9f7a [analyze_memory] Resolve startup and IRAM code sections, dedupe inline copies only across objects 2026-11-15 02:59:41 -06:00
J. Nick Koston 073773cfd8 [analyze_memory] Attribute ESP8266 string literals in RAM to components 2026-11-15 02:55:16 -06:00
J. Nick Koston 2b27ba016b [analyze_memory] Report object paths with forward slashes on every OS 2026-11-15 02:53:02 -06:00
J. Nick Koston e5e85609a9 [analyze_memory] Use tab escapes in the objdump test fixture 2026-11-15 02:47:16 -06:00
J. Nick Koston 180f952fd0 [analyze_memory] Keep alias names, dedupe only inline copies, preview only unnamed bytes 2026-11-15 02:44:53 -06:00
J. Nick Koston c607199b23 [analyze_memory] Scan only objects with unnamed bytes and show the strings a constant points to 2026-11-15 02:35:58 -06:00
J. Nick Koston 058a5be777 [analyze_memory] Count ESP8266 .rodata as RAM and list unnamed constants 2026-11-15 02:29:01 -06:00
6 changed files with 1353 additions and 24 deletions

No files matched your search

+517 -19
View File
@@ -1,7 +1,9 @@
"""Memory usage analyzer for ESPHome compiled binaries."""
from collections import Counter, defaultdict
from collections.abc import Container
from dataclasses import dataclass, field
from functools import cached_property
import logging
from pathlib import Path
import re
@@ -55,6 +57,7 @@ _COMPONENT_PREFIX_EXTERNAL = "[external]"
_COMPONENT_PREFIX_LIB = "[lib]"
_COMPONENT_CORE = f"{_COMPONENT_PREFIX_ESPHOME}core"
_COMPONENT_API = f"{_COMPONENT_PREFIX_ESPHOME}api"
_COMPONENT_ARDUINO_CORE = "arduino_core"
# Placement new storage suffix (generated by codegen Pvariable)
_PSTORAGE_SUFFIX = "__pstorage"
@@ -83,6 +86,28 @@ _NAMESPACE_STD = "std::"
# Type alias for symbol information: (symbol_name, size, component)
SymbolInfoType = tuple[str, int, str]
# Only ESP8266 has this section; its .rodata is copied to RAM at boot like .data
_ESP8266_FLASH_CODE_SECTION = ".irom0.text"
# readelf -SW row: [Nr] Name Type Address Off Size ES Flg
_OBJ_SECTION_PATTERN = re.compile(
r"^\s*\[\s*(\d+)\]\s+(\S+)\s+(\S+)\s+[\da-f]+\s+[\da-f]+\s+([\da-f]+)\s+[\da-f]+\s+(\S*)"
)
_RELOC_SECTION_PATTERN = re.compile(r"^Relocation section '\.rela(\S+)'")
# String literal section; group 1 is the function for -fdata-sections names
_STRING_SECTION_PATTERN = re.compile(r"^\.rodata(?:\.(.+))?\.str\d+\.\d+$")
# Data section of one symbol; group 1 is the symbol
_DATA_SECTION_PATTERN = re.compile(r"^\.(?:rodata|data)\.(.+)$")
# Code section; group 1 is the function for -ffunction-sections names, after any
# placement prefix (.text.startup.<fn>, .iram.text.<fn> for IRAM_ATTR)
_CODE_SECTION_PATTERN = re.compile(
r"^\.(?:literal|text|irom0\.text|iram\.text|iram0\.text)"
r"(?:\.(?:startup|unlikely|hot|exit)(?=\.|$))?(?:\.(.+))?$"
)
# readelf -x row: " 0x00000000 0000803f 0000803f 0000803f 0000803f ...?...?...?...?"
_HEX_DUMP_PREFIX = " 0x"
_HEX_DUMP_COLUMNS = slice(13, 48)
# RAM sections - symbols in these sections consume RAM
RAM_SECTIONS = frozenset([".data", ".bss"])
@@ -119,6 +144,29 @@ class SDKSymbol:
demangled: str = "" # Demangled name (populated after analysis)
@dataclass
class UnnamedConstant:
"""Constant without a symbol in an object's .rodata (RAM on ESP8266)."""
size: int
source_file: str
component: str
functions: list[str] # Functions that reference it, mangled
jump_table: bool # Holds code addresses (a switch table), not a value
data: bytes # Leading bytes, to recognize the value
strings: list[str] = field(default_factory=list) # String literals it points to
@dataclass
class StringLiteral:
"""A string literal kept in RAM (.rodata on ESP8266)."""
text: str
size: int # Bytes in RAM; strings shared or merged into a longer one count once
# Component -> functions or objects that use it, mangled
users: dict[str, set[str]] = field(default_factory=lambda: defaultdict(set))
@dataclass
class ComponentMemory:
"""Tracks memory usage for a component."""
@@ -152,6 +200,188 @@ class ComponentMemory:
return self.data_size + self.bss_size
@dataclass
class _RodataSection:
"""An object's .rodata section and the byte ranges its symbols cover."""
name: str
size: int
covered: list[tuple[int, int]] = field(default_factory=list) # (offset, size)
@cached_property
def _named(self) -> frozenset[int]:
return frozenset(
offset
for start, size in self.covered
for offset in range(start, min(start + size, self.size))
)
def is_unnamed(self, offset: int) -> bool:
return 0 <= offset < self.size and offset not in self._named
@property
def unnamed_size(self) -> int:
return self.size - len(self._named)
def _parse_string_sections(output: str) -> dict[str, dict[str, tuple[str, str | None]]]:
"""Find string literal sections in ``readelf -SW`` output.
Returns:
Object path -> section index -> (section name, function it is named after)
"""
sections: dict[str, dict[str, tuple[str, str | None]]] = defaultdict(dict)
file_path = ""
for line in output.splitlines():
if line.startswith("File: "):
file_path = line[6:]
elif (match := _OBJ_SECTION_PATTERN.match(line)) is not None:
index, name, section_type, size_hex, flags = match.groups()
if (
section_type == "PROGBITS"
and "S" in flags
and int(size_hex, 16)
and (string := _STRING_SECTION_PATTERN.match(name))
):
sections[file_path][index] = (name, string.group(1))
return sections
def _ram_string_size(text: bytes, rodata: bytes, literals: Container[bytes]) -> int:
"""Bytes a literal takes in RAM: 0 if dropped or only the tail of another literal."""
needle = text + b"\0"
start = rodata.find(needle)
while start != -1:
head = rodata.rfind(b"\0", 0, start) + 1
# Bytes before it that are not one of our literals are other data
if start == head or rodata[head : start + len(text)] not in literals:
return len(needle)
start = rodata.find(needle, start + 1)
return 0
def _parse_hex_dumps(output: str) -> dict[str, bytearray]:
"""Collect the bytes of each ``readelf -x`` section dump by section name."""
contents: dict[str, bytearray] = defaultdict(bytearray)
section: str | None = None
for line in output.splitlines():
if line.startswith("Hex dump of section"):
section = line.split("'")[1]
elif section is not None and line.startswith(_HEX_DUMP_PREFIX):
contents[section] += bytes.fromhex(line[_HEX_DUMP_COLUMNS].replace(" ", ""))
elif line and not line.startswith(" "):
section = None
return contents
def _parse_relocations(output: str) -> list[tuple[str, int, str, int]]:
"""List (relocated section, offset, target, addend) from ``readelf -rW`` output."""
relocations: list[tuple[str, int, str, int]] = []
section: str | None = None
for line in output.splitlines():
if line.startswith("Relocation section"):
match = _RELOC_SECTION_PATTERN.match(line)
section = match.group(1) if match else None
elif line[:1].isdigit() and section is not None:
# Offset Info Type Sym.Value Sym.Name + Addend
fields = line.split()
if len(fields) < 5:
continue
addend = int(fields[6], 16) if len(fields) > 6 else 0
if len(fields) > 6 and fields[5] == "-":
addend = -addend
relocations.append((section, int(fields[0], 16), fields[4], addend))
elif line and line[0] != " ":
section = None
return relocations
def _pointed_literal(
referrer: bytes, offset: int, addend: int, strings: bytes
) -> bytes:
"""Read the string literal a relocated pointer at ``offset`` refers to."""
# Xtensa keeps part of the addend in the relocated word
start = addend + int.from_bytes(referrer[offset : offset + 4], "little")
end = strings.find(b"\0", start)
return bytes(strings[start : end if end != -1 else len(strings)])
def _section_user(section: str) -> str | None:
"""The function or data symbol a -ffunction-sections/-fdata-sections name is for."""
match = _CODE_SECTION_PATTERN.match(section) or _DATA_SECTION_PATTERN.match(section)
return match.group(1) if match else None
@dataclass
class _FunctionDefinitions:
"""Where each function is defined across the scanned objects."""
weak: dict[str, set[str]] = field(default_factory=lambda: defaultdict(set))
strong: set[str] = field(default_factory=set) # Global, non-weak
count: Counter[str] = field(default_factory=Counter) # Objects defining it
def linked_copy_unknown(self, file_path: str, functions: list[str]) -> bool:
"""True if another object's copy of these functions may be the linked one."""
if self.weak[file_path].issuperset(functions):
# Inline copies: a strong definition elsewhere replaces them all
return not self.strong.isdisjoint(functions)
# Same-named local functions in several objects; the ELF can't tell them apart
return all(f not in self.strong and self.count[f] > 1 for f in functions)
def _parse_unnamed_rodata(
output: str,
) -> tuple[dict[str, dict[str, _RodataSection]], _FunctionDefinitions]:
"""Find .rodata sections with bytes no symbol covers in ``readelf -SW -sW`` output.
Returns:
Object path -> section index -> section, and where functions are defined
"""
sections: dict[str, dict[str, _RodataSection]] = defaultdict(dict)
definitions = _FunctionDefinitions()
file_path = ""
for line in output.splitlines():
if line.startswith("File: "):
file_path = line[6:]
elif match := _OBJ_SECTION_PATTERN.match(line):
index, name, section_type, size_hex, flags = match.groups()
# Merged sections (string literals) are shared between objects
if (
section_type == "PROGBITS"
and name.startswith(".rodata")
and "A" in flags
and "M" not in flags
):
sections[file_path][index] = _RodataSection(name, int(size_hex, 16))
else:
# Num: Value Size Type Bind Vis Ndx Name
fields = line.split()
if len(fields) < 8:
continue
if fields[3] == "OBJECT" and (
section := sections[file_path].get(fields[6])
):
section.covered.append((int(fields[1], 16), int(fields[2], 0)))
elif fields[3] == "FUNC" and fields[6] != "UND":
definitions.count[fields[7]] += 1
if fields[4] == "WEAK":
definitions.weak[file_path].add(fields[7])
elif fields[4] == "GLOBAL":
definitions.strong.add(fields[7])
unnamed = {
path: kept
for path, file_sections in sections.items()
if (
kept := {
index: section
for index, section in file_sections.items()
if section.unnamed_size
}
)
}
return unnamed, definitions
class MemoryAnalyzer:
"""Analyzes memory usage from ELF files."""
@@ -231,6 +461,10 @@ class MemoryAnalyzer:
self._heuristic_to_lib: dict[str, str] = {}
# Function call counts: mangled_name -> call_count
self._function_call_counts: Counter[str] = Counter()
# True on ESP8266, where .rodata is RAM; it is then counted as .data
self.rodata_in_ram = False
self._unnamed_constants: list[UnnamedConstant] = []
self._string_literals: list[StringLiteral] = []
def analyze(self) -> dict[str, ComponentMemory]:
"""Analyze the ELF file and return component memory usage."""
@@ -240,6 +474,7 @@ class MemoryAnalyzer:
self._scan_source_symbols()
self._categorize_symbols()
self._analyze_cswtch_symbols()
self._analyze_object_rodata()
self._analyze_sdk_libraries()
self._analyze_function_calls()
return dict(self.components)
@@ -253,18 +488,17 @@ class MemoryAnalyzer:
check=True,
)
# Parse section headers
for line in result.stdout.splitlines():
# Look for section entries
if not (match := _READELF_SECTION_PATTERN.match(line)):
continue
raw_sections = [
(match.group(1), int(match.group(2), 16))
for line in result.stdout.splitlines()
if (match := _READELF_SECTION_PATTERN.match(line))
]
self.rodata_in_ram = any(
name == _ESP8266_FLASH_CODE_SECTION for name, _ in raw_sections
)
section_name = match.group(1)
size_hex = match.group(2)
size = int(size_hex, 16)
# Map to standard section name
mapped_section = map_section_name(section_name)
for section_name, size in raw_sections:
mapped_section = self._standard_section(map_section_name(section_name))
if not mapped_section:
continue
@@ -272,6 +506,12 @@ class MemoryAnalyzer:
self.sections[mapped_section] = MemorySection(mapped_section)
self.sections[mapped_section].total_size += size
def _standard_section(self, section: str | None) -> str | None:
"""Count .rodata as .data where it is RAM (ESP8266)."""
if section == ".rodata" and self.rodata_in_ram:
return ".data"
return section
def _parse_symbols(self) -> None:
"""Parse symbols from ELF file."""
result = subprocess.run(
@@ -289,14 +529,18 @@ class MemoryAnalyzer:
continue
section, name, size, address = symbol_info
section = self._standard_section(section)
if section not in self.sections:
continue
# Every alias is in the ELF, even when its size is counted once
self._elf_symbol_names.add(name)
# Skip duplicate symbols at the same address (e.g., C1/C2 constructors)
if address in seen_addresses or section not in self.sections:
if address in seen_addresses:
continue
self.sections[section].symbols.append((name, size, ""))
self.sections[section].symbol_size += size
self._elf_symbol_names.add(name)
seen_addresses.add(address)
def _categorize_symbols(self) -> None:
@@ -925,10 +1169,12 @@ class MemoryAnalyzer:
Path to the directory containing .o files, or None if not found.
"""
# The ELF is typically at .pioenvs/<env>/firmware.elf
# Object files are in .pioenvs/<env>/src/ and .pioenvs/<env>/lib*/
# PlatformIO puts object files in .pioenvs/<env>/src/ and .pioenvs/<env>/lib*/,
# the native ESP8266 build in .pioenvs/<env>/obj/src/ and obj/lib/
pioenvs_dir = self.elf_path.parent
if pioenvs_dir.exists() and any(pioenvs_dir.glob("src/*.o")):
return pioenvs_dir
for obj_dir in (pioenvs_dir, pioenvs_dir / "obj"):
if any(obj_dir.glob("src/*.o")):
return obj_dir
return None
@staticmethod
@@ -1110,6 +1356,12 @@ class MemoryAnalyzer:
):
return f"{_COMPONENT_PREFIX_LIB}{self._lib_hash_to_name[parts[1]]}"
# Native ESP8266 build: lib/<owner>__<Name>/... and the Arduino core in core/
if len(parts) >= 3 and parts[0] == "lib":
return f"{_COMPONENT_PREFIX_LIB}{parts[1].rsplit('__', 1)[-1].lower()}"
if len(parts) >= 2 and parts[0] == "core":
return _COMPONENT_ARDUINO_CORE
# Other framework/library files - return the first path component
# e.g., FrameworkArduino/... -> FrameworkArduino
return parts[0] if parts else source_file
@@ -1191,9 +1443,7 @@ class MemoryAnalyzer:
and other_mem is not None
):
other_mem.add_section_size(section_name, -size)
if component not in self.components:
self.components[component] = ComponentMemory(component)
self.components[component].add_section_size(section_name, size)
self._add_component_size(component, section_name, size)
# Sort by size descending
self._cswtch_symbols.sort(key=lambda x: x[1], reverse=True)
@@ -1205,6 +1455,254 @@ class MemoryAnalyzer:
total_size,
)
def _analyze_object_rodata(self) -> None:
"""Attribute .rodata without a symbol, which is RAM on ESP8266, to components."""
if not self.rodata_in_ram or (obj_dir := self._find_object_files_dir()) is None:
return
result = run_tool(
[self.readelf_path, "-SW", "-sW"]
+ [str(f) for f in sorted(obj_dir.rglob("*.o"))]
)
if result is None or result.returncode != 0:
_LOGGER.debug("readelf failed scanning object files for constants")
return
self._attribute_unnamed_constants(result.stdout, obj_dir)
self._attribute_string_literals(result.stdout, obj_dir)
names = {n for c in self._unnamed_constants for n in c.functions} | {
n
for literal in self._string_literals
for users in literal.users.values()
for n in users
}
self._batch_demangle_symbols(sorted(names - self._demangle_cache.keys()))
def _attribute_unnamed_constants(self, output: str, obj_dir: Path) -> None:
"""Attribute constants GCC copies into objects built from constant initializers."""
unnamed, definitions = _parse_unnamed_rodata(output)
# Inline (weak) functions are emitted, with their constants, in every
# object that uses them; the linker keeps one copy
inline_copies: dict[tuple[tuple[str, ...], int], str] = {}
for file_path, sections in unnamed.items():
rel_path = Path(file_path).relative_to(obj_dir).as_posix()
component = self._source_file_to_component(rel_path)
for constant in self._scan_object_constants(
file_path, sections, rel_path, component
):
if definitions.linked_copy_unknown(file_path, constant.functions):
continue
if definitions.weak[file_path].issuperset(constant.functions):
key = (tuple(constant.functions), constant.size)
if inline_copies.setdefault(key, file_path) != file_path:
continue
self._unnamed_constants.append(constant)
self._charge_ram(component, constant.size)
self._unnamed_constants.sort(key=lambda c: c.size, reverse=True)
def _attribute_string_literals(self, output: str, obj_dir: Path) -> None:
"""Attribute string literals in RAM to the components whose code uses them.
The linker merges identical literals across objects, so each is counted
once, from the bytes it really takes in the ELF's .rodata.
"""
result = run_tool([self.readelf_path, "-x", ".rodata", str(self.elf_path)])
if result is None or result.returncode != 0:
return
rodata = bytes(_parse_hex_dumps(result.stdout).get(".rodata", b""))
# Literal -> component -> functions or data using it
users: dict[bytes, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
for file_path, sections in _parse_string_sections(output).items():
component = self._source_file_to_component(
Path(file_path).relative_to(obj_dir).as_posix()
)
for text, found in self._scan_object_strings(file_path, sections).items():
if text and (kept := found & self._elf_symbol_names):
users[text][component] |= kept
literals = (
StringLiteral(
text.decode("utf-8", "backslashreplace"),
_ram_string_size(text, rodata, users),
by_component,
)
for text, by_component in users.items()
)
self._string_literals = sorted(
(s for s in literals if s.size), key=lambda s: (-s.size, s.text)
)
for literal in self._string_literals:
# A literal used by several components is not charged to any of them
if len(literal.users) == 1:
self._charge_ram(next(iter(literal.users)), literal.size)
def _scan_object_strings(
self, file_path: str, sections: dict[str, tuple[str, str | None]]
) -> dict[bytes, set[str]]:
"""Map each string literal of one object to the functions or data using it.
Args:
file_path: Object file to read
sections: String sections by index, with the function each is named after
"""
result = run_tool(
[self.readelf_path, "-rW"]
+ [arg for index in sections for arg in ("-x", index)]
+ [file_path]
)
if result is None or result.returncode != 0:
_LOGGER.debug("readelf failed reading strings of %s", file_path)
return {}
contents = _parse_hex_dumps(result.stdout)
users: dict[bytes, set[str]] = defaultdict(set)
shared: set[str] = set()
for name, function in sections.values():
# A section named after its function holds only that function's literals
if function:
for text in bytes(contents[name]).split(b"\0")[:-1]:
users[text].add(function)
else:
shared.add(name)
relocations = _parse_relocations(result.stdout)
# Functions using each unnamed table, for literals reached through a pointer in it
table_users: dict[str, set[str]] = defaultdict(set)
for section, _, target, _ in relocations:
if (
target.startswith(".rodata")
and not _STRING_SECTION_PATTERN.match(target)
and (function := _section_user(section))
and _CODE_SECTION_PATTERN.match(section)
):
table_users[target].add(function)
# Literals in the shared section are found from the pointers that use them
refs = [
(section, found, offset, target, addend)
for section, offset, target, addend in relocations
if target in shared
and (
found := {u} if (u := _section_user(section)) else table_users[section]
)
]
if not refs:
return users
result = run_tool(
[self.readelf_path]
+ [arg for name in sorted({r[0] for r in refs}) for arg in ("-x", name)]
+ [file_path]
)
if result is None or result.returncode != 0:
_LOGGER.debug("readelf failed reading string users of %s", file_path)
return users
referrers = _parse_hex_dumps(result.stdout)
for section, found, offset, target, addend in refs:
text = _pointed_literal(
referrers[section], offset, addend, contents[target]
)
users[text] |= found
return users
def _scan_object_constants(
self,
file_path: str,
sections: dict[str, _RodataSection],
rel_path: str,
component: str,
) -> list[UnnamedConstant]:
"""Return the unnamed constants of one object that the link kept.
Args:
file_path: Object file to read
sections: Candidate sections by index
rel_path: Object path shown in the report
component: Component the object belongs to
"""
result = run_tool(
[self.readelf_path, "-rW"]
+ [arg for index in sections for arg in ("-x", index)]
+ [file_path]
)
if result is None or result.returncode != 0:
_LOGGER.debug("readelf failed reading constants of %s", file_path)
return []
by_name = {section.name: section for section in sections.values()}
contents = _parse_hex_dumps(result.stdout)
functions: dict[str, set[str]] = defaultdict(set)
jump_tables: set[str] = set()
# Section -> (offset, string section, addend) of each pointer to a literal
pointers: dict[str, list[tuple[int, str, int]]] = defaultdict(list)
for section, offset, target, addend in _parse_relocations(result.stdout):
if target in by_name:
code = _CODE_SECTION_PATTERN.match(section)
if code and code.group(1):
functions[target].add(code.group(1))
elif section in by_name and by_name[section].is_unnamed(offset):
if _CODE_SECTION_PATTERN.match(target):
jump_tables.add(section)
elif ".str" in target:
pointers[section].append((offset, target, addend))
strings = self._read_pointed_strings(file_path, pointers, contents)
return [
UnnamedConstant(
size=section.unnamed_size,
source_file=rel_path,
component=component,
functions=sorted(functions[name]),
jump_table=name in jump_tables,
data=bytes(
byte
for offset, byte in enumerate(contents[name])
if section.is_unnamed(offset)
)[:16],
strings=strings.get(name, []),
)
for name, section in by_name.items()
# Dropped by --gc-sections unless a function that uses it was kept
if not self._elf_symbol_names.isdisjoint(functions[name])
]
def _read_pointed_strings(
self,
file_path: str,
pointers: dict[str, list[tuple[int, str, int]]],
contents: dict[str, bytearray],
) -> dict[str, list[str]]:
"""Resolve each constant's pointers to string literals, in offset order."""
if not pointers:
return {}
targets = sorted({t for refs in pointers.values() for _, t, _ in refs})
result = run_tool(
[self.readelf_path]
+ [arg for target in targets for arg in ("-x", target)]
+ [file_path]
)
if result is None or result.returncode != 0:
_LOGGER.debug("readelf failed reading strings of %s", file_path)
return {}
literals = _parse_hex_dumps(result.stdout)
return {
name: [
_pointed_literal(
contents[name], offset, addend, literals[target]
).decode("utf-8", "backslashreplace")
for offset, target, addend in sorted(refs)
]
for name, refs in pointers.items()
}
def _charge_ram(self, component: str, size: int) -> None:
"""Charge RAM bytes that have no ELF symbol to a component."""
self._add_component_size(component, ".data", size)
if (data_section := self.sections.get(".data")) is not None:
data_section.symbol_size += size
def _add_component_size(self, component: str, section: str, size: int) -> None:
"""Add size to a component, creating it under its own name."""
if component not in self.components:
self.components[component] = ComponentMemory(component)
self.components[component].add_section_size(section, size)
def _analyze_function_calls(self) -> None:
"""Count function call sites by parsing disassembly output.
+84 -2
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
from collections import defaultdict
from collections.abc import Callable
import heapq
import json
from operator import itemgetter
from pathlib import Path
import sys
@@ -23,7 +24,7 @@ from . import (
from .toolchain import find_elf_path, find_idedata_path, idedata_candidates
if TYPE_CHECKING:
from . import ComponentMemory
from . import ComponentMemory, StringLiteral
def _format_pstorage_name(name: str) -> str:
@@ -46,6 +47,8 @@ class MemoryAnalyzerCLI(MemoryAnalyzer):
)
# Lower threshold for RAM symbols (RAM is more constrained)
RAM_SYMBOL_SIZE_THRESHOLD: int = 24
# String literals listed per component in the RAM strings report
STRINGS_PER_COMPONENT: int = 10
# Number of top symbols to show in the largest symbols report
TOP_SYMBOLS_LIMIT: int = 30
# Width for symbol name display in top symbols report
@@ -204,6 +207,75 @@ class MemoryAnalyzerCLI(MemoryAnalyzer):
f"{i + 1:>2}. {size:>7,} B {section_label:<8} {demangled_display:<{self.COL_TOP_SYMBOL_NAME}} {component}"
)
def _add_unnamed_constants_analysis(self, lines: list[str]) -> None:
"""Add the unnamed constants (ESP8266 .rodata in RAM) section."""
self._add_section_header(lines, "Unnamed Constants in RAM (.rodata)")
total_size = sum(c.size for c in self._unnamed_constants)
lines.append(
f"Total: {len(self._unnamed_constants)} constant(s), {total_size:,} B. "
"GCC copies these into objects built from constant initializers;"
)
lines.append(
"PROGMEM tables, assigned fields or a user-provided constructor avoid the copy."
)
lines.append("")
for constant in self._unnamed_constants:
kind = "switch table" if constant.jump_table else "constant"
lines.append(
f"{constant.size:>6,} B {kind:<12} {constant.component} ({constant.source_file})"
)
lines.extend(
f"{'':>9} used by {self._demangle_symbol(function)}"
for function in constant.functions
)
if constant.strings:
shown = ", ".join(f'"{text[:40]}"' for text in constant.strings[:4])
more = ", ..." if len(constant.strings) > 4 else ""
lines.append(f"{'':>9} points to {shown}{more}")
elif constant.data and not constant.jump_table:
more = " ..." if constant.size > len(constant.data) else ""
lines.append(f"{'':>9} bytes: {constant.data.hex(' ')}{more}")
lines.append("")
def _add_string_literals_analysis(self, lines: list[str]) -> None:
"""Add the string literals in RAM (ESP8266 .rodata) section, by component."""
self._add_section_header(lines, "String Literals in RAM (.rodata)")
total_size = sum(s.size for s in self._string_literals)
lines.append(
f"Total: {len(self._string_literals)} string(s), {total_size:,} B. "
"LOG_STR, ESPHOME_F or a PROGMEM table keeps a literal in flash."
)
lines.append(
"A literal used by several components is listed under each and charged to none."
)
lines.append("")
by_component: dict[str, list[StringLiteral]] = defaultdict(list)
for literal in self._string_literals:
for component in literal.users:
by_component[component].append(literal)
totals = {c: sum(s.size for s in ls) for c, ls in by_component.items()}
for component, literals in sorted(
by_component.items(), key=lambda x: (-totals[x[0]], x[0])
):
comp_total = totals[component]
lines.append(f"{component} ({comp_total:,} B, {len(literals)} strings):")
for literal in literals[: self.STRINGS_PER_COMPONENT]:
text = json.dumps(literal.text, ensure_ascii=False)
if len(text) > 50:
text = f'{text[:47]}..."'
users = sorted(
self._demangle_symbol(u) for u in literal.users[component]
)
used_by = users[0][:60] + (
f" (+{len(users) - 1})" if len(users) > 1 else ""
)
lines.append(f" {literal.size:>5,} B {text:<50} {used_by}")
if len(literals) > self.STRINGS_PER_COMPONENT:
lines.append(
f" ... and {len(literals) - self.STRINGS_PER_COMPONENT} more"
)
lines.append("")
def _add_cswtch_analysis(self, lines: list[str]) -> None:
"""Add CSWTCH (GCC switch table lookup) analysis section."""
self._add_section_header(lines, "CSWTCH Analysis (GCC Switch Table Lookups)")
@@ -394,6 +466,12 @@ class MemoryAnalyzerCLI(MemoryAnalyzer):
f"{total_flash:>{self.COL_TOTAL_FLASH - 2},} B | {total_ram:>{self.COL_TOTAL_RAM - 2},} B"
)
if self.rodata_in_ram:
lines.append("")
lines.append(
"ESP8266 keeps .rodata in RAM, so constants are counted in RAM (data)."
)
# Show unattributed RAM (SDK/framework overhead)
unattributed_bss, unattributed_data, unattributed_total = (
self.get_unattributed_ram()
@@ -657,6 +735,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer):
lines.append(f" ... and {len(large_ram_syms) - 10} more")
lines.append("")
if self._unnamed_constants:
self._add_unnamed_constants_analysis(lines)
if self._string_literals:
self._add_string_literals_analysis(lines)
# CSWTCH (GCC switch table) analysis
if self._cswtch_symbols:
self._add_cswtch_analysis(lines)
@@ -738,7 +821,6 @@ def main():
build_dir = sys.argv[1]
# Load build directory
import json
from esphome.platformio.toolchain import IDEData
+22 -3
View File
@@ -1,15 +1,34 @@
"""Tests for the memory analyzer."""
import subprocess
from unittest.mock import patch
from esphome.analyze_memory import MemoryAnalyzer
def make_analyzer(external_components: set[str] | None = None) -> MemoryAnalyzer:
"""Create a MemoryAnalyzer through its real constructor, without an ELF or a toolchain."""
def make_analyzer[T: MemoryAnalyzer](
external_components: set[str] | None = None, cls: type[T] = MemoryAnalyzer
) -> T:
"""Create an analyzer through its real constructor, without an ELF or a toolchain."""
with (
patch("esphome.analyze_memory.resolve_tool_path", return_value=None),
patch("esphome.analyze_memory.find_tool", return_value="nm"),
):
# Any existing file passes the ELF check; nothing reads it
return MemoryAnalyzer(__file__, external_components=external_components)
return cls(__file__, external_components=external_components)
def completed(stdout: str) -> subprocess.CompletedProcess[str]:
"""A successful tool run printing ``stdout``."""
return subprocess.CompletedProcess([], 0, stdout=stdout)
def hex_dump(section: str, data: bytes, address: int = 0) -> str:
"""Format bytes the way ``readelf -x`` does."""
lines = [f"Hex dump of section '{section}':"]
for start in range(0, len(data), 16):
chunk = data[start : start + 16]
words = " ".join(chunk[i : i + 4].hex() for i in range(0, len(chunk), 4))
text = "".join(chr(b) if 32 <= b < 127 else "." for b in chunk)
lines.append(f" 0x{address + start:08x} {words:<35} {text}")
return "\n".join(lines) + "\n\n"
@@ -1,5 +1,7 @@
"""Tests for source-file-to-component attribution in memory analyzer."""
import pytest
from . import make_analyzer
@@ -30,3 +32,19 @@ def test_source_file_to_component_known_component() -> None:
"src/esphome/components/wifi/wifi_component.cpp.o"
)
assert result == "[esphome]wifi"
@pytest.mark.parametrize(
("rel_path", "component"),
[
(
"lib/esp32async__ESPAsyncWebServer/WebHandlers.cpp.o",
"[lib]espasyncwebserver",
),
("lib/ESP8266WiFi/ESP8266WiFi.cpp.o", "[lib]esp8266wifi"),
("core/core_esp8266_main.cpp.o", "arduino_core"),
],
)
def test_native_library_objects(rel_path: str, component: str) -> None:
"""Objects of the native ESP8266 build's libraries and core get a component."""
assert make_analyzer()._source_file_to_component(rel_path) == component
@@ -0,0 +1,235 @@
"""Tests for attributing ESP8266 string literals in RAM to components."""
from pathlib import Path
import re
import subprocess
from unittest.mock import patch
import pytest
from esphome.analyze_memory import (
MemorySection,
StringLiteral,
_parse_string_sections,
_ram_string_size,
)
from esphome.analyze_memory.cli import MemoryAnalyzerCLI
from . import completed, hex_dump, make_analyzer
WEB_OBJ = "src/esphome/components/web_server/web_server.cpp.o"
LIGHT_OBJ = "src/esphome/components/light/light_json_schema.cpp.o"
COVER_JSON = "_ZN7esphome10web_server9WebServer11cover_json_Ev"
MATCH_URL = "_ZN7esphome10web_serverL9match_urlEPKcjbb"
DUMP_JSON = "_ZN7esphome5light15LightJSONSchema9dump_jsonEv"
_SECTIONS = """\
File: {web}
Section Headers:
[Nr] Name Type Addr Off Size ES Flg Lk Inf Al
[ 7] .rodata PROGBITS 00000000 000040 000008 00 A 0 0 4
[209] .rodata.str1.1 PROGBITS 00000000 000598 00000a 01 AMS 0 0 1
[481] .rodata.{cover_json}.str1.1 PROGBITS 00000000 003910 00000c 01 AMS 0 0 1
File: {light}
Section Headers:
[Nr] Name Type Addr Off Size ES Flg Lk Inf Al
[ 30] .rodata.{dump_json}.str1.1 PROGBITS 00000000 000100 000007 01 AMS 0 0 1
"""
def _sections(web: str, light: str) -> str:
"""``readelf -SW`` output for the two objects."""
return _SECTIONS.format(
web=web, light=light, cover_json=COVER_JSON, dump_json=DUMP_JSON
)
# "OPEN\0CLOSED\0" for cover_json_, "ON\0all\0none\0" shared by match_url
WEB_STRINGS = (
f"""\
Relocation section '.rela.text.{MATCH_URL}' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000004 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
"""
+ hex_dump(".rodata.str1.1", b"ON\0all\0none\0")
+ hex_dump(f".rodata.{COVER_JSON}.str1.1", b"OPEN\0CLOSED\0")
)
# The literal pool word of match_url holds the literal's offset, 3 ("all")
WEB_REFERRERS = hex_dump(f".text.{MATCH_URL}", bytes(4) + (3).to_bytes(4, "little"))
LIGHT_STRINGS = hex_dump(f".rodata.{DUMP_JSON}.str1.1", b"ON\0OFF\0")
# Linked .rodata: "OPEN" and "ON" were dropped
ELF_RODATA = hex_dump(".rodata", b"OFF\0CLOSED\0all\0", 0x3FFE86B0)
def test_parse_string_sections() -> None:
"""Only string literal sections are listed, named after their function or not."""
assert _parse_string_sections(_sections("web.o", "light.o")) == {
"web.o": {
"209": (".rodata.str1.1", None),
"481": (f".rodata.{COVER_JSON}.str1.1", COVER_JSON),
},
"light.o": {"30": (f".rodata.{DUMP_JSON}.str1.1", DUMP_JSON)},
}
@pytest.mark.parametrize(
("text", "expected"),
[
(b"OFF", 4),
(b"MON", 4),
# Stored only as the tail of "MON"
(b"ON", 0),
(b"missing", 0),
# After a float constant ending in 0x3f, not inside another literal
(b"all", 4),
],
)
def test_ram_string_size(text: bytes, expected: int) -> None:
"""A literal stored only as the tail of a longer one, or dropped, takes no RAM."""
rodata = b"MON\0OFF\0CLOSED\0\x00\x00\x80\x3fall\0"
literals = {b"MON", b"OFF", b"ON", b"CLOSED", b"all", b"missing"}
assert _ram_string_size(text, rodata, literals) == expected
@pytest.fixture
def obj_dir(tmp_path: Path) -> Path:
for rel in (WEB_OBJ, LIGHT_OBJ, "src/main.cpp.o"):
path = tmp_path / "obj" / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()
return tmp_path / "obj"
def _new_analyzer(obj_dir: Path, kept: set[str]) -> MemoryAnalyzerCLI:
analyzer = make_analyzer(cls=MemoryAnalyzerCLI)
analyzer.elf_path = obj_dir.parent / "firmware.elf"
analyzer.rodata_in_ram = True
analyzer.sections[".data"] = MemorySection(".data", total_size=1000)
analyzer._elf_symbol_names = kept
# Unnamed constants have their own tests
analyzer._attribute_unnamed_constants = lambda *_: None
return analyzer
def _analyze(obj_dir: Path, kept: set[str]) -> MemoryAnalyzerCLI:
analyzer = _new_analyzer(obj_dir, kept)
light = str(obj_dir / LIGHT_OBJ)
sections = _sections(str(obj_dir / WEB_OBJ), light)
def run_tool(cmd: list[str]) -> subprocess.CompletedProcess[str]:
if "-SW" in cmd:
return completed(sections)
if cmd[-1].endswith("firmware.elf"):
return completed(ELF_RODATA)
if cmd[-1] == light:
return completed(LIGHT_STRINGS)
if "-rW" not in cmd:
assert cmd[1:-1] == ["-x", f".text.{MATCH_URL}"]
return completed(WEB_REFERRERS)
return completed(WEB_STRINGS)
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
analyzer._analyze_object_rodata()
return analyzer
def test_string_literals_attributed(obj_dir: Path) -> None:
"""Each literal is counted once, with the bytes it takes in the linked .rodata."""
analyzer = _analyze(obj_dir, {COVER_JSON, MATCH_URL, DUMP_JSON})
assert analyzer._string_literals == [
StringLiteral("CLOSED", 7, {"[esphome]web_server": {COVER_JSON}}),
StringLiteral("OFF", 4, {"[esphome]light": {DUMP_JSON}}),
StringLiteral("all", 4, {"[esphome]web_server": {MATCH_URL}}),
]
# "OPEN" and "ON" are not in the ELF
assert analyzer.components["[esphome]web_server"].data_size == 11
assert analyzer.components["[esphome]light"].data_size == 4
assert analyzer.sections[".data"].symbol_size == 15
def test_string_literals_skip_dropped_functions(obj_dir: Path) -> None:
"""Literals of functions the linker dropped are not charged."""
analyzer = _analyze(obj_dir, {DUMP_JSON})
assert [s.text for s in analyzer._string_literals] == ["OFF"]
def test_string_literals_shared_between_components(obj_dir: Path) -> None:
"""A literal two components use is listed under both and charged to neither."""
analyzer = _new_analyzer(obj_dir, {COVER_JSON, DUMP_JSON})
with (
patch.object(
analyzer,
"_scan_object_strings",
side_effect=[{b"OFF": {COVER_JSON}}, {b"OFF": {DUMP_JSON}}],
),
patch("esphome.analyze_memory.run_tool", return_value=completed(ELF_RODATA)),
):
analyzer._attribute_string_literals(
_sections(str(obj_dir / WEB_OBJ), str(obj_dir / LIGHT_OBJ)), obj_dir
)
(literal,) = analyzer._string_literals
assert literal.users == {
"[esphome]web_server": {COVER_JSON},
"[esphome]light": {DUMP_JSON},
}
assert analyzer.sections[".data"].symbol_size == 0
analyzer._demangle_cache = {COVER_JSON: "cover_json_()", DUMP_JSON: "dump_json()"}
report = analyzer.generate_report()
# Each component's row names its own user
assert re.search(r'"OFF" +cover_json_\(\)', report)
assert re.search(r'"OFF" +dump_json\(\)', report)
def test_report_groups_by_component() -> None:
"""Strings are listed by component with the function that uses them."""
analyzer = make_analyzer(cls=MemoryAnalyzerCLI)
analyzer._demangle_cache = {
"setup": "setup",
COVER_JSON: "WebServer::cover_json_()",
}
analyzer._string_literals = [
StringLiteral("hunter2", 8, {"[esphome]core": {"setup"}}),
StringLiteral("CLOSED", 7, {"[esphome]web_server": {COVER_JSON}}),
]
report = analyzer.generate_report()
assert "String Literals in RAM (.rodata)" in report
assert "[esphome]core (8 B, 1 strings):" in report
assert '"hunter2"' in report
assert '"CLOSED"' in report
assert "WebServer::cover_json_()" in report
def test_string_literals_through_unnamed_table() -> None:
"""Literals reached through an unnamed pointer table go to the table's users."""
analyzer = make_analyzer()
relocations = """\
Relocation section '.rela.text.setup' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
Relocation section '.rela.rodata' at offset 0xe6fc contains 2 entries:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
00000004 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
""" + hex_dump(".rodata.str1.1", b"0.pool.ntp.org\0001.pool.ntp.org\0")
table = hex_dump(".rodata", (0).to_bytes(4, "little") + (15).to_bytes(4, "little"))
def run_tool(cmd: list[str]) -> subprocess.CompletedProcess[str]:
if "-rW" in cmd:
return completed(relocations)
assert cmd[1:-1] == ["-x", ".rodata"]
return completed(table)
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
users = analyzer._scan_object_strings(
"main.cpp.o", {"5": (".rodata.str1.1", None)}
)
assert users == {b"0.pool.ntp.org": {"setup"}, b"1.pool.ntp.org": {"setup"}}
@@ -0,0 +1,477 @@
"""Tests for ESP8266 .rodata accounting and unnamed constant attribution."""
from collections.abc import Callable
from pathlib import Path
import subprocess
from unittest.mock import patch
import pytest
from esphome.analyze_memory import (
_CODE_SECTION_PATTERN,
MemoryAnalyzer,
MemorySection,
UnnamedConstant,
_FunctionDefinitions,
_parse_unnamed_rodata,
_RodataSection,
)
from esphome.analyze_memory.cli import MemoryAnalyzerCLI
from . import completed, hex_dump, make_analyzer
LIGHT_OBJ = "src/esphome/components/light/light_state.cpp.o"
WEB_OBJ = "src/esphome/components/web_server/web_server.cpp.o"
SETUP = "_ZN7esphome5light10LightState5setupEv"
SAVE = "_ZN7esphome5light10LightState19save_remote_values_Ev"
MATCH_URL = "_ZN7esphome10web_serverL9match_urlEPKcjbb"
# readelf -SW -sW over all objects; the web section index and size vary per test
HEADERS_AND_SYMBOLS = """\
File: {light}
Section Headers:
[Nr] Name Type Addr Off Size ES Flg Lk Inf Al
[124] .rodata._ZN7esphome5light10LightStateC2EPNS0_11LightOutputE.str1.1 PROGBITS 00000000 00067d 000001 01 AMS 0 0 1
[174] .rodata PROGBITS 00000000 00139c 00002c 00 A 0 0 4
[221] .rodata.str1.1 PROGBITS 00000000 001f39 000005 01 AMS 0 0 1
[225] .rodata.BUILTIN_LED PROGBITS 00000000 001f54 000004 00 A 0 0 4
Symbol table '.symtab' contains 348 entries:
Num: Value Size Type Bind Vis Ndx Name
0: 00000000 0 NOTYPE LOCAL DEFAULT UND
300: 00000000 64 FUNC {bind:<6} DEFAULT 12 {setup}
301: 00000000 64 FUNC {bind:<6} DEFAULT 13 {save}
347: 00000000 4 OBJECT WEAK DEFAULT 225 BUILTIN_LED
File: {web}
Section Headers:
[Nr] Name Type Addr Off Size ES Flg Lk Inf Al
[{web_index:>3}] .rodata PROGBITS 00000000 000400 {web_size:06x} 00 A 0 0 4
Symbol table '.symtab' contains 1 entries:
Num: Value Size Type Bind Vis Ndx Name
0: 00000000 0 NOTYPE LOCAL DEFAULT UND
{web_symbols}"""
LIGHT_OBJECT_DUMP = f"""\
Relocation section '.rela.text.{SETUP}' at offset 0x7928 contains 2 entries:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
00000094 00003b14 R_XTENSA_SLOT0_OP 00000000 .text.{SETUP} + 82
Relocation section '.rela.text.{SAVE}' at offset 0x7d28 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
Relocation section '.rela.xt.prop' at offset 0x8000 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
000007d4 00003a01 R_XTENSA_32 00000000 .rodata + 0
Hex dump of section '.rodata':
0x00000000 0000803f 0000803f 0000803f 0000803f ...?...?...?...?
0x00000010 0000803f 0000803f 0000803f 0000803f ...?...?...?...?
0x00000020 0000803f 00000000 00000000 ...?........
"""
WEB_OBJECT_DUMP = f"""\
Relocation section '.rela.text.{MATCH_URL}' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
Hex dump of section '.rodata':
0x00000000 00000000 00000000 00000000 00000000 ................
"""
@pytest.fixture
def obj_dir(tmp_path: Path) -> Path:
"""Native ESP8266 build layout: objects under obj/, ELF next to it."""
for rel in (LIGHT_OBJ, WEB_OBJ, "src/main.cpp.o"):
path = tmp_path / "obj" / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()
return tmp_path / "obj"
def _esp8266_analyzer(
obj_dir: Path,
kept: set[str],
web_index: int = 9,
web_size: int = 0x1C,
web_dump: str = WEB_OBJECT_DUMP,
bind: str = "GLOBAL",
web_symbols: str = "",
) -> tuple[MemoryAnalyzer, Callable[..., subprocess.CompletedProcess[str]]]:
analyzer = make_analyzer()
analyzer.elf_path = obj_dir.parent / "firmware.elf"
analyzer.rodata_in_ram = True
analyzer.sections[".data"] = MemorySection(".data", total_size=1000)
analyzer._elf_symbol_names = kept
# String literals have their own tests
analyzer._attribute_string_literals = lambda *_: None
light = str(obj_dir / LIGHT_OBJ)
web = str(obj_dir / WEB_OBJ)
def run_tool(cmd: list[str]) -> subprocess.CompletedProcess[str]:
if "-SW" in cmd:
return completed(
HEADERS_AND_SYMBOLS.format(
light=light,
web=web,
web_index=web_index,
web_size=web_size,
bind=bind,
setup=SETUP,
save=SAVE,
web_symbols=web_symbols,
)
)
# Only sections with uncovered bytes are dumped
if cmd[-1] == light:
assert cmd[1:-1] == ["-rW", "-x", "174"]
return completed(LIGHT_OBJECT_DUMP)
return completed(web_dump)
return analyzer, run_tool
def test_find_object_files_dir_native_layout(obj_dir: Path) -> None:
"""The native ESP8266 build keeps objects under obj/."""
analyzer = make_analyzer()
analyzer.elf_path = obj_dir.parent / "firmware.elf"
assert analyzer._find_object_files_dir() == obj_dir
def test_find_object_files_dir_platformio_layout(tmp_path: Path) -> None:
"""PlatformIO keeps objects next to the ELF."""
(tmp_path / "src").mkdir()
(tmp_path / "src" / "main.cpp.o").touch()
analyzer = make_analyzer()
analyzer.elf_path = tmp_path / "firmware.elf"
assert analyzer._find_object_files_dir() == tmp_path
def _parse_sections(code_section: str) -> MemoryAnalyzer:
readelf = f"""\
[ 1] .data PROGBITS 3ffe8000 001000 000668 00 WAX 0 0 16
[ 4] {code_section} PROGBITS 40201010 00b010 06612c 00 WAX 0 0 16
[ 6] .rodata PROGBITS 3ffe86b0 0016b0 000b90 00 A 0 0 16
"""
analyzer = make_analyzer()
with patch(
"esphome.analyze_memory.subprocess.run", return_value=completed(readelf)
):
analyzer._parse_sections()
return analyzer
def test_parse_sections_esp8266_counts_rodata_as_data() -> None:
"""ESP8266 (.irom0.text) copies .rodata to RAM, so it is counted as .data."""
analyzer = _parse_sections(".irom0.text")
assert analyzer.rodata_in_ram
assert ".rodata" not in analyzer.sections
assert analyzer.sections[".data"].total_size == 0x668 + 0xB90
def test_parse_sections_other_platforms_keep_rodata() -> None:
"""Other platforms keep .rodata in flash."""
analyzer = _parse_sections(".flash.text")
assert not analyzer.rodata_in_ram
assert analyzer.sections[".rodata"].total_size == 0xB90
assert analyzer.sections[".data"].total_size == 0x668
def test_unnamed_constants_attributed_to_component(obj_dir: Path) -> None:
"""Unnamed .rodata in a kept function is listed and counted as the component's RAM."""
analyzer, run_tool = _esp8266_analyzer(obj_dir, {SETUP, MATCH_URL})
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
analyzer._analyze_object_rodata()
light, web = analyzer._unnamed_constants
assert light == UnnamedConstant(
size=44,
source_file=LIGHT_OBJ,
component="[esphome]light",
functions=[SAVE, SETUP],
jump_table=False,
data=bytes.fromhex("0000803f") * 4,
)
assert (web.size, web.component, web.functions) == (
28,
"[esphome]web_server",
[MATCH_URL],
)
assert analyzer.components["[esphome]light"].data_size == 44
assert analyzer.components["[esphome]web_server"].ram_total == 28
# Attributed bytes no longer count as unattributed RAM
assert analyzer.sections[".data"].symbol_size == 72
def test_unnamed_constants_skip_garbage_collected(obj_dir: Path) -> None:
"""A constant whose functions the linker dropped is not in the firmware."""
analyzer, run_tool = _esp8266_analyzer(obj_dir, {MATCH_URL})
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
analyzer._analyze_object_rodata()
assert [c.component for c in analyzer._unnamed_constants] == ["[esphome]web_server"]
def test_unnamed_constants_skipped_when_rodata_in_flash(obj_dir: Path) -> None:
"""Other platforms keep .rodata in flash, so there is nothing to scan."""
analyzer, run_tool = _esp8266_analyzer(obj_dir, {SETUP, MATCH_URL})
analyzer.rodata_in_ram = False
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool) as mock:
analyzer._analyze_object_rodata()
mock.assert_not_called()
assert analyzer._unnamed_constants == []
@pytest.mark.parametrize(("bind", "expected"), [("WEAK", 1), ("GLOBAL", 2)])
def test_unnamed_constants_inline_copies_counted_once(
obj_dir: Path, bind: str, expected: int
) -> None:
"""Only weak (inline) functions bring the same constant from several objects."""
# The second object emits the same functions, with the same binding
analyzer, run_tool = _esp8266_analyzer(
obj_dir,
{SETUP},
web_index=174,
web_size=0x2C,
web_dump=LIGHT_OBJECT_DUMP,
bind=bind,
web_symbols=(
f" 300: 00000000 64 FUNC {bind:<6} DEFAULT 12 {SETUP}\n"
f" 301: 00000000 64 FUNC {bind:<6} DEFAULT 13 {SAVE}\n"
),
)
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
analyzer._analyze_object_rodata()
assert len(analyzer._unnamed_constants) == expected
assert analyzer.sections[".data"].symbol_size == 44 * expected
def test_parse_symbols_keeps_alias_names() -> None:
"""C1/C2 constructor aliases share an address; both names are in the ELF."""
objdump = """4020b558 g F .irom0.text\t0000007e _ZN7esphome3FooC1Ev
4020b558 g F .irom0.text\t0000007e _ZN7esphome3FooC2Ev
"""
analyzer = make_analyzer()
analyzer.sections[".text"] = MemorySection(".text")
with patch(
"esphome.analyze_memory.subprocess.run", return_value=completed(objdump)
):
analyzer._parse_symbols()
assert {"_ZN7esphome3FooC1Ev", "_ZN7esphome3FooC2Ev"} <= analyzer._elf_symbol_names
assert analyzer.sections[".text"].symbol_size == 0x7E
def test_unnamed_constants_skip_named_bytes() -> None:
"""In a section shared with a named object, only the unnamed bytes are shown."""
analyzer = make_analyzer()
analyzer._elf_symbol_names = {"_Z3fooi"}
dump = """Relocation section '.rela.text._Z3fooi' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 4
Hex dump of section '.rodata':
0x00000000 aaaaaaaa 01020304 ........
"""
section = _RodataSection(".rodata", 8, covered=[(0, 4)])
with patch("esphome.analyze_memory.run_tool", return_value=completed(dump)):
(constant,) = analyzer._scan_object_constants(
"foo.o", {"5": section}, "src/foo.o", "[esphome]foo"
)
assert constant.size == 4
assert constant.data == bytes([1, 2, 3, 4])
def test_unnamed_constants_jump_table() -> None:
"""A .rodata section that points into code is a switch jump table."""
analyzer = make_analyzer()
analyzer._elf_symbol_names = {"_Z3fooi"}
dump = """\
Relocation section '.rela.text._Z3fooi' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
Relocation section '.rela.rodata' at offset 0x200 contains 2 entries:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003b01 R_XTENSA_32 00000000 .text._Z3fooi + 12
00000004 00003b01 R_XTENSA_32 00000000 .text._Z3fooi + 20
"""
with patch("esphome.analyze_memory.run_tool", return_value=completed(dump)):
(constant,) = analyzer._scan_object_constants(
"foo.o", {"5": _RodataSection(".rodata", 8)}, "src/foo.o", "[esphome]foo"
)
assert constant.jump_table
assert constant.functions == ["_Z3fooi"]
def test_report_lists_unnamed_constants() -> None:
"""The report names the component, the object, the functions and the bytes."""
analyzer = make_analyzer(cls=MemoryAnalyzerCLI)
analyzer.rodata_in_ram = True
analyzer._demangle_cache = {SETUP: "esphome::light::LightState::setup()"}
analyzer._unnamed_constants = [
UnnamedConstant(
size=44,
source_file=LIGHT_OBJ,
component="[esphome]light",
functions=[SETUP],
jump_table=False,
data=bytes.fromhex("0000803f") * 4,
)
]
report = analyzer.generate_report()
assert "ESP8266 keeps .rodata in RAM" in report
assert "Unnamed Constants in RAM (.rodata)" in report
assert f"44 B constant [esphome]light ({LIGHT_OBJ})" in report
assert "used by esphome::light::LightState::setup()" in report
assert "bytes: 00 00 80 3f 00 00 80 3f 00 00 80 3f 00 00 80 3f ..." in report
def test_unnamed_constants_resolve_string_pointers() -> None:
"""A constant of pointers to literals lists the strings, so it can be recognized."""
analyzer = make_analyzer()
analyzer._elf_symbol_names = {"setup"}
relocations = """\
Relocation section '.rela.text.setup' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 0
Relocation section '.rela.rodata' at offset 0xe6fc contains 3 entries:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
00000004 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
00000008 00000501 R_XTENSA_32 00000000 .rodata.str1.1 + 0
Hex dump of section '.rodata':
0x00000000 00000000 0f000000 1e000000 ............
"""
literals = hex_dump(
".rodata.str1.1", b"0.pool.ntp.org\0001.pool.ntp.org\0002.pool.ntp.org\0"
)
def run_tool(cmd: list[str]) -> subprocess.CompletedProcess[str]:
if "-rW" not in cmd:
assert cmd[1:-1] == ["-x", ".rodata.str1.1"]
return completed(literals)
return completed(relocations)
with patch("esphome.analyze_memory.run_tool", side_effect=run_tool):
(constant,) = analyzer._scan_object_constants(
"main.cpp.o",
{"5": _RodataSection(".rodata", 12)},
"src/main.cpp.o",
"[esphome]core",
)
assert constant.strings == ["0.pool.ntp.org", "1.pool.ntp.org", "2.pool.ntp.org"]
cli = make_analyzer(cls=MemoryAnalyzerCLI)
cli._unnamed_constants = [constant]
assert (
'points to "0.pool.ntp.org", "1.pool.ntp.org", "2.pool.ntp.org"'
in cli.generate_report()
)
@pytest.mark.parametrize(
("section", "function"),
[
(".text._Z3foov", "_Z3foov"),
(".literal._Z3foov", "_Z3foov"),
(".irom0.text._Z3foov", "_Z3foov"),
(".text.startup._GLOBAL__sub_I_setup", "_GLOBAL__sub_I_setup"),
(".text.unlikely._Z3foov", "_Z3foov"),
(".iram.text._Z3foov", "_Z3foov"),
(".text.hot_path", "hot_path"),
(".text", None),
],
)
def test_code_section_function(section: str, function: str | None) -> None:
"""Placement prefixes and IRAM code still resolve to the function."""
match = _CODE_SECTION_PATTERN.match(section)
assert match is not None
assert match.group(1) == function
def test_unnamed_constants_same_object_not_merged(obj_dir: Path) -> None:
"""Two equal constants of one inline function in one object are both counted."""
analyzer, run_tool = _esp8266_analyzer(obj_dir, {SETUP}, bind="WEAK")
constant = UnnamedConstant(
size=8,
source_file=LIGHT_OBJ,
component="[esphome]light",
functions=[SAVE, SETUP],
jump_table=False,
data=b"",
)
light = str(obj_dir / LIGHT_OBJ)
with (
patch("esphome.analyze_memory.run_tool", side_effect=run_tool),
patch.object(
analyzer,
"_scan_object_constants",
side_effect=lambda path, *_: [constant, constant] if path == light else [],
),
):
analyzer._analyze_object_rodata()
assert len(analyzer._unnamed_constants) == 2
def test_unnamed_constants_jump_table_only_from_unnamed_bytes() -> None:
"""Code pointers in a named table at the start do not make the rest a jump table."""
analyzer = make_analyzer()
analyzer._elf_symbol_names = {"_Z3fooi"}
dump = """\
Relocation section '.rela.text._Z3fooi' at offset 0x100 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003a01 R_XTENSA_32 00000000 .rodata + 4
Relocation section '.rela.rodata' at offset 0x200 contains 1 entry:
Offset Info Type Sym. Value Symbol's Name + Addend
00000000 00003b01 R_XTENSA_32 00000000 .text._Z3fooi + 12
"""
section = _RodataSection(".rodata", 8, covered=[(0, 4)])
with patch("esphome.analyze_memory.run_tool", return_value=completed(dump)):
(constant,) = analyzer._scan_object_constants(
"foo.o", {"5": section}, "src/foo.o", "[esphome]foo"
)
assert not constant.jump_table
def _definitions(rows: dict[str, list[tuple[str, str]]]) -> _FunctionDefinitions:
"""Parse symbol rows (binding, function) per object, as readelf -sW prints them."""
output = "".join(
f"File: {path}\n"
+ "".join(
f" {i}: 00000000 64 FUNC {bind:<6} DEFAULT 12 {name}\n"
for i, (bind, name) in enumerate(symbols)
)
for path, symbols in rows.items()
)
return _parse_unnamed_rodata(output)[1]
def test_linked_copy_unknown_weak_overridden_by_strong() -> None:
"""A strong definition replaces every weak copy, so their constants are not linked."""
definitions = _definitions(
{"a.o": [("WEAK", "_Z3foov")], "b.o": [("GLOBAL", "_Z3foov")]}
)
assert definitions.linked_copy_unknown("a.o", ["_Z3foov"])
assert not definitions.linked_copy_unknown("b.o", ["_Z3foov"])
def test_linked_copy_unknown_same_named_locals() -> None:
"""Local functions with one name in two objects can't be told apart in the ELF."""
definitions = _definitions(
{
"a.o": [("LOCAL", "_ZL3barv"), ("LOCAL", "_ZL3bazv")],
"b.o": [("LOCAL", "_ZL3barv")],
}
)
assert definitions.linked_copy_unknown("a.o", ["_ZL3barv"])
# Another, unique user decides it
assert not definitions.linked_copy_unknown("a.o", ["_ZL3barv", "_ZL3bazv"])