From 0f1ece6bdc2c390f6a027662545c8d78323c2c82 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 11:14:34 -0600 Subject: [PATCH 1/9] [analyze-memory] Attribute PlatformIO library symbols via nm archive scanning --- esphome/analyze_memory/__init__.py | 152 ++++++++++++++++++++++++++++- esphome/analyze_memory/cli.py | 14 ++- 2 files changed, 162 insertions(+), 4 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index d8abc8bafbc..5fa827795f8 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -43,6 +43,7 @@ _READELF_SECTION_PATTERN = re.compile( # Component category prefixes _COMPONENT_PREFIX_ESPHOME = "[esphome]" _COMPONENT_PREFIX_EXTERNAL = "[external]" +_COMPONENT_PREFIX_LIB = "[lib]" _COMPONENT_CORE = f"{_COMPONENT_PREFIX_ESPHOME}core" _COMPONENT_API = f"{_COMPONENT_PREFIX_ESPHOME}api" @@ -56,6 +57,9 @@ SymbolInfoType = tuple[str, int, str] # RAM sections - symbols in these sections consume RAM RAM_SECTIONS = frozenset([".data", ".bss"]) +# nm symbol types for global/weak defined symbols (used for library symbol mapping) +_NM_DEFINED_GLOBAL_TYPES = frozenset({"T", "D", "B", "R", "W", "V"}) + @dataclass class MemorySection: @@ -179,11 +183,16 @@ class MemoryAnalyzer: self._sdk_symbols: list[SDKSymbol] = [] # CSWTCH symbols: list of (name, size, source_file, component) self._cswtch_symbols: list[tuple[str, int, str, str]] = [] + # PlatformIO library symbol mapping: symbol_name -> library_name + self._lib_symbol_map: dict[str, str] = {} + # PlatformIO library hash to name mapping: "lib641" -> "espsoftwareserial" + self._lib_hash_to_name: dict[str, str] = {} def analyze(self) -> dict[str, ComponentMemory]: """Analyze the ELF file and return component memory usage.""" self._parse_sections() self._parse_symbols() + self._scan_pio_libraries() self._categorize_symbols() self._analyze_cswtch_symbols() self._analyze_sdk_libraries() @@ -328,6 +337,10 @@ class MemoryAnalyzer: # If no component match found, it's core return _COMPONENT_CORE + # Check PlatformIO library symbol map (more accurate than heuristic patterns) + if lib_name := self._lib_symbol_map.get(symbol_name): + return f"{_COMPONENT_PREFIX_LIB}{lib_name}" + # Check against symbol patterns for component, patterns in SYMBOL_PATTERNS.items(): if any(pattern in symbol_name for pattern in patterns): @@ -384,6 +397,135 @@ class MemoryAnalyzer: return "Other Core" + def _discover_pio_libraries(self) -> dict[str, Path]: + """Discover PlatformIO third-party libraries from the build directory. + + Scans ``lib/`` directories under ``.pioenvs//`` to find + library names and their ``.a`` archive paths. + + Returns: + Dictionary mapping lowercase library name to ``.a`` file path. + """ + # The ELF is typically at .pioenvs//firmware.elf + build_dir = self.elf_path.parent + libraries: dict[str, Path] = {} + + for entry in build_dir.iterdir(): + if not entry.is_dir() or not entry.name.startswith("lib"): + continue + # Validate that the suffix after "lib" is a hex hash + hex_part = entry.name[3:] + if not hex_part: + continue + try: + int(hex_part, 16) + except ValueError: + continue + + # Each lib/ directory contains a subdirectory named after the library + # and a .a archive named lib.a + for lib_subdir in entry.iterdir(): + if not lib_subdir.is_dir(): + continue + # The .a file is named lib.a (case-insensitive match) + # e.g., lib72a/ESPAsyncTCP/... has lib72a/libESPAsyncTCP.a + archive = entry / f"lib{lib_subdir.name}.a" + if not archive.exists(): + # Try case-insensitive: scan for any .a file + archives = list(entry.glob("*.a")) + archive = archives[0] if archives else None + if archive and archive.exists(): + libraries[lib_subdir.name.lower()] = archive + _LOGGER.debug( + "Discovered PlatformIO library: %s -> %s", + lib_subdir.name, + archive, + ) + + return libraries + + def _build_library_symbol_map(self, libraries: dict[str, Path]) -> dict[str, str]: + """Build a symbol-to-library mapping from library archives. + + Runs ``nm --defined-only`` on each ``.a`` file to collect global and + weak defined symbols. + + Args: + libraries: Dictionary mapping library name to ``.a`` file path. + + Returns: + Dictionary mapping symbol name to library name. + """ + symbol_map: dict[str, str] = {} + + if not self.nm_path: + return symbol_map + + for lib_name, archive_path in libraries.items(): + result = run_tool( + [self.nm_path, "--defined-only", str(archive_path)], + timeout=10, + ) + if result is None or result.returncode != 0: + continue + + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) < 3: + continue + + sym_type = parts[-2] + sym_name = parts[-1] + + # Include global defined symbols (uppercase) and weak symbols (W/V) + if sym_type in _NM_DEFINED_GLOBAL_TYPES: + symbol_map[sym_name] = lib_name + + return symbol_map + + def _scan_pio_libraries(self) -> None: + """Discover PlatformIO libraries and build symbol mapping. + + Orchestrates library discovery, symbol map construction, and + hash-to-name mapping for CSWTCH attribution. + """ + libraries = self._discover_pio_libraries() + if not libraries: + _LOGGER.debug("No PlatformIO third-party libraries found") + return + + _LOGGER.info( + "Scanning %d PlatformIO libraries: %s", + len(libraries), + ", ".join(sorted(libraries)), + ) + + self._lib_symbol_map = self._build_library_symbol_map(libraries) + + # Build hash-to-name mapping for CSWTCH attribution + # e.g., lib641 -> espsoftwareserial + build_dir = self.elf_path.parent + for entry in build_dir.iterdir(): + if not entry.is_dir() or not entry.name.startswith("lib"): + continue + hex_part = entry.name[3:] + if not hex_part: + continue + try: + int(hex_part, 16) + except ValueError: + continue + for lib_subdir in entry.iterdir(): + if lib_subdir.is_dir(): + self._lib_hash_to_name[entry.name] = lib_subdir.name.lower() + break + + _LOGGER.info( + "Built library symbol map: %d symbols from %d libraries", + len(self._lib_symbol_map), + len(libraries), + ) + def _find_object_files_dir(self) -> Path | None: """Find the directory containing object files for this build. @@ -559,9 +701,13 @@ class MemoryAnalyzer: if "esphome" in parts and "components" not in parts: return _COMPONENT_CORE - # Framework/library files - return the first path component - # e.g., lib65b/ESPAsyncTCP/... -> lib65b - # FrameworkArduino/... -> FrameworkArduino + # Framework/library files - check for PlatformIO library hash dirs + # e.g., lib65b/ESPAsyncTCP/... -> [lib]espasynctcp + if parts and parts[0] in self._lib_hash_to_name: + return f"{_COMPONENT_PREFIX_LIB}{self._lib_hash_to_name[parts[0]]}" + + # Other framework/library files - return the first path component + # e.g., FrameworkArduino/... -> FrameworkArduino return parts[0] if parts else source_file def _analyze_cswtch_symbols(self) -> None: diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index bb0eb7723ee..dbc19c6b89d 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -14,6 +14,7 @@ from . import ( _COMPONENT_CORE, _COMPONENT_PREFIX_ESPHOME, _COMPONENT_PREFIX_EXTERNAL, + _COMPONENT_PREFIX_LIB, RAM_SECTIONS, MemoryAnalyzer, ) @@ -407,6 +408,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): for name, mem in components if name.startswith(_COMPONENT_PREFIX_EXTERNAL) ] + library_components = [ + (name, mem) + for name, mem in components + if name.startswith(_COMPONENT_PREFIX_LIB) + ] top_esphome_components = sorted( esphome_components, key=lambda x: x[1].flash_total, reverse=True @@ -417,6 +423,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): external_components, key=lambda x: x[1].flash_total, reverse=True ) + # Include all library components + top_library_components = sorted( + library_components, key=lambda x: x[1].flash_total, reverse=True + ) + # Check if API component exists and ensure it's included api_component = None for name, mem in components: @@ -435,10 +446,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): if name in system_components_to_include ] - # Combine all components to analyze: top ESPHome + all external + API if not already included + system components + # Combine all components to analyze: top ESPHome + all external + libraries + API if not already included + system components components_to_analyze = ( list(top_esphome_components) + list(top_external_components) + + list(top_library_components) + system_components ) if api_component and api_component not in components_to_analyze: From 95e92716d935c3bff0bc98ddb2f9137c93bfd3ee Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 11:19:38 -0600 Subject: [PATCH 2/9] tweak --- esphome/analyze_memory/__init__.py | 105 ++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 5fa827795f8..6aeb0ef8a66 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -397,18 +397,22 @@ class MemoryAnalyzer: return "Other Core" - def _discover_pio_libraries(self) -> dict[str, Path]: + def _discover_pio_libraries( + self, + libraries: dict[str, Path], + hash_to_name: dict[str, str], + ) -> None: """Discover PlatformIO third-party libraries from the build directory. Scans ``lib/`` directories under ``.pioenvs//`` to find library names and their ``.a`` archive paths. - Returns: - Dictionary mapping lowercase library name to ``.a`` file path. + Args: + libraries: Dict to populate with library name -> ``.a`` path mappings. + hash_to_name: Dict to populate with dir name -> library name mappings + for CSWTCH attribution (e.g., ``lib641`` -> ``espsoftwareserial``). """ - # The ELF is typically at .pioenvs//firmware.elf build_dir = self.elf_path.parent - libraries: dict[str, Path] = {} for entry in build_dir.iterdir(): if not entry.is_dir() or not entry.name.startswith("lib"): @@ -427,6 +431,7 @@ class MemoryAnalyzer: for lib_subdir in entry.iterdir(): if not lib_subdir.is_dir(): continue + lib_name = lib_subdir.name.lower() # The .a file is named lib.a (case-insensitive match) # e.g., lib72a/ESPAsyncTCP/... has lib72a/libESPAsyncTCP.a archive = entry / f"lib{lib_subdir.name}.a" @@ -435,14 +440,57 @@ class MemoryAnalyzer: archives = list(entry.glob("*.a")) archive = archives[0] if archives else None if archive and archive.exists(): - libraries[lib_subdir.name.lower()] = archive + libraries[lib_name] = archive + hash_to_name[entry.name] = lib_name _LOGGER.debug( "Discovered PlatformIO library: %s -> %s", lib_subdir.name, archive, ) - return libraries + def _discover_idf_managed_components( + self, + libraries: dict[str, Path], + hash_to_name: dict[str, str], + ) -> None: + """Discover ESP-IDF managed component libraries from the build directory. + + ESP-IDF managed components (from the IDF component registry) use a + ``__`` naming convention. Source files live under + ``managed_components/__/`` and the compiled archives are at + ``esp-idf/__/lib__.a``. + + Args: + libraries: Dict to populate with library name -> ``.a`` path mappings. + hash_to_name: Dict to populate with dir name -> library name mappings + for CSWTCH attribution (e.g., ``espressif__mdns`` -> ``mdns``). + """ + build_dir = self.elf_path.parent + + managed_dir = build_dir / "managed_components" + if not managed_dir.is_dir(): + return + + espidf_dir = build_dir / "esp-idf" + + for entry in managed_dir.iterdir(): + if not entry.is_dir() or "__" not in entry.name: + continue + + # Extract the short name: espressif__mdns -> mdns + full_name = entry.name # e.g., espressif__mdns + short_name = full_name.split("__", 1)[1].lower() + + # Find the .a archive under esp-idf/__/ + archive = espidf_dir / full_name / f"lib{full_name}.a" + if archive.exists(): + libraries[short_name] = archive + hash_to_name[full_name] = short_name + _LOGGER.debug( + "Discovered IDF managed component: %s -> %s", + short_name, + archive, + ) def _build_library_symbol_map(self, libraries: dict[str, Path]) -> dict[str, str]: """Build a symbol-to-library mapping from library archives. @@ -484,42 +532,29 @@ class MemoryAnalyzer: return symbol_map def _scan_pio_libraries(self) -> None: - """Discover PlatformIO libraries and build symbol mapping. + """Discover third-party libraries and build symbol mapping. - Orchestrates library discovery, symbol map construction, and - hash-to-name mapping for CSWTCH attribution. + Scans both PlatformIO ``lib/`` directories (Arduino builds) and + ESP-IDF ``managed_components/`` (IDF builds) to find library archives. + Orchestrates symbol map construction and hash-to-name mapping for + CSWTCH attribution. """ - libraries = self._discover_pio_libraries() + libraries: dict[str, Path] = {} + self._discover_pio_libraries(libraries, self._lib_hash_to_name) + self._discover_idf_managed_components(libraries, self._lib_hash_to_name) + if not libraries: - _LOGGER.debug("No PlatformIO third-party libraries found") + _LOGGER.debug("No third-party libraries found") return _LOGGER.info( - "Scanning %d PlatformIO libraries: %s", + "Scanning %d libraries: %s", len(libraries), ", ".join(sorted(libraries)), ) self._lib_symbol_map = self._build_library_symbol_map(libraries) - # Build hash-to-name mapping for CSWTCH attribution - # e.g., lib641 -> espsoftwareserial - build_dir = self.elf_path.parent - for entry in build_dir.iterdir(): - if not entry.is_dir() or not entry.name.startswith("lib"): - continue - hex_part = entry.name[3:] - if not hex_part: - continue - try: - int(hex_part, 16) - except ValueError: - continue - for lib_subdir in entry.iterdir(): - if lib_subdir.is_dir(): - self._lib_hash_to_name[entry.name] = lib_subdir.name.lower() - break - _LOGGER.info( "Built library symbol map: %d symbols from %d libraries", len(self._lib_symbol_map), @@ -706,6 +741,14 @@ class MemoryAnalyzer: if parts and parts[0] in self._lib_hash_to_name: return f"{_COMPONENT_PREFIX_LIB}{self._lib_hash_to_name[parts[0]]}" + # ESP-IDF managed components: managed_components/espressif__mdns/... -> [lib]mdns + if ( + len(parts) >= 2 + and parts[0] == "managed_components" + and parts[1] in self._lib_hash_to_name + ): + return f"{_COMPONENT_PREFIX_LIB}{self._lib_hash_to_name[parts[1]]}" + # Other framework/library files - return the first path component # e.g., FrameworkArduino/... -> FrameworkArduino return parts[0] if parts else source_file From 17664750b79a271aa4c7cd140917190e30426715 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 11:20:54 -0600 Subject: [PATCH 3/9] tweak --- esphome/analyze_memory/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 6aeb0ef8a66..8de094ed37c 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -57,8 +57,9 @@ SymbolInfoType = tuple[str, int, str] # RAM sections - symbols in these sections consume RAM RAM_SECTIONS = frozenset([".data", ".bss"]) -# nm symbol types for global/weak defined symbols (used for library symbol mapping) -_NM_DEFINED_GLOBAL_TYPES = frozenset({"T", "D", "B", "R", "W", "V"}) +# nm symbol types for defined symbols (used for library symbol mapping) +# Uppercase = global, lowercase = local/static, W/V = weak +_NM_DEFINED_SYMBOL_TYPES = frozenset({"T", "t", "D", "d", "B", "b", "R", "r", "W", "V"}) @dataclass @@ -525,8 +526,8 @@ class MemoryAnalyzer: sym_type = parts[-2] sym_name = parts[-1] - # Include global defined symbols (uppercase) and weak symbols (W/V) - if sym_type in _NM_DEFINED_GLOBAL_TYPES: + # Include all defined symbols (global, local, and weak) + if sym_type in _NM_DEFINED_SYMBOL_TYPES: symbol_map[sym_name] = lib_name return symbol_map From 6dd7f45cc99f20efadf28cc829bb9b27e7d25bc2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:12:35 -0600 Subject: [PATCH 4/9] tweak --- esphome/analyze_memory/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 8de094ed37c..62aa59e22c6 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -57,9 +57,10 @@ SymbolInfoType = tuple[str, int, str] # RAM sections - symbols in these sections consume RAM RAM_SECTIONS = frozenset([".data", ".bss"]) -# nm symbol types for defined symbols (used for library symbol mapping) -# Uppercase = global, lowercase = local/static, W/V = weak -_NM_DEFINED_SYMBOL_TYPES = frozenset({"T", "t", "D", "d", "B", "b", "R", "r", "W", "V"}) +# nm symbol types for global/weak defined symbols (used for library symbol mapping) +# Only global (uppercase) and weak symbols are safe to use - local symbols (lowercase) +# can have name collisions across compilation units +_NM_DEFINED_GLOBAL_TYPES = frozenset({"T", "D", "B", "R", "W", "V"}) @dataclass @@ -526,8 +527,8 @@ class MemoryAnalyzer: sym_type = parts[-2] sym_name = parts[-1] - # Include all defined symbols (global, local, and weak) - if sym_type in _NM_DEFINED_SYMBOL_TYPES: + # Include global defined symbols (uppercase) and weak symbols (W/V) + if sym_type in _NM_DEFINED_GLOBAL_TYPES: symbol_map[sym_name] = lib_name return symbol_map From 30558262d84350e144c77ad228701b524cf8c93d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:27:09 -0600 Subject: [PATCH 5/9] fix dupes --- esphome/analyze_memory/__init__.py | 100 ++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 17 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 62aa59e22c6..751f4d5c7f7 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -189,6 +189,8 @@ class MemoryAnalyzer: self._lib_symbol_map: dict[str, str] = {} # PlatformIO library hash to name mapping: "lib641" -> "espsoftwareserial" self._lib_hash_to_name: dict[str, str] = {} + # Heuristic category to library redirect: "mdns_lib" -> "[lib]mdns" + self._heuristic_to_lib: dict[str, str] = {} def analyze(self) -> dict[str, ComponentMemory]: """Analyze the ELF file and return component memory usage.""" @@ -346,12 +348,12 @@ class MemoryAnalyzer: # Check against symbol patterns for component, patterns in SYMBOL_PATTERNS.items(): if any(pattern in symbol_name for pattern in patterns): - return component + return self._heuristic_to_lib.get(component, component) # Check against demangled patterns for component, patterns in DEMANGLED_PATTERNS.items(): if any(pattern in demangled for pattern in patterns): - return component + return self._heuristic_to_lib.get(component, component) # Special cases that need more complex logic @@ -401,16 +403,18 @@ class MemoryAnalyzer: def _discover_pio_libraries( self, - libraries: dict[str, Path], + libraries: dict[str, list[Path]], hash_to_name: dict[str, str], ) -> None: """Discover PlatformIO third-party libraries from the build directory. Scans ``lib/`` directories under ``.pioenvs//`` to find - library names and their ``.a`` archive paths. + library names and their ``.a`` archive or ``.o`` file paths. Args: - libraries: Dict to populate with library name -> ``.a`` path mappings. + libraries: Dict to populate with library name -> file path list mappings. + Prefers ``.a`` archives when available, falls back to ``.o`` files + (e.g., pioarduino ESP32 Arduino builds only produce ``.o`` files). hash_to_name: Dict to populate with dir name -> library name mappings for CSWTCH attribution (e.g., ``lib641`` -> ``espsoftwareserial``). """ @@ -442,17 +446,28 @@ class MemoryAnalyzer: archives = list(entry.glob("*.a")) archive = archives[0] if archives else None if archive and archive.exists(): - libraries[lib_name] = archive + libraries[lib_name] = [archive] hash_to_name[entry.name] = lib_name _LOGGER.debug( "Discovered PlatformIO library: %s -> %s", lib_subdir.name, archive, ) + else: + # No .a archive (e.g., pioarduino CMake builds) - use .o files + obj_files = sorted(lib_subdir.rglob("*.o")) + if obj_files: + libraries[lib_name] = obj_files + hash_to_name[entry.name] = lib_name + _LOGGER.debug( + "Discovered PlatformIO library (objects): %s -> %d .o files", + lib_subdir.name, + len(obj_files), + ) def _discover_idf_managed_components( self, - libraries: dict[str, Path], + libraries: dict[str, list[Path]], hash_to_name: dict[str, str], ) -> None: """Discover ESP-IDF managed component libraries from the build directory. @@ -463,7 +478,7 @@ class MemoryAnalyzer: ``esp-idf/__/lib__.a``. Args: - libraries: Dict to populate with library name -> ``.a`` path mappings. + libraries: Dict to populate with library name -> file path list mappings. hash_to_name: Dict to populate with dir name -> library name mappings for CSWTCH attribution (e.g., ``espressif__mdns`` -> ``mdns``). """ @@ -486,7 +501,7 @@ class MemoryAnalyzer: # Find the .a archive under esp-idf/__/ archive = espidf_dir / full_name / f"lib{full_name}.a" if archive.exists(): - libraries[short_name] = archive + libraries[short_name] = [archive] hash_to_name[full_name] = short_name _LOGGER.debug( "Discovered IDF managed component: %s -> %s", @@ -494,14 +509,17 @@ class MemoryAnalyzer: archive, ) - def _build_library_symbol_map(self, libraries: dict[str, Path]) -> dict[str, str]: - """Build a symbol-to-library mapping from library archives. + def _build_library_symbol_map( + self, libraries: dict[str, list[Path]] + ) -> dict[str, str]: + """Build a symbol-to-library mapping from library archives or object files. - Runs ``nm --defined-only`` on each ``.a`` file to collect global and - weak defined symbols. + Runs ``nm --defined-only`` on each ``.a`` or ``.o`` file to collect + global and weak defined symbols. Args: - libraries: Dictionary mapping library name to ``.a`` file path. + libraries: Dictionary mapping library name to list of file paths + (``.a`` archives or ``.o`` object files). Returns: Dictionary mapping symbol name to library name. @@ -511,9 +529,9 @@ class MemoryAnalyzer: if not self.nm_path: return symbol_map - for lib_name, archive_path in libraries.items(): + for lib_name, file_paths in libraries.items(): result = run_tool( - [self.nm_path, "--defined-only", str(archive_path)], + [self.nm_path, "--defined-only", *(str(p) for p in file_paths)], timeout=10, ) if result is None or result.returncode != 0: @@ -533,6 +551,51 @@ class MemoryAnalyzer: return symbol_map + @staticmethod + def _build_heuristic_to_lib_mapping( + library_names: set[str], + ) -> dict[str, str]: + """Build mapping from heuristic pattern categories to discovered libraries. + + Heuristic categories like ``mdns_lib``, ``web_server_lib``, ``async_tcp`` + exist as approximations for library attribution. When we discover the + actual library, symbols matching those heuristics should be redirected + to the ``[lib]`` category instead. + + The mapping is built by checking if the normalized category name + (stripped of ``_lib`` suffix and underscores) appears as a substring + of any discovered library name. + + Examples:: + + mdns_lib -> mdns -> in "mdns" or "esp8266mdns" -> [lib]mdns + web_server_lib -> webserver -> in "espasyncwebserver" -> [lib]espasyncwebserver + async_tcp -> asynctcp -> in "espasynctcp" -> [lib]espasynctcp + + Args: + library_names: Set of discovered library names (lowercase). + + Returns: + Dictionary mapping heuristic category to ``[lib]`` string. + """ + mapping: dict[str, str] = {} + all_categories = set(SYMBOL_PATTERNS) | set(DEMANGLED_PATTERNS) + + for category in all_categories: + base = category.removesuffix("_lib").replace("_", "") + for lib_name in library_names: + if base in lib_name: + mapping[category] = f"{_COMPONENT_PREFIX_LIB}{lib_name}" + break + + if mapping: + _LOGGER.debug( + "Heuristic-to-library redirects: %s", + ", ".join(f"{k} -> {v}" for k, v in sorted(mapping.items())), + ) + + return mapping + def _scan_pio_libraries(self) -> None: """Discover third-party libraries and build symbol mapping. @@ -541,7 +604,7 @@ class MemoryAnalyzer: Orchestrates symbol map construction and hash-to-name mapping for CSWTCH attribution. """ - libraries: dict[str, Path] = {} + libraries: dict[str, list[Path]] = {} self._discover_pio_libraries(libraries, self._lib_hash_to_name) self._discover_idf_managed_components(libraries, self._lib_hash_to_name) @@ -556,6 +619,9 @@ class MemoryAnalyzer: ) self._lib_symbol_map = self._build_library_symbol_map(libraries) + self._heuristic_to_lib = self._build_heuristic_to_lib_mapping( + set(libraries.keys()) + ) _LOGGER.info( "Built library symbol map: %d symbols from %d libraries", From 6099dd3065dae2e9b949b250577bf0f4e963cd96 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:34:00 -0600 Subject: [PATCH 6/9] wip --- esphome/analyze_memory/__init__.py | 43 +++++++++++++----------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 751f4d5c7f7..4c908b9fac2 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -185,9 +185,10 @@ class MemoryAnalyzer: self._sdk_symbols: list[SDKSymbol] = [] # CSWTCH symbols: list of (name, size, source_file, component) self._cswtch_symbols: list[tuple[str, int, str, str]] = [] - # PlatformIO library symbol mapping: symbol_name -> library_name + # Library symbol mapping: symbol_name -> library_name self._lib_symbol_map: dict[str, str] = {} - # PlatformIO library hash to name mapping: "lib641" -> "espsoftwareserial" + # Library dir to name mapping: "lib641" -> "espsoftwareserial", + # "espressif__mdns" -> "mdns" self._lib_hash_to_name: dict[str, str] = {} # Heuristic category to library redirect: "mdns_lib" -> "[lib]mdns" self._heuristic_to_lib: dict[str, str] = {} @@ -196,7 +197,7 @@ class MemoryAnalyzer: """Analyze the ELF file and return component memory usage.""" self._parse_sections() self._parse_symbols() - self._scan_pio_libraries() + self._scan_libraries() self._categorize_symbols() self._analyze_cswtch_symbols() self._analyze_sdk_libraries() @@ -433,37 +434,31 @@ class MemoryAnalyzer: continue # Each lib/ directory contains a subdirectory named after the library - # and a .a archive named lib.a for lib_subdir in entry.iterdir(): if not lib_subdir.is_dir(): continue lib_name = lib_subdir.name.lower() - # The .a file is named lib.a (case-insensitive match) + + # Prefer .a archive (lib.a), fall back to .o files # e.g., lib72a/ESPAsyncTCP/... has lib72a/libESPAsyncTCP.a archive = entry / f"lib{lib_subdir.name}.a" - if not archive.exists(): - # Try case-insensitive: scan for any .a file - archives = list(entry.glob("*.a")) - archive = archives[0] if archives else None - if archive and archive.exists(): - libraries[lib_name] = [archive] + if archive.exists(): + file_paths = [archive] + elif archives := list(entry.glob("*.a")): + # Case-insensitive fallback + file_paths = [archives[0]] + else: + # No .a archive (e.g., pioarduino CMake builds) - use .o files + file_paths = sorted(lib_subdir.rglob("*.o")) + + if file_paths: + libraries[lib_name] = file_paths hash_to_name[entry.name] = lib_name _LOGGER.debug( "Discovered PlatformIO library: %s -> %s", lib_subdir.name, - archive, + file_paths[0], ) - else: - # No .a archive (e.g., pioarduino CMake builds) - use .o files - obj_files = sorted(lib_subdir.rglob("*.o")) - if obj_files: - libraries[lib_name] = obj_files - hash_to_name[entry.name] = lib_name - _LOGGER.debug( - "Discovered PlatformIO library (objects): %s -> %d .o files", - lib_subdir.name, - len(obj_files), - ) def _discover_idf_managed_components( self, @@ -596,7 +591,7 @@ class MemoryAnalyzer: return mapping - def _scan_pio_libraries(self) -> None: + def _scan_libraries(self) -> None: """Discover third-party libraries and build symbol mapping. Scans both PlatformIO ``lib/`` directories (Arduino builds) and From 4efdfda8fb0942df4abdddba03ec153904885d24 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:34:37 -0600 Subject: [PATCH 7/9] wip --- esphome/analyze_memory/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 4c908b9fac2..6e02e7c3b2f 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -342,7 +342,7 @@ class MemoryAnalyzer: # If no component match found, it's core return _COMPONENT_CORE - # Check PlatformIO library symbol map (more accurate than heuristic patterns) + # Check library symbol map (more accurate than heuristic patterns) if lib_name := self._lib_symbol_map.get(symbol_name): return f"{_COMPONENT_PREFIX_LIB}{lib_name}" From 5caa54761bdac9e16d6ad21008fc642c428de007 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:51:11 -0600 Subject: [PATCH 8/9] tweaks --- esphome/analyze_memory/__init__.py | 92 ++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 6e02e7c3b2f..9af03b76b82 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -62,6 +62,11 @@ RAM_SECTIONS = frozenset([".data", ".bss"]) # can have name collisions across compilation units _NM_DEFINED_GLOBAL_TYPES = frozenset({"T", "D", "B", "R", "W", "V"}) +# Pattern matching compiler-generated local names that can collide across compilation +# units (e.g., packet$19, buf$20, flag$5261). These are unsafe for name-based lookup. +# Does NOT match mangled C++ names with optimization suffixes (e.g., func$isra$0). +_COMPILER_LOCAL_PATTERN = re.compile(r"^[a-zA-Z_]\w*\$\d+$") + @dataclass class MemorySection: @@ -591,13 +596,78 @@ class MemoryAnalyzer: return mapping + def _parse_map_file(self) -> dict[str, str] | None: + """Parse linker map file to build authoritative symbol-to-library mapping. + + The linker map file contains the definitive source attribution for every + symbol, including local/static ones that ``nm`` cannot safely export. + + Map file format (GNU ld):: + + .text._mdns_service_task + 0x400e9fdc 0x65c .pioenvs/env/esp-idf/espressif__mdns/libespressif__mdns.a(mdns.c.o) + + Each section entry has a ``.section.symbol_name`` line followed by an + indented line with address, size, and source path. + + Returns: + Symbol-to-library dict, or ``None`` if no usable map file exists. + """ + map_path = self.elf_path.with_suffix(".map") + if not map_path.exists() or map_path.stat().st_size < 10000: + return None + + _LOGGER.info("Parsing linker map file: %s", map_path.name) + + symbol_map: dict[str, str] = {} + current_symbol: str | None = None + section_prefixes = (".text.", ".rodata.", ".data.", ".bss.", ".literal.") + + for line in map_path.read_text().splitlines(): + # Match section.symbol line: " .text.symbol_name" + # Single space indent, starts with dot + if len(line) > 2 and line[0] == " " and line[1] == ".": + stripped = line.strip() + for prefix in section_prefixes: + if stripped.startswith(prefix): + current_symbol = stripped[len(prefix) :] + break + else: + current_symbol = None + continue + + # Match source attribution line: " 0xADDR 0xSIZE source_path" + if current_symbol is None: + continue + + fields = line.split() + # Skip compiler-generated local names (e.g., packet$19, buf$20) + # that can collide across compilation units + if ( + len(fields) >= 3 + and fields[0].startswith("0x") + and fields[1].startswith("0x") + and not _COMPILER_LOCAL_PATTERN.match(current_symbol) + ): + source_path = fields[2] + # Check if source path contains a known library directory + for dir_key, lib_name in self._lib_hash_to_name.items(): + if dir_key in source_path: + symbol_map[current_symbol] = lib_name + break + + current_symbol = None + + return symbol_map or None + def _scan_libraries(self) -> None: """Discover third-party libraries and build symbol mapping. Scans both PlatformIO ``lib/`` directories (Arduino builds) and ESP-IDF ``managed_components/`` (IDF builds) to find library archives. - Orchestrates symbol map construction and hash-to-name mapping for - CSWTCH attribution. + + Uses the linker map file for authoritative symbol attribution when + available, falling back to ``nm`` scanning with heuristic redirects. """ libraries: dict[str, list[Path]] = {} self._discover_pio_libraries(libraries, self._lib_hash_to_name) @@ -613,13 +683,27 @@ class MemoryAnalyzer: ", ".join(sorted(libraries)), ) - self._lib_symbol_map = self._build_library_symbol_map(libraries) + # Heuristic redirect catches local symbols (e.g., mdns_task_buffer$14) + # that can't be safely added to the symbol map due to name collisions self._heuristic_to_lib = self._build_heuristic_to_lib_mapping( set(libraries.keys()) ) + # Try linker map file first (authoritative, includes local symbols) + map_symbols = self._parse_map_file() + if map_symbols is not None: + self._lib_symbol_map = map_symbols + _LOGGER.info( + "Built library symbol map from linker map: %d symbols", + len(self._lib_symbol_map), + ) + return + + # Fall back to nm scanning (global symbols only) + self._lib_symbol_map = self._build_library_symbol_map(libraries) + _LOGGER.info( - "Built library symbol map: %d symbols from %d libraries", + "Built library symbol map from nm: %d symbols from %d libraries", len(self._lib_symbol_map), len(libraries), ) From b728ebd5fb36ae9c9cf60d7c24f5cab3d04e7f0a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 8 Feb 2026 14:56:57 -0600 Subject: [PATCH 9/9] tweaks --- esphome/analyze_memory/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 9af03b76b82..fe08a5f9664 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -623,7 +623,7 @@ class MemoryAnalyzer: current_symbol: str | None = None section_prefixes = (".text.", ".rodata.", ".data.", ".bss.", ".literal.") - for line in map_path.read_text().splitlines(): + for line in map_path.read_text(encoding="utf-8").splitlines(): # Match section.symbol line: " .text.symbol_name" # Single space indent, starts with dot if len(line) > 2 and line[0] == " " and line[1] == ".":