From 18e3eac419808d4d176e7d6b5840b081fca4c354 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 14:14:32 -0500 Subject: [PATCH 1/2] Export a surgery fingerprint for linker-script cache stamps --- esphome/components/esp8266/build_surgery.py | 21 ++++++++++++++++++- .../components/esp8266/test_build_surgery.py | 13 ++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py index 9aeee79216..c0034f7bad 100644 --- a/esphome/components/esp8266/build_surgery.py +++ b/esphome/components/esp8266/build_surgery.py @@ -11,11 +11,15 @@ as plain functions. Keep both in sync when changing either. from __future__ import annotations from collections.abc import Collection +import hashlib import re # Move the NONOS SDK wifi rate tables from flash to DRAM; see # relocate_ratetable.py.script for the full background (NONOS SDK issue 320). RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +_RATETABLE_COMMENT = ( + "/* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" +) # Match the whole line: "_data_start" is also a substring of the # "_dport0_data_start" line in the earlier .dport0.data section _RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) @@ -40,7 +44,7 @@ def relocate_ratetable(content: str) -> str: insert_pos = match.end() return ( content[:insert_pos] - + "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" + + f"\n {_RATETABLE_COMMENT}" + f"\n {RATETABLE_RULE}" + content[insert_pos:] ) @@ -84,3 +88,18 @@ def segment_length(content: str, segment_name: str) -> int | None: """Read a memory segment's length from linker script content.""" match = _segment_line_re(segment_name).search(content) return int(match.group(2), 16) if match else None + + +def surgery_fingerprint() -> str: + """Fingerprint of every behavioral input to the surgeries. + + Linker-script caches include it so an edit here invalidates them. + Native-toolchain-only, like ``segment_length``; no script twin. + """ + parts = ( + RATETABLE_RULE, + _RATETABLE_COMMENT, + _RATETABLE_ANCHOR.pattern, + repr(sorted(_TESTING_SEGMENT_SIZES.items())), + ) + return hashlib.sha256("|".join(parts).encode()).hexdigest() diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py index 0e5a26294a..64e36f1591 100644 --- a/tests/unit_tests/components/esp8266/test_build_surgery.py +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -77,3 +77,16 @@ def test_board_build_covers_every_board() -> None: """Every supported board has native build metadata (the table may carry extras that BOARDS does not expose).""" assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) + + +def test_surgery_fingerprint_tracks_inputs() -> None: + """The fingerprint changes with any behavioral input, so linker-script + caches stamped with it self-invalidate on surgery edits.""" + from unittest.mock import patch + + from esphome.components.esp8266 import build_surgery + + base = build_surgery.surgery_fingerprint() + assert base == build_surgery.surgery_fingerprint() + with patch.object(build_surgery, "_TESTING_SEGMENT_SIZES", {"iram1_0_seg": "0x1"}): + assert build_surgery.surgery_fingerprint() != base From d64592d2eae510306fdefe3bad6dc0d69a9fe3f7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 20 Aug 2026 14:16:46 -0500 Subject: [PATCH 2/2] Simplify library classification, share the manifest flag lexing, and home the build tool with the generators --- esphome/arduino8266/component.py | 32 +++++------- .../{arduino8266 => build_gen}/build_tool.py | 0 esphome/espidf/component.py | 12 +---- esphome/platformio/library.py | 17 +++++++ .../test_build_tool.py} | 2 +- .../unit_tests/test_arduino8266_component.py | 50 +++++++++---------- 6 files changed, 55 insertions(+), 58 deletions(-) rename esphome/{arduino8266 => build_gen}/build_tool.py (100%) rename tests/unit_tests/{test_arduino8266_build_tool.py => build_gen/test_build_tool.py} (97%) diff --git a/esphome/arduino8266/component.py b/esphome/arduino8266/component.py index 3d70f142ca..a39f3f9a8b 100644 --- a/esphome/arduino8266/component.py +++ b/esphome/arduino8266/component.py @@ -31,11 +31,10 @@ from esphome.platformio.library import ( convert_libraries, ensure_list, is_lib_ignored, - join_flag_args, + lex_build_flags, lib_ignore_set, normalize_dependencies, parse_library_properties, - split_flag_entry, ) _LOGGER = logging.getLogger(__name__) @@ -77,14 +76,7 @@ def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) # PlatformIO shell-lexes each build.flags entry - flag_tokens = join_flag_args( - ( - token - for entry in ensure_list(build.get("flags", [])) - for token in split_flag_entry(entry, f"library {name}") - ), - f"library {name}", - ) + flag_tokens = lex_build_flags(build.get("flags", []), f"library {name}") lib = ArduinoLibrary(name=name) include_flags: list[str] = [] @@ -153,20 +145,20 @@ def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]: for library in CORE.platformio_libraries.values(): if is_lib_ignored(library.name, lib_ignore): continue - # A version pin means a registry package ("pngle@1.1.0"), never a - # framework-bundled library. + # Only a bare name with a matching framework directory is bundled: a + # version pin means a registry package ("pngle@1.1.0"), and a bare + # name without the directory resolves from the registry at the + # latest version, matching PlatformIO (a typo fails loudly as a + # registry lookup error). if ( - library.repository - or library.version - or not library.name - or "/" in library.name + not library.repository + and not library.version + and library.name + and "/" not in library.name + and (framework_path / "libraries" / library.name).is_dir() ): - external.append(library) - elif (framework_path / "libraries" / library.name).is_dir(): bundled.append(_bundled_library(framework_path, library.name)) else: - # A bare registry name; resolved at the latest version, matching - # PlatformIO (a typo fails loudly as a registry lookup error). external.append(library) converted: list[ArduinoLibrary] = [] diff --git a/esphome/arduino8266/build_tool.py b/esphome/build_gen/build_tool.py similarity index 100% rename from esphome/arduino8266/build_tool.py rename to esphome/build_gen/build_tool.py diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 604a61af00..97d66bcc51 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -27,8 +27,7 @@ from esphome.platformio.library import ( collect_filtered_files, convert_libraries, ensure_list, - join_flag_args, - split_flag_entry, + lex_build_flags, split_list_by_condition, ) @@ -107,14 +106,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # flag and its argument (e.g. "-include cp_custom_alloc.h"); bare # -I/-L/-l tokens re-glue to their argument ("-I foo" -> "-Ifoo") so the # prefix classifiers below still route them. - build_flags = join_flag_args( - ( - token - for entry in build_flags - for token in split_flag_entry(entry, f"library {component.name}") - ), - f"library {component.name}", - ) + build_flags = lex_build_flags(build_flags, f"library {component.name}") # List all sources files build_src_files = collect_filtered_files( diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 4ae256444a..6283a566e4 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -562,6 +562,23 @@ def split_flag_entry(entry: str, owner: str) -> list[str]: raise EsphomeError(f"Malformed build flag {entry!r} in {owner}: {err}") from err +def lex_build_flags(entries, owner: str) -> list[str]: + """Shell-lex a manifest ``build.flags`` list into joined tokens. + + The composition every backend needs: each entry is lexed the way + PlatformIO's ParseFlags does, and bare ``-I``/``-L``/``-l``/``-D`` + tokens re-glue to their argument across the whole stream. + """ + return join_flag_args( + ( + token + for entry in ensure_list(entries) + for token in split_flag_entry(entry, owner) + ), + owner, + ) + + def join_flag_args(tokens: Iterable[str], owner: str) -> list[str]: """Join a bare ``-I``/``-L``/``-l``/``-D`` with its following token, the way PlatformIO's ParseFlags lexes them.""" diff --git a/tests/unit_tests/test_arduino8266_build_tool.py b/tests/unit_tests/build_gen/test_build_tool.py similarity index 97% rename from tests/unit_tests/test_arduino8266_build_tool.py rename to tests/unit_tests/build_gen/test_build_tool.py index 66d11a6700..82ca90a8c7 100644 --- a/tests/unit_tests/test_arduino8266_build_tool.py +++ b/tests/unit_tests/build_gen/test_build_tool.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch import pytest -from esphome.arduino8266 import build_tool +from esphome.build_gen import build_tool def test_ar_removes_stale_archive(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_arduino8266_component.py b/tests/unit_tests/test_arduino8266_component.py index d26b1439c7..222a881f85 100644 --- a/tests/unit_tests/test_arduino8266_component.py +++ b/tests/unit_tests/test_arduino8266_component.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch @@ -38,6 +39,25 @@ def _make_framework(tmp_path: Path) -> Path: return framework +@contextmanager +def _emitting_converter(*converted): + """Patch convert_libraries to emit the given components via the backend.""" + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + for c in converted: + backend.emit(c) + return list(converted) + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + yield mock_extra + + def test_library_info_src_layout(tmp_path: Path) -> None: framework = _make_framework(tmp_path) lib = component._bundled_library(framework, "ESP8266WiFi") @@ -176,17 +196,7 @@ def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: }, ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - assert backend.platform == "espressif8266" - assert backend.framework == "arduino" - assert backend.cache_key == "arduino8266" - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script") as mock_extra, - ): + with _emitting_converter(converted) as mock_extra: libs = component.resolve_libraries(framework) mock_extra.assert_called_once_with( @@ -209,14 +219,7 @@ def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script"), - ): + with _emitting_converter(converted): libs = component.resolve_libraries(framework) # Wire appears once (from the explicit registration), not twice @@ -297,14 +300,7 @@ def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script"), - ): + with _emitting_converter(converted): libs = component.resolve_libraries(framework) assert [lib.name for lib in libs] == ["some__External"]