diff --git a/esphome/arduino8266/component.py b/esphome/arduino8266/component.py index 3d70f142ca..a39f3f9a8b 100644 --- a/esphome/arduino8266/component.py +++ b/esphome/arduino8266/component.py @@ -31,11 +31,10 @@ from esphome.platformio.library import ( convert_libraries, ensure_list, is_lib_ignored, - join_flag_args, + lex_build_flags, lib_ignore_set, normalize_dependencies, parse_library_properties, - split_flag_entry, ) _LOGGER = logging.getLogger(__name__) @@ -77,14 +76,7 @@ def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) # PlatformIO shell-lexes each build.flags entry - flag_tokens = join_flag_args( - ( - token - for entry in ensure_list(build.get("flags", [])) - for token in split_flag_entry(entry, f"library {name}") - ), - f"library {name}", - ) + flag_tokens = lex_build_flags(build.get("flags", []), f"library {name}") lib = ArduinoLibrary(name=name) include_flags: list[str] = [] @@ -153,20 +145,20 @@ def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]: for library in CORE.platformio_libraries.values(): if is_lib_ignored(library.name, lib_ignore): continue - # A version pin means a registry package ("pngle@1.1.0"), never a - # framework-bundled library. + # Only a bare name with a matching framework directory is bundled: a + # version pin means a registry package ("pngle@1.1.0"), and a bare + # name without the directory resolves from the registry at the + # latest version, matching PlatformIO (a typo fails loudly as a + # registry lookup error). if ( - library.repository - or library.version - or not library.name - or "/" in library.name + not library.repository + and not library.version + and library.name + and "/" not in library.name + and (framework_path / "libraries" / library.name).is_dir() ): - external.append(library) - elif (framework_path / "libraries" / library.name).is_dir(): bundled.append(_bundled_library(framework_path, library.name)) else: - # A bare registry name; resolved at the latest version, matching - # PlatformIO (a typo fails loudly as a registry lookup error). external.append(library) converted: list[ArduinoLibrary] = [] diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 3199b52f04..63fb8a07b9 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -99,6 +99,36 @@ _LWIP_VARIANTS = ( ) _LWIP_DEFAULT = (536, 1, 0, "lwip2-536-feat") +# knob define -> MMU_* defines, first match wins (as in platformio-build.py) +_MMU_VARIANTS = ( + ( + "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48", + ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED", + ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED", + [ + "MMU_IRAM_SIZE=0x8000", + "MMU_ICACHE_SIZE=0x4000", + "MMU_SEC_HEAP_SIZE=0x4000", + "MMU_SEC_HEAP=0x40108000", + ], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K", + ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000", "MMU_EXTERNAL_HEAP=128"], + ), + ( + "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K", + ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000", "MMU_EXTERNAL_HEAP=256"], + ), +) +_MMU_DEFAULT = ("MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000") + _ASFLAGS = ["-mlongcalls", "-mtext-section-literals"] _CFLAGS = [ "-std=gnu17", @@ -219,40 +249,22 @@ def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig: "VTABLES_IN_FLASH", ) - if "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48" in defines: - mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000"] - elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM48_SECHEAP_SHARED" in defines: - mmu = ["MMU_IRAM_SIZE=0xC000", "MMU_ICACHE_SIZE=0x4000", "MMU_IRAM_HEAP"] - elif "PIO_FRAMEWORK_ARDUINO_MMU_CACHE16_IRAM32_SECHEAP_NOTSHARED" in defines: - mmu = [ - "MMU_IRAM_SIZE=0x8000", - "MMU_ICACHE_SIZE=0x4000", - "MMU_SEC_HEAP_SIZE=0x4000", - "MMU_SEC_HEAP=0x40108000", - ] - elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_128K" in defines: - mmu = [ - "MMU_IRAM_SIZE=0x8000", - "MMU_ICACHE_SIZE=0x8000", - "MMU_EXTERNAL_HEAP=128", - ] - elif "PIO_FRAMEWORK_ARDUINO_MMU_EXTERNAL_1024K" in defines: - mmu = [ - "MMU_IRAM_SIZE=0x8000", - "MMU_ICACHE_SIZE=0x8000", - "MMU_EXTERNAL_HEAP=256", - ] - elif "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM" in defines: - if "MMU_IRAM_SIZE" not in defines or "MMU_ICACHE_SIZE" not in defines: - raise EsphomeError( - "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM requires MMU_IRAM_SIZE and " - "MMU_ICACHE_SIZE build flags" + mmu = next((variant for knob, variant in _MMU_VARIANTS if knob in defines), None) + if mmu is None: + if "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM" in defines: + if "MMU_IRAM_SIZE" not in defines or "MMU_ICACHE_SIZE" not in defines: + raise EsphomeError( + "PIO_FRAMEWORK_ARDUINO_MMU_CUSTOM requires MMU_IRAM_SIZE and " + "MMU_ICACHE_SIZE build flags" + ) + # Sorted so build.ninja and the linker-script stamp stay + # byte-stable across runs (the flag set has no deterministic + # iteration order). + mmu = sorted( + body for name, body in defines.items() if name.startswith("MMU_") ) - # Sorted so build.ninja and the linker-script stamp stay byte-stable - # across runs (the flag set has no deterministic iteration order). - mmu = sorted(body for name, body in defines.items() if name.startswith("MMU_")) - else: - mmu = ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"] + else: + mmu = list(_MMU_DEFAULT) return _BuildConfig( nonosdk=nonosdk, @@ -418,10 +430,9 @@ def generate_ld_scripts( stamp_content = ( " ".join(cmd) + f" testing={CORE.testing_mode}" - + f" {build_surgery.RATETABLE_RULE}" - + f" {build_surgery.TESTING_IRAM_SIZE}" - + f" {build_surgery.TESTING_DRAM_SIZE}" - + f" {build_surgery.TESTING_FLASH_SIZE}" + # One fingerprint instead of enumerating surgery internals here, so + # any behavioral edit in build_surgery self-invalidates the cache + + f" {build_surgery.surgery_fingerprint()}" ) if not ( output.is_file() @@ -580,7 +591,7 @@ def write_project(paths: dict[str, Path]) -> bool: + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"] ) - build_tool = Path(__file__).parent.parent / "arduino8266" / "build_tool.py" + build_tool = Path(__file__).parent / "build_tool.py" ccache = ccache_path() # $in/$out stay unquoted in the rule commands: ninja shell-escapes its diff --git a/esphome/arduino8266/build_tool.py b/esphome/build_gen/build_tool.py similarity index 100% rename from esphome/arduino8266/build_tool.py rename to esphome/build_gen/build_tool.py diff --git a/esphome/components/esp8266/build_surgery.py b/esphome/components/esp8266/build_surgery.py index 9aeee79216..c0034f7bad 100644 --- a/esphome/components/esp8266/build_surgery.py +++ b/esphome/components/esp8266/build_surgery.py @@ -11,11 +11,15 @@ as plain functions. Keep both in sync when changing either. from __future__ import annotations from collections.abc import Collection +import hashlib import re # Move the NONOS SDK wifi rate tables from flash to DRAM; see # relocate_ratetable.py.script for the full background (NONOS SDK issue 320). RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +_RATETABLE_COMMENT = ( + "/* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" +) # Match the whole line: "_data_start" is also a substring of the # "_dport0_data_start" line in the earlier .dport0.data section _RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) @@ -40,7 +44,7 @@ def relocate_ratetable(content: str) -> str: insert_pos = match.end() return ( content[:insert_pos] - + "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" + + f"\n {_RATETABLE_COMMENT}" + f"\n {RATETABLE_RULE}" + content[insert_pos:] ) @@ -84,3 +88,18 @@ def segment_length(content: str, segment_name: str) -> int | None: """Read a memory segment's length from linker script content.""" match = _segment_line_re(segment_name).search(content) return int(match.group(2), 16) if match else None + + +def surgery_fingerprint() -> str: + """Fingerprint of every behavioral input to the surgeries. + + Linker-script caches include it so an edit here invalidates them. + Native-toolchain-only, like ``segment_length``; no script twin. + """ + parts = ( + RATETABLE_RULE, + _RATETABLE_COMMENT, + _RATETABLE_ANCHOR.pattern, + repr(sorted(_TESTING_SEGMENT_SIZES.items())), + ) + return hashlib.sha256("|".join(parts).encode()).hexdigest() diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 604a61af00..97d66bcc51 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -27,8 +27,7 @@ from esphome.platformio.library import ( collect_filtered_files, convert_libraries, ensure_list, - join_flag_args, - split_flag_entry, + lex_build_flags, split_list_by_condition, ) @@ -107,14 +106,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # flag and its argument (e.g. "-include cp_custom_alloc.h"); bare # -I/-L/-l tokens re-glue to their argument ("-I foo" -> "-Ifoo") so the # prefix classifiers below still route them. - build_flags = join_flag_args( - ( - token - for entry in build_flags - for token in split_flag_entry(entry, f"library {component.name}") - ), - f"library {component.name}", - ) + build_flags = lex_build_flags(build_flags, f"library {component.name}") # List all sources files build_src_files = collect_filtered_files( diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 4ae256444a..6283a566e4 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -562,6 +562,23 @@ def split_flag_entry(entry: str, owner: str) -> list[str]: raise EsphomeError(f"Malformed build flag {entry!r} in {owner}: {err}") from err +def lex_build_flags(entries, owner: str) -> list[str]: + """Shell-lex a manifest ``build.flags`` list into joined tokens. + + The composition every backend needs: each entry is lexed the way + PlatformIO's ParseFlags does, and bare ``-I``/``-L``/``-l``/``-D`` + tokens re-glue to their argument across the whole stream. + """ + return join_flag_args( + ( + token + for entry in ensure_list(entries) + for token in split_flag_entry(entry, owner) + ), + owner, + ) + + def join_flag_args(tokens: Iterable[str], owner: str) -> list[str]: """Join a bare ``-I``/``-L``/``-l``/``-D`` with its following token, the way PlatformIO's ParseFlags lexes them.""" diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 5f7d2cebdd..6eab144b91 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -414,7 +414,9 @@ def test_generate_ld_scripts(tmp_path: Path) -> None: # An edit to the surgery constants invalidates the stamp (a stale linker # script would otherwise persist until an esphome clean) with ( - patch.object(arduino8266.build_surgery, "TESTING_FLASH_SIZE", "0x3000000"), + patch.object( + arduino8266.build_surgery, "surgery_fingerprint", return_value="changed" + ), patch.object(arduino8266.subprocess, "run", return_value=result) as mock_run, ): _run_generate_ld_scripts(paths) diff --git a/tests/unit_tests/test_arduino8266_build_tool.py b/tests/unit_tests/build_gen/test_build_tool.py similarity index 97% rename from tests/unit_tests/test_arduino8266_build_tool.py rename to tests/unit_tests/build_gen/test_build_tool.py index 66d11a6700..82ca90a8c7 100644 --- a/tests/unit_tests/test_arduino8266_build_tool.py +++ b/tests/unit_tests/build_gen/test_build_tool.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch import pytest -from esphome.arduino8266 import build_tool +from esphome.build_gen import build_tool def test_ar_removes_stale_archive(tmp_path: Path) -> None: diff --git a/tests/unit_tests/components/esp8266/test_build_surgery.py b/tests/unit_tests/components/esp8266/test_build_surgery.py index 0e5a26294a..64e36f1591 100644 --- a/tests/unit_tests/components/esp8266/test_build_surgery.py +++ b/tests/unit_tests/components/esp8266/test_build_surgery.py @@ -77,3 +77,16 @@ def test_board_build_covers_every_board() -> None: """Every supported board has native build metadata (the table may carry extras that BOARDS does not expose).""" assert set(BOARDS) <= set(ESP8266_BOARD_BUILD) + + +def test_surgery_fingerprint_tracks_inputs() -> None: + """The fingerprint changes with any behavioral input, so linker-script + caches stamped with it self-invalidate on surgery edits.""" + from unittest.mock import patch + + from esphome.components.esp8266 import build_surgery + + base = build_surgery.surgery_fingerprint() + assert base == build_surgery.surgery_fingerprint() + with patch.object(build_surgery, "_TESTING_SEGMENT_SIZES", {"iram1_0_seg": "0x1"}): + assert build_surgery.surgery_fingerprint() != base diff --git a/tests/unit_tests/test_arduino8266_component.py b/tests/unit_tests/test_arduino8266_component.py index d26b1439c7..222a881f85 100644 --- a/tests/unit_tests/test_arduino8266_component.py +++ b/tests/unit_tests/test_arduino8266_component.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch @@ -38,6 +39,25 @@ def _make_framework(tmp_path: Path) -> Path: return framework +@contextmanager +def _emitting_converter(*converted): + """Patch convert_libraries to emit the given components via the backend.""" + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + for c in converted: + backend.emit(c) + return list(converted) + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + yield mock_extra + + def test_library_info_src_layout(tmp_path: Path) -> None: framework = _make_framework(tmp_path) lib = component._bundled_library(framework, "ESP8266WiFi") @@ -176,17 +196,7 @@ def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: }, ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - assert backend.platform == "espressif8266" - assert backend.framework == "arduino" - assert backend.cache_key == "arduino8266" - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script") as mock_extra, - ): + with _emitting_converter(converted) as mock_extra: libs = component.resolve_libraries(framework) mock_extra.assert_called_once_with( @@ -209,14 +219,7 @@ def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script"), - ): + with _emitting_converter(converted): libs = component.resolve_libraries(framework) # Wire appears once (from the explicit registration), not twice @@ -297,14 +300,7 @@ def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} ) - def fake_convert(libraries: list, backend: LibraryBackend) -> list: - backend.emit(converted) - return [converted] - - with ( - patch.object(component, "convert_libraries", side_effect=fake_convert), - patch.object(component, "apply_extra_script"), - ): + with _emitting_converter(converted): libs = component.resolve_libraries(framework) assert [lib.name for lib in libs] == ["some__External"]