Merge branch 'esp8266-native-library-backend' into esp8266-native-build-spec

esphome/arduino8266/component.py

@@ -31,11 +31,10 @@ from esphome.platformio.library import (
     convert_libraries,
     ensure_list,
     is_lib_ignored,
-    join_flag_args,
+    lex_build_flags,
     lib_ignore_set,
     normalize_dependencies,
     parse_library_properties,
-    split_flag_entry,
 )
 
 _LOGGER = logging.getLogger(__name__)
@@ -77,14 +76,7 @@ def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary:
 
     src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER))
     # PlatformIO shell-lexes each build.flags entry
-    flag_tokens = join_flag_args(
-        (
-            token
-            for entry in ensure_list(build.get("flags", []))
-            for token in split_flag_entry(entry, f"library {name}")
-        ),
-        f"library {name}",
-    )
+    flag_tokens = lex_build_flags(build.get("flags", []), f"library {name}")
 
     lib = ArduinoLibrary(name=name)
     include_flags: list[str] = []
@@ -153,20 +145,20 @@ def resolve_libraries(framework_path: Path) -> list[ArduinoLibrary]:
     for library in CORE.platformio_libraries.values():
         if is_lib_ignored(library.name, lib_ignore):
             continue
-        # A version pin means a registry package ("pngle@1.1.0"), never a
-        # framework-bundled library.
+        # Only a bare name with a matching framework directory is bundled: a
+        # version pin means a registry package ("pngle@1.1.0"), and a bare
+        # name without the directory resolves from the registry at the
+        # latest version, matching PlatformIO (a typo fails loudly as a
+        # registry lookup error).
         if (
-            library.repository
-            or library.version
-            or not library.name
-            or "/" in library.name
+            not library.repository
+            and not library.version
+            and library.name
+            and "/" not in library.name
+            and (framework_path / "libraries" / library.name).is_dir()
         ):
-            external.append(library)
-        elif (framework_path / "libraries" / library.name).is_dir():
             bundled.append(_bundled_library(framework_path, library.name))
         else:
-            # A bare registry name; resolved at the latest version, matching
-            # PlatformIO (a typo fails loudly as a registry lookup error).
             external.append(library)
 
     converted: list[ArduinoLibrary] = []

esphome/components/esp8266/build_surgery.py

@@ -11,11 +11,15 @@ as plain functions. Keep both in sync when changing either.
 from __future__ import annotations
 
 from collections.abc import Collection
+import hashlib
 import re
 
 # Move the NONOS SDK wifi rate tables from flash to DRAM; see
 # relocate_ratetable.py.script for the full background (NONOS SDK issue 320).
 RATETABLE_RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)"
+_RATETABLE_COMMENT = (
+    "/* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */"
+)
 # Match the whole line: "_data_start" is also a substring of the
 # "_dport0_data_start" line in the earlier .dport0.data section
 _RATETABLE_ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE)
@@ -40,7 +44,7 @@ def relocate_ratetable(content: str) -> str:
     insert_pos = match.end()
     return (
         content[:insert_pos]
-        + "\n    /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */"
+        + f"\n    {_RATETABLE_COMMENT}"
         + f"\n    {RATETABLE_RULE}"
         + content[insert_pos:]
     )
@@ -84,3 +88,18 @@ def segment_length(content: str, segment_name: str) -> int | None:
     """Read a memory segment's length from linker script content."""
     match = _segment_line_re(segment_name).search(content)
     return int(match.group(2), 16) if match else None
+
+
+def surgery_fingerprint() -> str:
+    """Fingerprint of every behavioral input to the surgeries.
+
+    Linker-script caches include it so an edit here invalidates them.
+    Native-toolchain-only, like ``segment_length``; no script twin.
+    """
+    parts = (
+        RATETABLE_RULE,
+        _RATETABLE_COMMENT,
+        _RATETABLE_ANCHOR.pattern,
+        repr(sorted(_TESTING_SEGMENT_SIZES.items())),
+    )
+    return hashlib.sha256("|".join(parts).encode()).hexdigest()

esphome/espidf/component.py

@@ -27,8 +27,7 @@ from esphome.platformio.library import (
     collect_filtered_files,
     convert_libraries,
     ensure_list,
-    join_flag_args,
-    split_flag_entry,
+    lex_build_flags,
     split_list_by_condition,
 )
 
@@ -107,14 +106,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str:
     # flag and its argument (e.g. "-include cp_custom_alloc.h"); bare
     # -I/-L/-l tokens re-glue to their argument ("-I foo" -> "-Ifoo") so the
     # prefix classifiers below still route them.
-    build_flags = join_flag_args(
-        (
-            token
-            for entry in build_flags
-            for token in split_flag_entry(entry, f"library {component.name}")
-        ),
-        f"library {component.name}",
-    )
+    build_flags = lex_build_flags(build_flags, f"library {component.name}")
 
     # List all sources files
     build_src_files = collect_filtered_files(

esphome/platformio/library.py

@@ -562,6 +562,23 @@ def split_flag_entry(entry: str, owner: str) -> list[str]:
         raise EsphomeError(f"Malformed build flag {entry!r} in {owner}: {err}") from err
 
 
+def lex_build_flags(entries, owner: str) -> list[str]:
+    """Shell-lex a manifest ``build.flags`` list into joined tokens.
+
+    The composition every backend needs: each entry is lexed the way
+    PlatformIO's ParseFlags does, and bare ``-I``/``-L``/``-l``/``-D``
+    tokens re-glue to their argument across the whole stream.
+    """
+    return join_flag_args(
+        (
+            token
+            for entry in ensure_list(entries)
+            for token in split_flag_entry(entry, owner)
+        ),
+        owner,
+    )
+
+
 def join_flag_args(tokens: Iterable[str], owner: str) -> list[str]:
     """Join a bare ``-I``/``-L``/``-l``/``-D`` with its following token,
     the way PlatformIO's ParseFlags lexes them."""

tests/unit_tests/build_gen/test_build_tool.py

@@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
-from esphome.arduino8266 import build_tool
+from esphome.build_gen import build_tool
 
 
 def test_ar_removes_stale_archive(tmp_path: Path) -> None:

tests/unit_tests/components/esp8266/test_build_surgery.py

@@ -77,3 +77,16 @@ def test_board_build_covers_every_board() -> None:
     """Every supported board has native build metadata (the table may carry
     extras that BOARDS does not expose)."""
     assert set(BOARDS) <= set(ESP8266_BOARD_BUILD)
+
+
+def test_surgery_fingerprint_tracks_inputs() -> None:
+    """The fingerprint changes with any behavioral input, so linker-script
+    caches stamped with it self-invalidate on surgery edits."""
+    from unittest.mock import patch
+
+    from esphome.components.esp8266 import build_surgery
+
+    base = build_surgery.surgery_fingerprint()
+    assert base == build_surgery.surgery_fingerprint()
+    with patch.object(build_surgery, "_TESTING_SEGMENT_SIZES", {"iram1_0_seg": "0x1"}):
+        assert build_surgery.surgery_fingerprint() != base

tests/unit_tests/test_arduino8266_component.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from contextlib import contextmanager
 from pathlib import Path
 from unittest.mock import patch
 
@@ -38,6 +39,25 @@ def _make_framework(tmp_path: Path) -> Path:
     return framework
 
 
+@contextmanager
+def _emitting_converter(*converted):
+    """Patch convert_libraries to emit the given components via the backend."""
+
+    def fake_convert(libraries: list, backend: LibraryBackend) -> list:
+        assert backend.platform == "espressif8266"
+        assert backend.framework == "arduino"
+        assert backend.cache_key == "arduino8266"
+        for c in converted:
+            backend.emit(c)
+        return list(converted)
+
+    with (
+        patch.object(component, "convert_libraries", side_effect=fake_convert),
+        patch.object(component, "apply_extra_script") as mock_extra,
+    ):
+        yield mock_extra
+
+
 def test_library_info_src_layout(tmp_path: Path) -> None:
     framework = _make_framework(tmp_path)
     lib = component._bundled_library(framework, "ESP8266WiFi")
@@ -176,17 +196,7 @@ def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None:
         },
     )
 
-    def fake_convert(libraries: list, backend: LibraryBackend) -> list:
-        assert backend.platform == "espressif8266"
-        assert backend.framework == "arduino"
-        assert backend.cache_key == "arduino8266"
-        backend.emit(converted)
-        return [converted]
-
-    with (
-        patch.object(component, "convert_libraries", side_effect=fake_convert),
-        patch.object(component, "apply_extra_script") as mock_extra,
-    ):
+    with _emitting_converter(converted) as mock_extra:
         libs = component.resolve_libraries(framework)
 
     mock_extra.assert_called_once_with(
@@ -209,14 +219,7 @@ def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None:
         "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]}
     )
 
-    def fake_convert(libraries: list, backend: LibraryBackend) -> list:
-        backend.emit(converted)
-        return [converted]
-
-    with (
-        patch.object(component, "convert_libraries", side_effect=fake_convert),
-        patch.object(component, "apply_extra_script"),
-    ):
+    with _emitting_converter(converted):
         libs = component.resolve_libraries(framework)
 
     # Wire appears once (from the explicit registration), not twice
@@ -297,14 +300,7 @@ def test_resolve_libraries_lib_ignore_covers_bundled_dependencies(
         "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]}
     )
 
-    def fake_convert(libraries: list, backend: LibraryBackend) -> list:
-        backend.emit(converted)
-        return [converted]
-
-    with (
-        patch.object(component, "convert_libraries", side_effect=fake_convert),
-        patch.object(component, "apply_extra_script"),
-    ):
+    with _emitting_converter(converted):
         libs = component.resolve_libraries(framework)
 
     assert [lib.name for lib in libs] == ["some__External"]