Merge branch 'esp8266-native-ninja-emission' into esp8266-arduino-toolchain

# Conflicts:
#	esphome/components/esp8266/__init__.py

esphome/build_gen/arduino8266.py

@@ -14,7 +14,6 @@ from the build flags with the same precedence as the PlatformIO builder.
 
 from __future__ import annotations
 
-from contextlib import suppress
 from dataclasses import dataclass
 import hashlib
 import logging
@@ -35,7 +34,7 @@ from esphome.components.esp8266 import build_surgery
 from esphome.components.esp8266.boards import (
     BOARDS,
     ESP8266_BOARD_BUILD,
-    ESP8266_LD_SCRIPTS,
+    board_ld_script,
 )
 from esphome.components.esp8266.const import (
     KEY_BOARD,
@@ -108,6 +107,8 @@ def _apply_surgery(fn, *args: object) -> str:
 
 
 # Every supported board's f_flash is 40 MHz; re-check on a platform bump
+# board_flash_mode's closed set (cv.one_of in esp8266/__init__.py)
+_FLASH_MODES = frozenset({"qio", "qout", "dio", "dout"})
 _FLASH_FREQ_MHZ = 40
 
 # From platformio-build.py. Knob suffix -> SDK define; the first entry is
@@ -154,6 +155,10 @@ _LWIP_VARIANTS = {
         1460, 0, 0, "lwip2-1460"
     ),
 }
+# The default is PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY's variant: upstream
+# has no branch for that spelling (it is the else), so any listed knob wins
+# over it -- sntp emits LOW_MEMORY while esp8266 always emits
+# HIGHER_BANDWIDTH_LOW_FLASH, and the latter must win as under PlatformIO
 _LWIP_DEFAULT = _LwipVariant(536, 1, 0, "lwip2-536-feat")
 
 # Knob define -> MMU_* defines; first match wins, in insertion order (as
@@ -451,12 +456,9 @@ def _flash_ld_name(board: str) -> str:
     """
     override = _pio_option("board_build.ldscript", "")
     if not override:
-        # The same per-board override the PlatformIO path pins (layout
-        # preservation, see boards.py)
-        board_data = BOARDS[board]
-        return board_data.get(
-            "ldscript", ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]][1]
-        )
+        # The same shared rule the PlatformIO path pins (layout
+        # preservation, see boards.board_ld_script)
+        return board_ld_script(BOARDS[board])
     if Path(override).name != override:
         raise EsphomeError(
             f"board_build.ldscript must be a bare script name, got {override!r}"
@@ -708,8 +710,13 @@ def generate_ld_scripts(
             _LOGGER.warning("Linker-script preprocessor: %s", result.stderr.strip())
             note_persisted = _write_note(stderr_note, result.stderr.strip(), warn=True)
         else:
-            with suppress(OSError):
+            try:
                 stderr_note.unlink(missing_ok=True)
+            except OSError as err:
+                # A kept stale note would re-emit an obsolete diagnostic on
+                # every cache hit; skip the stamp so -E re-derives the truth
+                _LOGGER.debug("Could not remove %s: %s", stderr_note, err)
+                note_persisted = False
         if "SECTIONS" not in result.stdout:
             # A degenerate zero-exit run must not be stamped as a good cache
             raise EsphomeError(
@@ -735,11 +742,12 @@ def generate_ld_scripts(
                 "Linker-script preprocessor: %s",
                 stderr_note.read_text(encoding="utf-8"),
             )
-        except (OSError, UnicodeDecodeError):
+        except (OSError, UnicodeDecodeError) as err:
             _LOGGER.warning(
                 "A cached linker-script preprocessor diagnostic exists at %s "
-                "but could not be read",
+                "but could not be read: %s",
                 stderr_note,
+                err,
             )
 
     if CORE.testing_mode:
@@ -817,12 +825,17 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
     config = _resolve_build_config(flag_defines)
     esp8266_data = CORE.data[KEY_ESP8266]
     board = esp8266_data[KEY_BOARD]
-    # Config validation (_validate_native_toolchain) already gates boards;
+    # Config validation already gates boards;
     # kept as defense-in-depth for direct calls, since CONF_BOARD itself is
     # a free-form string
     if board not in ESP8266_BOARD_BUILD:
         raise EsphomeError(f"Board '{board}' is not supported by the native toolchain")
     board_build = ESP8266_BOARD_BUILD[board]
+    flash_mode = esp8266_data[KEY_FLASH_MODE]
+    if flash_mode not in _FLASH_MODES:
+        # Lands unquoted in the elf2bin command and a -D body; validation
+        # (cv.one_of on board_flash_mode) already gates it, defense-in-depth
+        raise EsphomeError(f"Invalid flash mode {flash_mode!r}")
     flash_ld_name = _flash_ld_name(board)
 
     generate_ld_scripts(paths, config, flash_ld_name)
@@ -876,9 +889,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         project_lib_dirs,
         project_libs,
     ) = _project_flags(unflags, build_tokens)
-    defines = _defines_flags(
-        config, esp8266_data[KEY_FLASH_MODE], board, board_build["defines"]
-    )
+    defines = _defines_flags(config, flash_mode, board, board_build["defines"])
     includes = [f"-I{_q(d)}" for d in include_dirs]
 
     common = _CCFLAGS + defines + includes + project_compile_flags
@@ -995,7 +1006,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         "rule elf2bin",
         # --flash_size deliberately stays board-derived, as under
         # PlatformIO (which reads upload.maximum_size, not the ldscript).
-        f"  command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {esp8266_data[KEY_FLASH_MODE]} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out",
+        f"  command = $python {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out",
         "  description = BIN $out",
         "rule copy",
         "  command = $python $buildtool copy $in $out",
@@ -1062,7 +1073,19 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         lines.append(f"build {_e(archive)}: ar {' '.join(objs)}")
         archives.append(archive)
 
-    src_extra = f"-include {_q(src_dir / 'esphome' / 'components' / 'esp8266' / 'throw_stubs.h')}"
+    # One source of truth with the PlatformIO path: esp8266/__init__ pins
+    # build_src_flags (the throw_stubs force-include); -include paths
+    # resolve against the source root
+    src_parts: list[str] = []
+    src_it = iter(
+        lex_build_flags(_pio_option("build_src_flags", ""), "build_src_flags")
+    )
+    for tok in src_it:
+        if tok == "-include":
+            src_parts.append(f"-include {_q(src_dir / next(src_it, ''))}")
+        else:
+            src_parts.append(_shell_token(tok))
+    src_extra = " ".join(src_parts)
     # One shared variable instead of repeating the flags line on every src
     # edge (hundreds of edges in a real project)
     lines.append(f"srcflags = {src_extra}")

esphome/components/esp8266/__init__.py

@@ -38,7 +38,7 @@ from esphome.platformio.toolchain import copy_ccache_script
 from esphome.storage_json import StorageJSON
 from esphome.types import ConfigType
 
-from .boards import BOARDS, ESP8266_BOARD_BUILD, ESP8266_LD_SCRIPTS
+from .boards import BOARDS, ESP8266_BOARD_BUILD, ESP8266_LD_SCRIPTS, board_ld_script
 from .const import (
     CONF_EARLY_PIN_INIT,
     CONF_ENABLE_SERIAL,
@@ -503,7 +503,7 @@ async def to_code(config: ConfigType) -> None:
         else:
             # A per-board override preserves a layout the board shipped
             # with (see d1_wroom_02 in boards.py)
-            ld_script = board_data.get("ldscript", ld_scripts[1])
+            ld_script = board_ld_script(board_data)
 
         if ld_script is not None:
             cg.add_platformio_option("board_build.ldscript", ld_script)

esphome/components/esp8266/boards.py

@@ -1,3 +1,5 @@
+from .const import KEY_FLASH_SIZE
+
 FLASH_SIZE_1_MB = 2**20
 FLASH_SIZE_512_KB = FLASH_SIZE_1_MB // 2
 FLASH_SIZE_2_MB = 2 * FLASH_SIZE_1_MB
@@ -182,6 +184,17 @@ for x in platform-espressif8266/boards/*.json; do
 done | sort
 """
 
+
+def board_ld_script(board_data: dict) -> str:
+    """The modern (core > 2.4.2) flash linker script for a board: its
+    shipped-layout override, else the size default (the no-FS layout).
+
+    Single source of truth for the PlatformIO pinning in __init__ and the
+    native generator's fallback, so the per-board rule cannot drift.
+    """
+    return board_data.get("ldscript", ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]][1])
+
+
 BOARDS = {
     "agruminolemon": {
         "name": "Lifely Agrumino Lemon v4",

tests/unit_tests/build_gen/test_arduino8266.py

@@ -51,6 +51,10 @@ def _setup_core(tmp_path: Path) -> Generator[None]:
         KEY_FLASH_MODE: "dout",
         KEY_SCANF_FLOAT: False,
     }
+    # The producer esp8266/__init__ pins unconditionally
+    CORE.platformio_options = {
+        "build_src_flags": "-include esphome/components/esp8266/throw_stubs.h"
+    }
     yield
     # CORE.reset() (the suite-wide autouse fixture) does not clear this flag
     CORE.testing_mode = False
@@ -345,6 +349,8 @@ def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None:
         ),
         ("PIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH", "lwip2-1460-feat", 1460, 1, 0),
         ("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY_LOW_FLASH", "lwip2-536", 536, 0, 0),
+        # LOW_MEMORY has no upstream branch: it is the default (else) variant
+        ("PIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY", "lwip2-536-feat", 536, 1, 0),
     ],
 )
 def test_build_config_lwip_variants(
@@ -359,6 +365,17 @@ def test_build_config_lwip_variants(
     assert f"LWIP_IPV6={ipv6}" in config.knob_defines
 
 
+def test_lwip_low_memory_loses_to_listed_knobs() -> None:
+    """The ordinary SNTP multi-server config: sntp emits LOW_MEMORY, esp8266
+    always emits HIGHER_BANDWIDTH_LOW_FLASH, and the listed knob must win
+    exactly as in platformio-build.py's elif chain."""
+    config = _resolve(
+        "-DPIO_FRAMEWORK_ARDUINO_LWIP2_LOW_MEMORY",
+        "-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH",
+    )
+    assert config.lwip_lib == "lwip2-1460"
+
+
 @pytest.mark.parametrize(
     ("knob", "expected"),
     [
@@ -1005,6 +1022,38 @@ def test_generate_ld_scripts_lost_warn_note_vetoes_the_stamp(
     assert caplog.text.count("Linker-script preprocessor: warning: something") == 2
 
 
+def test_generate_ld_scripts_unremovable_stale_note_vetoes_the_stamp(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A stale warn note that cannot be removed skips the stamp, so the
+    obsolete diagnostic is not re-emitted on cache hits forever."""
+    paths = _make_framework(tmp_path)
+    _set_flags()
+    warn = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="warning: old")
+    clean = MagicMock(returncode=0, stdout=_COMMON_LD_H_OUTPUT, stderr="")
+    with patch.object(arduino8266.subprocess, "run", return_value=warn):
+        _run_generate_ld_scripts(paths)
+
+    real_unlink = Path.unlink
+
+    def fail_note_unlink(self: Path, missing_ok: bool = False) -> None:
+        if self.name.endswith(".stderr"):
+            raise OSError("locked")
+        real_unlink(self, missing_ok=missing_ok)
+
+    # Flags changed -> regenerate; clean stderr but the stale note is stuck
+    _set_flags("-DVTABLES_IN_DRAM")
+    with (
+        patch.object(arduino8266.subprocess, "run", return_value=clean),
+        patch.object(Path, "unlink", fail_note_unlink),
+    ):
+        _run_generate_ld_scripts(paths)
+    # Unstamped: the next build re-runs -E instead of trusting the cache
+    with patch.object(arduino8266.subprocess, "run", return_value=clean) as run3:
+        _run_generate_ld_scripts(paths)
+    run3.assert_called_once()
+
+
 def test_build_config_mmu_knob_with_raw_mmu_flag_raises() -> None:
     """A variant knob plus a raw MMU_* define would split the compile line
     from the linker script; refuse like the no-knob case."""