Merge branch 'esp8266-native-ninja-emission' into esp8266-arduino-toolchain

esphome/arduino/library.py

@@ -28,7 +28,6 @@ from esphome.platformio.library import (
     ConvertedLibrary,
     InvalidLibrary,
     LibraryBackend,
-    _parse_library_json,
     check_library_data,
     collect_filtered_files,
     convert_libraries,
@@ -37,6 +36,7 @@ from esphome.platformio.library import (
     lex_build_flags,
     lib_ignore_set,
     normalize_dependencies,
+    parse_library_json,
     parse_library_properties,
 )
 
@@ -138,7 +138,7 @@ def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary:
     lib_dir = framework_path / "libraries" / name
     manifest_json = lib_dir / "library.json"
     if manifest_json.is_file():
-        data = _parse_library_json(manifest_json)
+        data = parse_library_json(manifest_json)
     else:
         manifest = lib_dir / "library.properties"
         data = parse_library_properties(manifest) if manifest.is_file() else {}
@@ -174,6 +174,9 @@ def resolve_libraries(
             and "/" not in library.name
             and (framework_path / "libraries" / library.name).is_dir()
         ):
+            # A bundled library's own manifest dependencies are deliberately
+            # not walked (PlatformIO's lib_ldf_mode=off does not either);
+            # core add_library() calls list what they need explicitly.
             bundled.append(_bundled_library(framework_path, library.name))
         else:
             external.append(library)

esphome/arduino8266/framework.py

@@ -24,14 +24,11 @@ import os
 from pathlib import Path
 from typing import NamedTuple
 
+from esphome.build_helpers.ccache import ccache_defaults_env, resolve_ccache_path
 from esphome.build_helpers.ninja import find_ninja
+from esphome.build_helpers.tools_cache import tools_cache_path
 from esphome.core import EsphomeError, Version
-from esphome.framework_helpers import (
-    ccache_defaults_env,
-    resolve_ccache_path,
-    str_to_lst_of_str,
-    tools_cache_path,
-)
+from esphome.framework_helpers import str_to_lst_of_str
 from esphome.platformio.registry import install_package
 
 _LOGGER = logging.getLogger(__name__)

esphome/build_gen/arduino8266.py

@@ -153,6 +153,8 @@ _CCFLAGS = [
     "-free",
     "-fipa-pta",
 ]
+# Upstream's -u _scanf_float is deliberately absent: it is re-added from
+# KEY_SCANF_FLOAT at emission (the remove_float_scanf extra script's job).
 _LINKFLAGS = [
     "-Os",
     "-nostdlib",
@@ -207,14 +209,10 @@ def _flag_defines() -> dict[str, str]:
     """Map define name -> full ``NAME[=VALUE]`` for every -D build flag."""
     defines: dict[str, str] = {}
     for flag in CORE.build_flags:
-        # Shell-lex multi-token entries the way PlatformIO does, so a knob
-        # in "-DKNOB -DOTHER" or a spaced "-D KNOB" is still detected;
-        # single tokens pass verbatim to keep quoting in their bodies intact.
-        tokens = (
-            join_flag_args(split_flag_entry(flag, "esphome"), "esphome")
-            if " " in flag
-            else (flag,)
-        )
+        # Shell-lex every entry the way PlatformIO's ParseFlags does, so a
+        # knob in "-DKNOB -DOTHER", a spaced "-D KNOB", and quoted bodies all
+        # read identically to _project_flags (and the compile line).
+        tokens = join_flag_args(split_flag_entry(flag, "esphome"), "esphome")
         for tok in tokens:
             if tok.startswith("-D") and len(tok) > 2:
                 body = tok[2:]
@@ -266,6 +264,15 @@ def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig:
                 body for name, body in defines.items() if name.startswith("MMU_")
             )
         else:
+            if "MMU_IRAM_SIZE" in defines or "MMU_ICACHE_SIZE" in defines:
+                # Same diagnostic the PlatformIO builder prints: without the
+                # knob the linker script keeps the default layout while the
+                # compile line carries the custom sizes
+                _LOGGER.warning(
+                    "Detected custom MMU flags; use "
+                    "-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM to disable the "
+                    "default configuration"
+                )
             mmu = list(_MMU_DEFAULT)
 
     return _BuildConfig(
@@ -495,6 +502,9 @@ def write_project(paths: InstalledPaths) -> bool:
         cache_key="arduino8266",
     )
 
+    if not src_dir.is_dir():
+        # Generated project state, not install state: clean-all would not help
+        raise EsphomeError(f"Generated source directory {src_dir} is missing")
     # A missing install directory would otherwise surface as a wall of
     # include errors; failing here names the path instead.
     include_dirs = [
@@ -505,7 +515,7 @@ def write_project(paths: InstalledPaths) -> bool:
         sdk / "lwip2" / "include",
         variant_dir,
     ]
-    for required in include_dirs:
+    for required in include_dirs[1:]:
         if not required.is_dir():
             raise EsphomeError(
                 f"{_INCOMPLETE_INSTALL}: missing {required}; {_CLEAN_HINT}"
@@ -534,7 +544,14 @@ def write_project(paths: InstalledPaths) -> bool:
         + common
         + get_project_cxx_compile_flags()
     )
-    asflags = _ASFLAGS + defines + includes + project_compile_flags
+    # PlatformIO's ASPPCOM carries defines and includes but not CCFLAGS,
+    # so only -D/-I user flags reach assembly there; match it.
+    asflags = (
+        _ASFLAGS
+        + defines
+        + includes
+        + [f for f in project_compile_flags if f.startswith(("-D", "-I"))]
+    )
 
     # build_unflags applies to the framework flag sets too (compile and link),
     # as under PlatformIO (a silently ignored ``build_unflags: -Os`` would
@@ -546,7 +563,7 @@ def write_project(paths: InstalledPaths) -> bool:
     if esp8266_data[KEY_SCANF_FLOAT]:
         link_flags += ["-u", "_scanf_float"]
     link_flags += project_link_flags
-    link_flags += [flag for lib in libraries for flag in lib.link_flags]
+    link_flags += [_shell_token(flag) for lib in libraries for flag in lib.link_flags]
     flash_ld = _active_flash_ld_name(flash_ld_name)
     link_flags += ["-T", flash_ld]
 
@@ -616,7 +633,7 @@ def write_project(paths: InstalledPaths) -> bool:
         f"asflags = {' '.join(asflags)}",
         f"linkflags = {' '.join(link_flags)}",
         f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}",
-        f"libflags = {' '.join(f'-l{lib}' for lib in system_libs)}",
+        f"libflags = {' '.join(_shell_token(f'-l{lib}') for lib in system_libs)}",
         "",
     ]
 
@@ -625,7 +642,8 @@ def write_project(paths: InstalledPaths) -> bool:
         core_exclude |= _CORE_EXCLUDE_WAVEFORM
 
     archives = []
-    variant_sources = _collect_sources(variant_dir) if variant_dir.is_dir() else []
+    # variant_dir existence was already enforced with the include dirs
+    variant_sources = _collect_sources(variant_dir)
     if variant_sources:
         objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant")
         lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}")

esphome/build_gen/build_tool.py

@@ -24,9 +24,13 @@ def main() -> int:
         # Expand the response file here instead of passing @rspfile: GNU ar
         # treats backslashes in response files as escapes, corrupting Windows
         # paths ("sub\a.o" -> "suba.o").
-        objects = Path(rspfile).read_text(encoding="utf-8").split()
+        # One path per line (rspfile_content = $in_newline, written without
+        # escaping), so a path containing a space survives. Expanding into
+        # argv trades away the OS command-line length limit rspfiles dodge;
+        # the relative object paths used here stay far below it.
+        objects = Path(rspfile).read_text(encoding="utf-8").splitlines()
         return subprocess.run(
-            [ar, "rc", archive, *objects], check=False, close_fds=False
+            [ar, "rc", archive, *filter(None, objects)], check=False, close_fds=False
         ).returncode
     if mode == "copy":
         src, dst = sys.argv[2:4]

esphome/build_helpers/ccache.py

@@ -0,0 +1,98 @@
+"""Shared ccache policy for build backends.
+
+One place for the probe, the enable/override rules, and the ``CCACHE_*``
+defaults, so the backends cannot drift apart.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+import subprocess
+
+from esphome.framework_helpers import strip_win_long_path_prefix
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def _ccache_runs(ccache: str) -> bool:
+    """Return True when the ``ccache`` found on PATH actually runs.
+
+    ``shutil.which`` proves existence, not runnability: on Windows it also
+    matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
+    target is gone. Wrapping compiles around such a find fails every compile
+    step with an opaque OS error, so probe once and fall back to compiling
+    without ccache when the probe fails.
+    """
+    try:
+        subprocess.run(
+            [ccache, "--version"],
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            timeout=15,
+            close_fds=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        _LOGGER.warning(
+            "Ignoring ccache at %s because it failed to run; compiling without ccache",
+            ccache,
+        )
+        return False
+    return True
+
+
+def resolve_ccache_path() -> str | None:
+    """The ccache binary to wrap compiles with, or None when disabled.
+
+    Shared policy for every backend: on by default when a runnable ccache is
+    on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1``
+    warns when no binary is found and skips the runnability probe. The
+    Windows extended-length prefix is stripped before probing so the probe
+    validates the exact string the build will execute (#18399).
+    """
+    import shutil
+
+    from esphome.helpers import get_bool_env
+
+    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
+    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
+        return None
+    ccache = shutil.which("ccache")
+    if ccache is None:
+        if explicit:
+            _LOGGER.warning(
+                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
+                "compiling without ccache"
+            )
+        return None
+    ccache = strip_win_long_path_prefix(ccache)
+    if not explicit and not _ccache_runs(ccache):
+        return None
+    return ccache
+
+
+def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
+    """Default ``CCACHE_*`` values for a build subprocess (not os.environ).
+
+    Values the user already set in the environment are respected. Depend
+    mode is on: both native backends emit depfiles (-MMD / CMake), which
+    keeps cache-miss overhead low.
+    """
+    from esphome.core import CORE
+
+    # build_path is set during preload for every config-loading command; unset
+    # means the caller built the environment too early. Fail loudly rather
+    # than silently drop CCACHE_BASEDIR (losing cross-device cache hits).
+    if CORE.build_path is None:
+        raise ValueError(
+            "CORE.build_path must be set before constructing the build environment"
+        )
+    defaults = {
+        "CCACHE_DIR": str(cache_dir),
+        "CCACHE_NOHASHDIR": "true",
+        "CCACHE_DEPEND": "1",
+        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
+    }
+    return {k: v for k, v in defaults.items() if k not in os.environ}

esphome/build_helpers/ninja.py

@@ -52,7 +52,10 @@ def quote_arg(tok: str) -> str:
     return f'"{quoted}"'
 
 
-_NEEDS_QUOTE = re.compile(r'[\s"\']')
+# Force-quote any token containing a character outside the shlex.quote-style
+# safe set: ninja hands POSIX commands to /bin/sh -c, so bare (, ;, <, *, `
+# and friends would be re-parsed as shell syntax.
+_NEEDS_QUOTE = re.compile(r"[^\w@%+=:,./-]")
 
 
 def shell_token(tok: str, force: bool = False) -> str:

esphome/build_helpers/tools_cache.py

@@ -0,0 +1,22 @@
+"""Machine-global tools cache location shared by the native backends."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def tools_cache_path(env_var: str, subdir: str) -> Path:
+    """A backend's machine-global tools directory, with an env override.
+
+    A blank/whitespace override is treated as unset: ``Path("")`` resolves
+    to the CWD, which ``clean-all`` would then delete.
+    """
+    import platformdirs
+
+    from esphome.helpers import get_str_env
+
+    if prefix := get_str_env(env_var, "").strip():
+        return Path(prefix).expanduser().resolve()
+    return (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir
+    ).resolve()

esphome/components/esp8266/boards.py

@@ -361,31 +361,30 @@ BOARDS = {
     },
 }
 
-"""
-ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the
-native toolchain mirrors; regenerate against the tag when bumping it):
-
-git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266
-python3 - <<'EOF'
-import json, glob, os
-for f in sorted(glob.glob("platform-espressif8266/boards/*.json")):
-    b = json.load(open(f))["build"]
-    extra = b["extra_flags"]
-    extra = extra.split() if isinstance(extra, str) else extra
-    defines = [
-        e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266")
-    ]
-    entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "")
-    board = os.path.splitext(os.path.basename(f))[0]
-    print(f'    "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},')
-EOF
-"""
 
 # Per-board Arduino core build metadata for the native (PlatformIO-free)
 # toolchain: the variant directory (supplies pins_arduino.h) and the
 # board-identity defines the PlatformIO builder passes via build.extra_flags.
 # -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by
 # the generator; only the per-board defines are listed here.
+#
+# ESP8266_BOARD_BUILD generate with (v4.2.1 is the platform version the
+# native toolchain mirrors; regenerate against the tag when bumping it):
+#
+# git clone -b v4.2.1 https://github.com/platformio/platform-espressif8266
+# python3 - <<'EOF'
+# import json, glob, os
+# for f in sorted(glob.glob("platform-espressif8266/boards/*.json")):
+#     b = json.load(open(f))["build"]
+#     extra = b["extra_flags"]
+#     extra = extra.split() if isinstance(extra, str) else extra
+#     defines = [
+#         e[2:] for e in extra if e not in ("-DESP8266", "-DARDUINO_ARCH_ESP8266")
+#     ]
+#     entries = ", ".join(f'"{d}"' for d in defines) + ("," if len(defines) == 1 else "")
+#     board = os.path.splitext(os.path.basename(f))[0]
+#     print(f'    "{board}": {{"variant": "{b["variant"]}", "defines": ({entries})}},')
+# EOF
 ESP8266_BOARD_BUILD = {
     "agruminolemon": {
         "variant": "agruminolemonv4",

esphome/components/esp8266/build_surgery.py

@@ -82,6 +82,14 @@ def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str
     raises, since a silently kept real memory limit would fail grouped
     builds far from the cause.
     """
+    for segment in _TESTING_SEGMENT_SIZES:
+        if segment not in segments and _segment_line_re(segment).search(content):
+            # A known segment left unpatched would keep its real memory limit
+            # and silently under-provision the testing build
+            raise RuntimeError(
+                f"Testing-mode segment {segment} is present in the linker "
+                "script but was not selected for patching"
+            )
     for segment in segments:
         if segment not in _TESTING_SEGMENT_SIZES:
             raise RuntimeError(f"Unknown testing-mode segment {segment!r}")
@@ -103,15 +111,14 @@ def segment_length(content: str, segment_name: str) -> int | None:
 
 
 def surgery_fingerprint() -> str:
-    """Fingerprint of every behavioral input to the surgeries.
+    """Fingerprint of this module's source, covering every behavioral input.
 
-    Linker-script caches include it so an edit here invalidates them.
+    Linker-script caches include it so an edit here invalidates them; hashing
+    the source over-invalidates on comment edits, which is the safe direction.
     Native-toolchain-only, like ``segment_length``; no script twin.
     """
-    parts = (
-        RATETABLE_RULE,
-        _RATETABLE_COMMENT,
-        _RATETABLE_ANCHOR.pattern,
-        repr(sorted(_TESTING_SEGMENT_SIZES.items())),
-    )
-    return hashlib.sha256("|".join(parts).encode()).hexdigest()
+    import inspect
+    import sys
+
+    source = inspect.getsource(sys.modules[__name__])
+    return hashlib.sha256(source.encode()).hexdigest()

esphome/components/nrf52/framework.py

@@ -7,6 +7,7 @@ import shutil
 import sys
 import tempfile
 
+from esphome.build_helpers.tools_cache import tools_cache_path
 import esphome.config_validation as cv
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
@@ -18,7 +19,6 @@ from esphome.framework_helpers import (
     rmdir,
     run_command_ok,
     str_to_lst_of_str,
-    tools_cache_path,
 )
 
 _LOGGER = logging.getLogger(__name__)

esphome/espidf/framework.py

@@ -11,11 +11,12 @@ import re
 import shutil
 from typing import Any, NoReturn
 
+from esphome.build_helpers.ccache import ccache_defaults_env
+from esphome.build_helpers.tools_cache import tools_cache_path
 from esphome.core import Version
 from esphome.framework_helpers import (
     PathType,
     archive_extract_all,
-    ccache_defaults_env,
     create_venv,
     download_from_mirrors,
     download_with_resume,
@@ -25,7 +26,6 @@ from esphome.framework_helpers import (
     run_command,
     run_command_ok,
     str_to_lst_of_str,
-    tools_cache_path,
 )
 from esphome.helpers import get_bool_env, write_file_if_changed
 
@@ -89,7 +89,7 @@ def get_idf_tools_path() -> Path:
         Path object pointing to the ESP-IDF tools directory
     """
     # Machine-global so all projects share the multi-GB install instead of
-    # a per-config-directory copy; see framework_helpers.tools_cache_path
+    # a per-config-directory copy; see build_helpers.tools_cache.tools_cache_path
     # for the env-override and normalization rules.
     return tools_cache_path("ESPHOME_ESP_IDF_PREFIX", "idf")
 

esphome/framework_helpers.py

@@ -1171,23 +1171,6 @@ def download_from_mirrors(
     raise ValueError("download_from_mirrors called with an empty mirrors list")
 
 
-def tools_cache_path(env_var: str, subdir: str) -> Path:
-    """A backend's machine-global tools directory, with an env override.
-
-    A blank/whitespace override is treated as unset: ``Path("")`` resolves
-    to the CWD, which ``clean-all`` would then delete.
-    """
-    import platformdirs
-
-    from esphome.helpers import get_str_env
-
-    if prefix := get_str_env(env_var, "").strip():
-        return Path(prefix).expanduser().resolve()
-    return (
-        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / subdir
-    ).resolve()
-
-
 def strip_win_long_path_prefix(path: str) -> str:
     r"""Strip the Windows extended-length path prefix from ``path``.
 
@@ -1220,85 +1203,3 @@ def strip_win_long_path_prefix(path: str) -> str:
     if path.startswith("\\\\?\\"):
         return path[len("\\\\?\\") :]
     return path
-
-
-def _ccache_runs(ccache: str) -> bool:
-    """Return True when the ``ccache`` found on PATH actually runs.
-
-    ``shutil.which`` proves existence, not runnability: on Windows it also
-    matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
-    target is gone. Wrapping compiles around such a find fails every compile
-    step with an opaque OS error, so probe once and fall back to compiling
-    without ccache when the probe fails.
-    """
-    try:
-        subprocess.run(
-            [ccache, "--version"],
-            check=True,
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-            timeout=15,
-            close_fds=False,
-        )
-    except (OSError, subprocess.SubprocessError):
-        _LOGGER.warning(
-            "Ignoring ccache at %s because it failed to run; compiling without ccache",
-            ccache,
-        )
-        return False
-    return True
-
-
-def resolve_ccache_path() -> str | None:
-    """The ccache binary to wrap compiles with, or None when disabled.
-
-    Shared policy for every backend: on by default when a runnable ccache is
-    on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1``
-    warns when no binary is found and skips the runnability probe. The
-    Windows extended-length prefix is stripped before probing so the probe
-    validates the exact string the build will execute (#18399).
-    """
-    import shutil
-
-    from esphome.helpers import get_bool_env
-
-    explicit = "ESPHOME_CCACHE_ENABLE" in os.environ
-    if explicit and not get_bool_env("ESPHOME_CCACHE_ENABLE"):
-        return None
-    ccache = shutil.which("ccache")
-    if ccache is None:
-        if explicit:
-            _LOGGER.warning(
-                "ESPHOME_CCACHE_ENABLE is set but no ccache binary is on PATH; "
-                "compiling without ccache"
-            )
-        return None
-    ccache = strip_win_long_path_prefix(ccache)
-    if not explicit and not _ccache_runs(ccache):
-        return None
-    return ccache
-
-
-def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
-    """Default ``CCACHE_*`` values for a build subprocess (not os.environ).
-
-    Values the user already set in the environment are respected. Depend
-    mode is on: both native backends emit depfiles (-MMD / CMake), which
-    keeps cache-miss overhead low.
-    """
-    from esphome.core import CORE
-
-    # build_path is set during preload for every config-loading command; unset
-    # means the caller built the environment too early. Fail loudly rather
-    # than silently drop CCACHE_BASEDIR (losing cross-device cache hits).
-    if CORE.build_path is None:
-        raise ValueError(
-            "CORE.build_path must be set before constructing the build environment"
-        )
-    defaults = {
-        "CCACHE_DIR": str(cache_dir),
-        "CCACHE_NOHASHDIR": "true",
-        "CCACHE_DEPEND": "1",
-        "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
-    }
-    return {k: v for k, v in defaults.items() if k not in os.environ}

esphome/platformio/extra_script.py

@@ -35,7 +35,7 @@ import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from esphome.platformio.library import ensure_list
+from esphome.core import EsphomeError
 
 if TYPE_CHECKING:
     from esphome.platformio.library import ConvertedLibrary
@@ -89,9 +89,17 @@ def apply_extra_script(
     extra_flags = captured_as_build_flags(result, library_dir=source_path)
     if not extra_flags:
         return
-    flags = ensure_list(component.data.setdefault("build", {}).setdefault("flags", []))
-    flags.extend(extra_flags)
-    component.data["build"]["flags"] = flags
+    flags = component.data.setdefault("build", {}).setdefault("flags", [])
+    if isinstance(flags, str):
+        flags = [flags]
+    elif not isinstance(flags, list):
+        # A null/dict value coerced through a list wrapper would inject a
+        # non-string into the compiler command line; fail naming the library
+        raise EsphomeError(
+            f"Library {component.name} has a malformed build.flags "
+            f"({type(flags).__name__}); expected a string or list"
+        )
+    component.data["build"]["flags"] = [*flags, *extra_flags]
 
 
 # Keys we know how to translate back into ESPHome's build-flag pipeline.

esphome/platformio/library.py

@@ -459,7 +459,7 @@ def check_library_data(data: dict, platform: str | None, framework: str):
         )
 
 
-def _parse_library_json(library_json_path: PathType):
+def parse_library_json(library_json_path: PathType):
     """
     Load and parse a JSON file describing a library.
 
@@ -893,7 +893,7 @@ def convert_libraries(
             has_json = library_json_path.is_file()
             has_properties = library_properties_path.is_file()
         if has_json:
-            component.data = _parse_library_json(library_json_path)
+            component.data = parse_library_json(library_json_path)
         elif has_properties:
             component.data = parse_library_properties(library_properties_path)
         else:

esphome/platformio/toolchain.py

@@ -9,9 +9,10 @@ from typing import TYPE_CHECKING, Any
 
 import platformdirs
 
+from esphome.build_helpers.ccache import resolve_ccache_path
 from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE
 from esphome.core import CORE, EsphomeError
-from esphome.framework_helpers import resolve_ccache_path, strip_win_long_path_prefix
+from esphome.framework_helpers import strip_win_long_path_prefix
 from esphome.helpers import (
     add_git_ceiling_directory,
     copy_file_if_changed,

tests/unit_tests/build_gen/test_arduino8266.py

@@ -442,7 +442,6 @@ def test_generate_ld_scripts_testing_mode(tmp_path: Path) -> None:
     (paths.framework / "tools" / "sdk" / "ld" / "eagle.flash.4m.ld").write_text(
         "MEMORY\n{\n"
         "  dram0_0_seg :    org = 0x3FFE8000, len = 0x14000\n"
-        "  iram1_0_seg :    org = 0x40100000, len = 0x8000\n"
         "  irom0_0_seg :    org = 0x40201010, len = 0xfeff0\n"
         "}\n"
     )
@@ -677,3 +676,67 @@ def test_ninja_path_escaping() -> None:
     """Build-statement paths and command-line paths escape differently."""
     assert arduino8266._e("a b:$c") == "a$ b$:$$c"
     assert arduino8266._q("/a b/$x") == '"/a b/$$x"'
+
+
+def test_write_project_asm_excludes_non_define_user_flags(tmp_path: Path) -> None:
+    """The ASPPCOM command under PlatformIO never sees CCFLAGS, so only -D/-I user flags
+    reach assembly compiles."""
+    paths = _make_framework(tmp_path)
+    _set_flags("-DUSER_KNOB=1", "-Wno-volatile")
+    content = _write_ninja(paths)
+    asflags = next(line for line in content.splitlines() if line.startswith("asflags"))
+    assert "-DUSER_KNOB=1" in asflags
+    assert "-Wno-volatile" not in asflags
+    cxxflags = next(
+        line for line in content.splitlines() if line.startswith("cxxflags")
+    )
+    assert "-Wno-volatile" in cxxflags
+
+
+def test_write_project_returns_changed(tmp_path: Path) -> None:
+    """The documented contract: True when build.ninja changed, False on an
+    identical regeneration (pins byte-stable output too)."""
+    paths = _make_framework(tmp_path)
+    _set_flags()
+    src = CORE.relative_src_path()
+    (src / "esphome" / "components" / "esp8266").mkdir(parents=True, exist_ok=True)
+    (src / "main.cpp").write_text("")
+    with (
+        patch.object(arduino8266, "generate_ld_scripts"),
+        patch("esphome.arduino.library.resolve_libraries", return_value=[]),
+        patch("esphome.arduino8266.framework.ccache_path", return_value=None),
+    ):
+        assert arduino8266.write_project(paths) is True
+        assert arduino8266.write_project(paths) is False
+
+
+def test_write_project_missing_src_dir_raises(tmp_path: Path) -> None:
+    """A missing generated source tree is its own error, not an install one."""
+    paths = _make_framework(tmp_path)
+    _set_flags()
+    with (
+        patch.object(arduino8266, "generate_ld_scripts"),
+        patch("esphome.arduino.library.resolve_libraries", return_value=[]),
+        patch.object(
+            arduino8266.CORE, "relative_src_path", return_value=tmp_path / "nope"
+        ),
+        pytest.raises(EsphomeError, match="source directory"),
+    ):
+        arduino8266.write_project(paths)
+
+
+def test_build_config_custom_mmu_without_knob_warns(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Custom MMU sizes without the CUSTOM knob keep the default layout and
+    warn, as the PlatformIO builder does."""
+    _set_flags("-DMMU_IRAM_SIZE=0xC000")
+    config = _resolve_build_config(_flag_defines())
+    assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"]
+    assert "Detected custom MMU flags" in caplog.text
+
+
+def test_flag_defines_lexes_quoted_single_tokens() -> None:
+    """A quoted single-token define reads the same as on the compile line."""
+    _set_flags('-DMMU_SEC_HEAP="0x40108000"')
+    assert _flag_defines()["MMU_SEC_HEAP"] == "MMU_SEC_HEAP=0x40108000"

tests/unit_tests/build_helpers/test_idedata.py

@@ -451,3 +451,17 @@ def test_load_or_build_idedata_never_caches_bad_compiler(tmp_path: Path) -> None
         )
     assert data["cxx_path"] == "/usr/bin/python3"
     assert not cache.exists()
+
+
+def test_load_or_build_idedata_cache_hit_skips_rebuild(tmp_path: Path) -> None:
+    """A valid cache newer than the compile DB is served without re-parsing."""
+    compile_commands = _write_compile_commands(tmp_path)
+    cache = tmp_path / "c.json"
+    cache.write_text(json.dumps({"cc_path": "/tools/gcc", "cached": True}))
+    os.utime(cache, (compile_commands.stat().st_mtime + 5,) * 2)
+    with patch.object(idedata, "idedata_from_build") as mock_build:
+        data = idedata.load_or_build_idedata(
+            compile_commands, tmp_path / "f.elf", cache
+        )
+    mock_build.assert_not_called()
+    assert data["cached"] is True

tests/unit_tests/build_helpers/test_ninja.py

@@ -48,3 +48,31 @@ def test_find_ninja_missing_everywhere(tmp_path: Path) -> None:
         pytest.raises(EsphomeError, match="ninja not found"),
     ):
         ninja_helper.find_ninja()
+
+
+def test_escape_ninja_specials() -> None:
+    assert ninja_helper.escape("a b:c$d") == "a$ b$:c$$d"
+
+
+def test_quote_arg_windows_argv_rule() -> None:
+    # Backslash runs double only before a quote (subprocess.list2cmdline rule)
+    assert ninja_helper.quote_arg('-DX=a\\"b c') == '"-DX=a\\\\\\"b c"'
+    assert ninja_helper.quote_arg("a b\\") == '"a b\\\\"'
+
+
+def test_shell_token_quotes_only_when_needed() -> None:
+    assert ninja_helper.shell_token("-Os") == "-Os"
+    assert ninja_helper.shell_token("-DP=C:\\x y") == '"-DP=C:\\x y"'
+    assert ninja_helper.shell_token("plain", force=True) == '"plain"'
+
+
+def test_shell_token_quotes_shell_metacharacters() -> None:
+    """Tokens like -DMASK=(1<<3) must not reach /bin/sh -c bare."""
+    assert ninja_helper.shell_token("-DMASK=(1<<3)") == '"-DMASK=(1<<3)"'
+    assert ninja_helper.shell_token("-DX=a;b") == '"-DX=a;b"'
+    assert ninja_helper.shell_token("-DX=$HOME") == '"-DX=$$HOME"'
+
+
+def test_quote_path_force_quotes() -> None:
+    assert ninja_helper.quote_path(Path("a b")) == '"a b"'
+    assert ninja_helper.quote_path("simple") == '"simple"'

tests/unit_tests/components/esp8266/test_build_surgery.py

@@ -49,7 +49,8 @@ def test_relocate_ratetable_inserts_after_data_start() -> None:
     patched = relocate_ratetable(_COMMON_LD_SNIPPET)
     assert RATETABLE_RULE in patched
     # Inserted after the .data section's anchor, not the .dport0.data one
-    assert patched.index("_data_start = ABSOLUTE(.);") < patched.index(RATETABLE_RULE)
+    # (whose closing brace bounds the decoy block)
+    assert RATETABLE_RULE not in patched[: patched.index("} >dport0_0_seg")]
     assert patched.index(RATETABLE_RULE) < patched.index("*(.data)")
     # Idempotent on an already-patched script
     assert relocate_ratetable(patched) == patched
@@ -106,14 +107,20 @@ def test_board_build_covers_every_board() -> None:
     assert set(BOARDS) <= set(ESP8266_BOARD_BUILD)
 
 
-def test_surgery_fingerprint_tracks_inputs() -> None:
-    """The fingerprint changes with any behavioral input, so linker-script
-    caches stamped with it self-invalidate on surgery edits."""
-    from unittest.mock import patch
+def test_surgery_fingerprint_covers_module_source() -> None:
+    """The fingerprint hashes the module source, so any surgery edit
+    invalidates linker-script caches stamped with it."""
+    import hashlib
+    import inspect
 
     from esphome.components.esp8266 import build_surgery
 
-    base = build_surgery.surgery_fingerprint()
-    assert base == build_surgery.surgery_fingerprint()
-    with patch.object(build_surgery, "_TESTING_SEGMENT_SIZES", {"iram1_0_seg": "0x1"}):
-        assert build_surgery.surgery_fingerprint() != base
+    expected = hashlib.sha256(inspect.getsource(build_surgery).encode()).hexdigest()
+    assert build_surgery.surgery_fingerprint() == expected
+
+
+def test_testing_memory_patches_present_but_unselected_raises() -> None:
+    """A known segment left off the caller's list must fail, not silently
+    keep its real memory limit."""
+    with pytest.raises(RuntimeError, match="not selected"):
+        apply_testing_memory_patches(_FLASH_LD_SNIPPET, ("dram0_0_seg",))

tests/unit_tests/test_arduino8266_framework.py

@@ -103,7 +103,7 @@ def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> No
     monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1")
     with (
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers._ccache_runs", side_effect=AssertionError),
+        patch("esphome.build_helpers.ccache._ccache_runs", side_effect=AssertionError),
     ):
         assert framework.ccache_path() == "/usr/bin/ccache"
 

tests/unit_tests/test_espidf_component.py

@@ -25,10 +25,10 @@ from esphome.platformio.library import (
     GitSource,
     URLSource,
     _node_key,
-    _parse_library_json,
     _resolve_registry_version,
     collect_filtered_files,
     normalize_dependencies,
+    parse_library_json,
     parse_library_properties,
     split_list_by_condition,
 )
@@ -368,11 +368,11 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component):
         generate_idf_component_yml(tmp_component)
 
 
-def test_parse_library_json(tmp_path):
+def testparse_library_json(tmp_path):
     f = tmp_path / "library.json"
     f.write_text(json.dumps({"name": "test"}))
 
-    result = _parse_library_json(f)
+    result = parse_library_json(f)
     assert result["name"] == "test"