Merge branch 'esp8266-native-build-spec' into esp8266-native-ninja-emission

# Conflicts:
#	tests/unit_tests/build_gen/test_arduino8266.py

esphome/arduino/library.py

@@ -28,7 +28,6 @@ from esphome.platformio.library import (
     ConvertedLibrary,
     InvalidLibrary,
     LibraryBackend,
-    _parse_library_json,
     check_library_data,
     collect_filtered_files,
     convert_libraries,
@@ -37,6 +36,7 @@ from esphome.platformio.library import (
     lex_build_flags,
     lib_ignore_set,
     normalize_dependencies,
+    parse_library_json,
     parse_library_properties,
 )
 
@@ -138,7 +138,7 @@ def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary:
     lib_dir = framework_path / "libraries" / name
     manifest_json = lib_dir / "library.json"
     if manifest_json.is_file():
-        data = _parse_library_json(manifest_json)
+        data = parse_library_json(manifest_json)
     else:
         manifest = lib_dir / "library.properties"
         data = parse_library_properties(manifest) if manifest.is_file() else {}
@@ -174,6 +174,9 @@ def resolve_libraries(
             and "/" not in library.name
             and (framework_path / "libraries" / library.name).is_dir()
         ):
+            # A bundled library's own manifest dependencies are deliberately
+            # not walked (PlatformIO's lib_ldf_mode=off does not either);
+            # core add_library() calls list what they need explicitly.
             bundled.append(_bundled_library(framework_path, library.name))
         else:
             external.append(library)

esphome/build_gen/arduino8266.py

@@ -153,6 +153,8 @@ _CCFLAGS = [
     "-free",
     "-fipa-pta",
 ]
+# Upstream's -u _scanf_float is deliberately absent: it is re-added from
+# KEY_SCANF_FLOAT at emission (the remove_float_scanf extra script's job).
 _LINKFLAGS = [
     "-Os",
     "-nostdlib",
@@ -207,14 +209,10 @@ def _flag_defines() -> dict[str, str]:
     """Map define name -> full ``NAME[=VALUE]`` for every -D build flag."""
     defines: dict[str, str] = {}
     for flag in CORE.build_flags:
-        # Shell-lex multi-token entries the way PlatformIO does, so a knob
-        # in "-DKNOB -DOTHER" or a spaced "-D KNOB" is still detected;
-        # single tokens pass verbatim to keep quoting in their bodies intact.
-        tokens = (
-            join_flag_args(split_flag_entry(flag, "esphome"), "esphome")
-            if " " in flag
-            else (flag,)
-        )
+        # Shell-lex every entry the way PlatformIO's ParseFlags does, so a
+        # knob in "-DKNOB -DOTHER", a spaced "-D KNOB", and quoted bodies all
+        # read identically to _project_flags (and the compile line).
+        tokens = join_flag_args(split_flag_entry(flag, "esphome"), "esphome")
         for tok in tokens:
             if tok.startswith("-D") and len(tok) > 2:
                 body = tok[2:]
@@ -266,6 +264,15 @@ def _resolve_build_config(defines: dict[str, str]) -> _BuildConfig:
                 body for name, body in defines.items() if name.startswith("MMU_")
             )
         else:
+            if "MMU_IRAM_SIZE" in defines or "MMU_ICACHE_SIZE" in defines:
+                # Same diagnostic the PlatformIO builder prints: without the
+                # knob the linker script keeps the default layout while the
+                # compile line carries the custom sizes
+                _LOGGER.warning(
+                    "Detected custom MMU flags; use "
+                    "-DPIO_FRAMEWORK_ARDUINO_MMU_CUSTOM to disable the "
+                    "default configuration"
+                )
             mmu = list(_MMU_DEFAULT)
 
     return _BuildConfig(

esphome/build_gen/build_tool.py

@@ -24,9 +24,13 @@ def main() -> int:
         # Expand the response file here instead of passing @rspfile: GNU ar
         # treats backslashes in response files as escapes, corrupting Windows
         # paths ("sub\a.o" -> "suba.o").
-        objects = Path(rspfile).read_text(encoding="utf-8").split()
+        # One path per line (rspfile_content = $in_newline, written without
+        # escaping), so a path containing a space survives. Expanding into
+        # argv trades away the OS command-line length limit rspfiles dodge;
+        # the relative object paths used here stay far below it.
+        objects = Path(rspfile).read_text(encoding="utf-8").splitlines()
         return subprocess.run(
-            [ar, "rc", archive, *objects], check=False, close_fds=False
+            [ar, "rc", archive, *filter(None, objects)], check=False, close_fds=False
         ).returncode
     if mode == "copy":
         src, dst = sys.argv[2:4]

esphome/build_helpers/ninja.py

@@ -52,7 +52,10 @@ def quote_arg(tok: str) -> str:
     return f'"{quoted}"'
 
 
-_NEEDS_QUOTE = re.compile(r'[\s"\']')
+# Force-quote any token containing a character outside the shlex.quote-style
+# safe set: ninja hands POSIX commands to /bin/sh -c, so bare (, ;, <, *, `
+# and friends would be re-parsed as shell syntax.
+_NEEDS_QUOTE = re.compile(r"[^\w@%+=:,./-]")
 
 
 def shell_token(tok: str, force: bool = False) -> str:

esphome/platformio/library.py

@@ -459,7 +459,7 @@ def check_library_data(data: dict, platform: str | None, framework: str):
         )
 
 
-def _parse_library_json(library_json_path: PathType):
+def parse_library_json(library_json_path: PathType):
     """
     Load and parse a JSON file describing a library.
 
@@ -893,7 +893,7 @@ def convert_libraries(
             has_json = library_json_path.is_file()
             has_properties = library_properties_path.is_file()
         if has_json:
-            component.data = _parse_library_json(library_json_path)
+            component.data = parse_library_json(library_json_path)
         elif has_properties:
             component.data = parse_library_properties(library_properties_path)
         else:

tests/unit_tests/build_gen/test_arduino8266.py

@@ -723,3 +723,20 @@ def test_write_project_missing_src_dir_raises(tmp_path: Path) -> None:
         pytest.raises(EsphomeError, match="source directory"),
     ):
         arduino8266.write_project(paths)
+
+
+def test_build_config_custom_mmu_without_knob_warns(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Custom MMU sizes without the CUSTOM knob keep the default layout and
+    warn, as the PlatformIO builder does."""
+    _set_flags("-DMMU_IRAM_SIZE=0xC000")
+    config = _resolve_build_config(_flag_defines())
+    assert config.mmu_defines == ["MMU_IRAM_SIZE=0x8000", "MMU_ICACHE_SIZE=0x8000"]
+    assert "Detected custom MMU flags" in caplog.text
+
+
+def test_flag_defines_lexes_quoted_single_tokens() -> None:
+    """A quoted single-token define reads the same as on the compile line."""
+    _set_flags('-DMMU_SEC_HEAP="0x40108000"')
+    assert _flag_defines()["MMU_SEC_HEAP"] == "MMU_SEC_HEAP=0x40108000"

tests/unit_tests/build_helpers/test_ninja.py

@@ -62,11 +62,17 @@ def test_quote_arg_windows_argv_rule() -> None:
 
 def test_shell_token_quotes_only_when_needed() -> None:
     assert ninja_helper.shell_token("-Os") == "-Os"
-    assert ninja_helper.shell_token("-DX=$HOME") == "-DX=$$HOME"
     assert ninja_helper.shell_token("-DP=C:\\x y") == '"-DP=C:\\x y"'
     assert ninja_helper.shell_token("plain", force=True) == '"plain"'
 
 
+def test_shell_token_quotes_shell_metacharacters() -> None:
+    """Tokens like -DMASK=(1<<3) must not reach /bin/sh -c bare."""
+    assert ninja_helper.shell_token("-DMASK=(1<<3)") == '"-DMASK=(1<<3)"'
+    assert ninja_helper.shell_token("-DX=a;b") == '"-DX=a;b"'
+    assert ninja_helper.shell_token("-DX=$HOME") == '"-DX=$$HOME"'
+
+
 def test_quote_path_force_quotes() -> None:
     assert ninja_helper.quote_path(Path("a b")) == '"a b"'
     assert ninja_helper.quote_path("simple") == '"simple"'

tests/unit_tests/test_espidf_component.py

@@ -25,10 +25,10 @@ from esphome.platformio.library import (
     GitSource,
     URLSource,
     _node_key,
-    _parse_library_json,
     _resolve_registry_version,
     collect_filtered_files,
     normalize_dependencies,
+    parse_library_json,
     parse_library_properties,
     split_list_by_condition,
 )
@@ -368,11 +368,11 @@ def test_generate_idf_component_yml_missing_path_raises(tmp_component):
         generate_idf_component_yml(tmp_component)
 
 
-def test_parse_library_json(tmp_path):
+def testparse_library_json(tmp_path):
     f = tmp_path / "library.json"
     f.write_text(json.dumps({"name": "test"}))
 
-    result = _parse_library_json(f)
+    result = parse_library_json(f)
     assert result["name"] == "test"