Lex every build_flags entry with re-quoted emission and lock the shared cache install

esphome/arduino8266/framework.py

@@ -189,6 +189,27 @@ def _install_package(
     marker = dest / ".esphome_extracted"
     if marker.is_file():
         return
+    from filelock import FileLock
+
+    # The cache is machine-global; serialize concurrent cold builds so one
+    # process cannot wipe the directory another is extracting into (same
+    # filelock pattern as platformio/toolchain.py and git.py).
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    with FileLock(f"{dest}.lock"):
+        _install_package_locked(name, version, dest, mirrors, expect, marker)
+
+
+def _install_package_locked(
+    name: str,
+    version: str,
+    dest: Path,
+    mirrors: list[str],
+    expect: Collection[str],
+    marker: Path,
+) -> None:
+    if marker.is_file():
+        # Another process finished the install while we waited for the lock
+        return
     rmdir(dest, msg=f"Clean up incomplete {name} install")
     # A persistent download location (not a temp dir) so an interrupted
     # download resumes across esphome runs via download_with_resume's .part

esphome/build_gen/arduino8266.py

@@ -299,30 +299,44 @@ def _defines_flags(
     ]
 
 
+def _unflag_tokens() -> set[str]:
+    """``build_unflags`` entries shell-lexed to tokens, as PlatformIO matches."""
+    return {
+        tok
+        for entry in CORE.build_unflags
+        for tok in split_flag_entry(entry, "esphome build_unflags")
+    }
+
+
+def _shell_token(tok: str) -> str:
+    """Quote a lexed token for the shell-expanded ninja command line.
+
+    Lexing strips the quoting a user wrote (``-DX="a b"`` becomes the single
+    token ``-DX=a b``); re-quote on the way out so the compiler receives the
+    same argv element SCons would pass under PlatformIO.
+    """
+    if not re.search(r'[\s"\']', tok):
+        return tok
+    return '"' + tok.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
 def _project_flags() -> tuple[list[str], list[str], list[Path], list[str]]:
     """Split the ESPHome build flags into compile, linker, -L, and -l lists.
 
-    Unlike ``framework_helpers.get_project_compile_flags`` this keeps every
-    non-linker flag, matching how PlatformIO passes ``build_flags`` to the
-    compiler verbatim, and applies ``build_unflags``. ``-L``/``-l`` are
-    classified out so they reach the link line as they do under PlatformIO.
+    Every entry is shell-lexed the way PlatformIO's ``ParseFlags`` does, so a
+    linker flag anywhere in an entry reaches the link line and
+    ``build_unflags`` matches individual tokens (``-Os`` inside ``-Os -g3``).
+    Lexed tokens are re-quoted at emission via ``_shell_token``.
     """
-    unflags = set(CORE.build_unflags)
-    flags = [f for f in sorted(CORE.build_flags) if f not in unflags]
+    unflags = _unflag_tokens()
     compile_flags: list[str] = []
     link_flags: list[str] = []
     lib_dirs: list[Path] = []
     libs: list[str] = []
-    for flag in flags:
-        # Shell-lex only linker entries so forms like "-L /opt/blobs" work as
-        # they do under PlatformIO. Other entries pass verbatim: lexing them
-        # would strip the quotes in defines like -DBOARD="...".
-        tokens = (
-            join_flag_args(split_flag_entry(flag, "esphome"), "esphome")
-            if flag.startswith(("-L", "-l"))
-            else [flag]
-        )
-        for tok in tokens:
+    for flag in sorted(CORE.build_flags):
+        for tok in join_flag_args(split_flag_entry(flag, "esphome"), "esphome"):
+            if tok in unflags:
+                continue
             if tok.startswith("-Wl,"):
                 link_flags.append(tok)
             elif tok.startswith("-L"):
@@ -330,8 +344,8 @@ def _project_flags() -> tuple[list[str], list[str], list[Path], list[str]]:
             elif tok.startswith("-l"):
                 libs.append(tok[2:])
             else:
-                compile_flags.append(tok)
-    return compile_flags, link_flags, lib_dirs, libs
+                compile_flags.append(_shell_token(tok))
+    return compile_flags, [_shell_token(t) for t in link_flags], lib_dirs, libs
 
 
 def _collect_sources(root: Path, exclude: set[str] = frozenset()) -> list[Path]:
@@ -512,7 +526,7 @@ def write_project(paths: dict[str, Path]) -> bool:
     # build_unflags applies to the framework flag sets too (compile and link),
     # as under PlatformIO (a silently ignored ``build_unflags: -Os`` would
     # diverge between the toolchains).
-    unflags = set(CORE.build_unflags)
+    unflags = _unflag_tokens()
     cflags = [f for f in cflags if f not in unflags]
     cxxflags = [f for f in cxxflags if f not in unflags]
     asflags = [f for f in asflags if f not in unflags]
@@ -627,7 +641,7 @@ def write_project(paths: dict[str, Path]) -> bool:
             lib.sources,
             lib_root,
             f"lib/{lib.name}",
-            flags=" ".join(lib.flags),
+            flags=" ".join(_shell_token(f) for f in lib.flags),
         )
         archive = f"lib{lib.name}.a"
         lines.append(f"build {_e(archive)}: ar {' '.join(objs)}")

tests/unit_tests/build_gen/test_arduino8266.py

@@ -607,3 +607,29 @@ def test_flag_defines_lexes_multi_token_entries() -> None:
     assert defines["FOO"] == "FOO=1"
     config = _resolve_build_config(defines)
     assert config.lwip_lib == "lwip2-1460"
+
+
+def test_project_flags_lexes_every_entry() -> None:
+    """A linker flag anywhere in an entry reaches the link line (PIO parity)."""
+    _set_flags("-DFOO=1 -lbar")
+    compile_flags, _link, _dirs, libs = arduino8266._project_flags()
+    assert libs == ["bar"]
+    assert "-DFOO=1" in compile_flags
+
+
+def test_project_flags_unflags_match_tokens() -> None:
+    """build_unflags removes a token embedded in a multi-token entry."""
+    _set_flags("-Os -g3")
+    CORE.build_unflags = {"-Os"}
+    compile_flags, _link, _dirs, _libs = arduino8266._project_flags()
+    assert "-g3" in compile_flags
+    assert "-Os" not in compile_flags
+
+
+def test_project_flags_requotes_lexed_defines() -> None:
+    """A quoted spaced value stays one compiler argument after lex/emit."""
+    _set_flags('-DGREETING="hello world"')
+    compile_flags, _link, _dirs, _libs = arduino8266._project_flags()
+    # shlex folds the quotes (as PIO's ParseFlags does); _shell_token
+    # re-quotes the spaced token so the shell passes one argv element
+    assert compile_flags == ['"-DGREETING=hello world"']

tests/unit_tests/test_arduino8266_framework.py

@@ -402,3 +402,30 @@ def test_install_package_unexpected_layout_raises(tmp_path: Path) -> None:
         mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir()
         framework._install_package("pkg", "1.0.0", dest, ["http://m"], expect=("bin",))
     assert not (dest / ".esphome_extracted").exists()
+
+
+def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None:
+    """A concurrent install finishing while we wait for the lock is detected."""
+    dest = tmp_path / "pkg"
+    marker = dest / ".esphome_extracted"
+
+    class _FakeLock:
+        def __init__(self, *_a, **_kw) -> None:
+            pass
+
+        def __enter__(self):
+            dest.mkdir(parents=True, exist_ok=True)
+            marker.touch()
+            return self
+
+        def __exit__(self, *args) -> None:
+            pass
+
+    with (
+        patch("filelock.FileLock", _FakeLock),
+        patch.object(framework, "download_from_mirrors") as mock_download,
+        patch.object(framework, "rmdir") as mock_rmdir,
+    ):
+        framework._install_package("pkg", "1.0.0", dest, ["http://m"])
+    mock_download.assert_not_called()
+    mock_rmdir.assert_not_called()