Move the extraScript machinery into the platformio package and tidy the extraction

tests/unit_tests/build_helpers/test_extra_script.py

@@ -1,230 +0,0 @@
-"""Tests for the shared extraScript machinery (build_helpers.extra_script)."""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-
-import pytest
-
-from esphome.platformio.library import ConvertedLibrary as IDFComponent, URLSource
-
-
-def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
-    from esphome.build_helpers.extra_script import (
-        captured_as_build_flags,
-        run_extra_script,
-    )
-
-    (tmp_path / "src" / "esp32").mkdir(parents=True)
-    script = tmp_path / "extra_script.py"
-    script.write_text(
-        "Import('env')\n"
-        "mcu = env.get('BOARD_MCU')\n"
-        "env.Append(\n"
-        "    LIBPATH=[join('src', mcu)],\n"
-        "    LIBS=['algobsec'],\n"
-        "    CPPDEFINES=['FOO', ('BAR', '1')],\n"
-        "    LINKFLAGS=['-Wl,--gc-sections'],\n"
-        ")\n"
-    )
-    # The script uses bare ``join`` (PIO's extra-scripts run inside SCons
-    # where this is in scope). Inject it via the script header so the
-    # shim's exec namespace can resolve it.
-    script.write_text("from os.path import join\n" + script.read_text())
-
-    result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
-
-    assert result.libpath == [str(Path("src") / "esp32")]
-    assert result.libs == ["algobsec"]
-    assert ("BAR", "1") in result.cppdefines
-    assert "FOO" in result.cppdefines
-    assert result.linkflags == ["-Wl,--gc-sections"]
-
-    flags = captured_as_build_flags(result, library_dir=tmp_path)
-    sep = os.sep
-    assert f"-Lsrc{sep}esp32" in flags
-    assert "-lalgobsec" in flags
-    assert "-DFOO" in flags
-    assert "-DBAR=1" in flags
-    assert "-Wl,--gc-sections" in flags
-
-
-def test_extra_script_libpath_relative_resolves_against_library_dir(
-    tmp_path, monkeypatch
-):
-    """Relative LIBPATH entries must resolve against ``library_dir``, not the
-    caller's CWD (the shim restores CWD before ``captured_as_build_flags``
-    runs)."""
-    from esphome.build_helpers.extra_script import (
-        ExtraScriptResult,
-        captured_as_build_flags,
-    )
-
-    (tmp_path / "lib" / "esp32").mkdir(parents=True)
-    elsewhere = tmp_path.parent / "not_the_library_dir"
-    elsewhere.mkdir(exist_ok=True)
-    monkeypatch.chdir(elsewhere)
-
-    result = ExtraScriptResult(libpath=["lib/esp32"])
-    flags = captured_as_build_flags(result, library_dir=tmp_path)
-
-    sep = os.sep
-    assert flags == [f"-Llib{sep}esp32"]
-
-
-def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
-    from esphome.build_helpers.extra_script import (
-        ExtraScriptResult,
-        captured_as_build_flags,
-    )
-
-    outside = tmp_path.parent / "system_lib"
-    outside.mkdir(exist_ok=True)
-    result = ExtraScriptResult(libpath=[str(outside)])
-
-    flags = captured_as_build_flags(result, library_dir=tmp_path)
-    assert flags == [f"-L{outside.resolve()}"]
-
-
-def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
-    from esphome.build_helpers.extra_script import run_extra_script
-
-    script = tmp_path / "broken.py"
-    script.write_text("raise RuntimeError('boom')\n")
-
-    with caplog.at_level("WARNING"):
-        result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
-
-    assert result.libpath == []
-    assert result.libs == []
-    assert "broken.py" in caplog.text
-
-
-def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
-    from esphome.espidf.component import _apply_extra_script
-
-    library_dir = tmp_path / "lib"
-    library_dir.mkdir()
-    outside = tmp_path / "evil.py"
-    outside.write_text("env.Append(LIBS=['pwned'])\n")
-
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = library_dir
-    c.data = {"build": {"extraScript": "../evil.py"}}
-
-    _apply_extra_script(c)
-
-    # Nothing was folded into flags: the traversal was rejected before
-    # the script could run.
-    assert "flags" not in c.data["build"]
-
-
-def test_apply_extra_script_merges_into_existing_flags(tmp_path, monkeypatch):
-    from esphome.components import esp32 as esp32_module
-
-    monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
-
-    from esphome.espidf.component import _apply_extra_script
-
-    (tmp_path / "src").mkdir()
-    script = tmp_path / "extra.py"
-    script.write_text("env.Append(LIBS=['algobsec'])\n")
-
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}}
-
-    _apply_extra_script(c)
-
-    assert "-DEXISTING" in c.data["build"]["flags"]
-    assert "-lalgobsec" in c.data["build"]["flags"]
-
-
-def test_apply_extra_script_callable_target_and_str_flags(tmp_path) -> None:
-    """The shared helper resolves a callable idf_target lazily and normalizes
-    a string ``build.flags`` value into a list before extending it."""
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    (tmp_path / "src").mkdir()
-    script = tmp_path / "extra.py"
-    script.write_text("env.Append(LIBS=[env.get('BOARD_MCU')])\n")
-
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "extra.py", "flags": "-DBASE=1"}}
-
-    apply_extra_script(c, lambda: "esp8266")
-
-    assert c.data["build"]["flags"] == ["-DBASE=1", "-lesp8266"]
-
-
-def test_apply_extra_script_no_script_and_no_flags(tmp_path) -> None:
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    # No extraScript declared: nothing happens, the target is never resolved
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {}}
-    apply_extra_script(c, lambda: pytest.fail("target resolved without a script"))
-
-    # A script that captures nothing leaves the flags untouched
-    script = tmp_path / "noop.py"
-    script.write_text("pass\n")
-    c.data = {"build": {"extraScript": "noop.py"}}
-    apply_extra_script(c, "esp8266")
-    assert "flags" not in c.data["build"]
-
-
-def test_apply_extra_script_ignores_uncaptured_env_calls(tmp_path) -> None:
-    """Un-captured env vars and unsupported env methods are silent no-ops."""
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    script = tmp_path / "extra.py"
-    script.write_text(
-        "env.Replace(CC='clang')\nenv.Append(UNCAPTURED=['x'], LIBS='single')\n"
-    )
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "extra.py"}}
-    apply_extra_script(c, "esp8266")
-    assert c.data["build"]["flags"] == ["-lsingle"]
-
-
-def test_apply_extra_script_swallows_script_errors(tmp_path, caplog) -> None:
-    """A raising extra-script is best-effort: logged and skipped."""
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    script = tmp_path / "extra.py"
-    script.write_text("raise RuntimeError('boom')\n")
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "extra.py"}}
-    apply_extra_script(c, "esp8266")
-    assert "flags" not in c.data["build"]
-    assert "skipping" in caplog.text
-
-
-def test_apply_extra_script_pio_platform(tmp_path) -> None:
-    """The backend's platform token is exposed to the script as PIOPLATFORM."""
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    script = tmp_path / "extra.py"
-    script.write_text("env.Append(LIBS=[env.get('PIOPLATFORM')])\n")
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "extra.py"}}
-    apply_extra_script(c, "esp8266", pio_platform="espressif8266")
-    assert c.data["build"]["flags"] == ["-lespressif8266"]
-
-
-def test_apply_extra_script_missing_script_logged(tmp_path, caplog) -> None:
-    """A declared but absent extraScript is skipped with a visible warning:
-    its captured link flags are lost."""
-    from esphome.build_helpers.extra_script import apply_extra_script
-
-    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
-    c.path = tmp_path
-    c.data = {"build": {"extraScript": "nope.py"}}
-    apply_extra_script(c, "esp8266")
-    assert "not found" in caplog.text

tests/unit_tests/build_helpers/test_idedata.py

@@ -21,7 +21,7 @@ def _entry(directory: str, file: str, command: str) -> dict:
     return {"directory": directory, "file": file, "command": command}
 
 
-def test_parse_entry_extracts_fields() -> None:
+def testparse_entry_extracts_fields() -> None:
     """cxx_path, defines, includes and remaining flags are split apart."""
     entry = _entry(
         f"{ABS}build",
@@ -30,7 +30,7 @@ def test_parse_entry_extracts_fields() -> None:
         f"-I{ABS}inc/a -isystem {ABS}sys/b -std=gnu++20 -c app.cpp -o app.cpp.o",
     )
 
-    cxx_path, defines, includes, cxx_flags = idedata._parse_entry(entry)
+    cxx_path, defines, includes, cxx_flags = idedata.parse_entry(entry)
 
     assert cxx_path == "/tools/xtensa-esp32-elf-g++"
     assert "USE_ESP32" in defines
@@ -45,7 +45,7 @@ def test_parse_entry_extracts_fields() -> None:
     assert "app.cpp.o" not in cxx_flags
 
 
-def test_parse_entry_space_separated_args() -> None:
+def testparse_entry_space_separated_args() -> None:
     """``-D X`` / ``-I path`` (separate arg) and ``-isystem<path>`` (joined)."""
     entry = _entry(
         f"{ABS}build",
@@ -53,14 +53,14 @@ def test_parse_entry_space_separated_args() -> None:
         f"g++ -D FOO=1 -I {ABS}inc/sep -isystem{ABS}sys/joined -c x.cpp",
     )
 
-    _, defines, includes, _ = idedata._parse_entry(entry)
+    _, defines, includes, _ = idedata.parse_entry(entry)
 
     assert "FOO=1" in defines
     assert f"{ABS}inc/sep" in includes
     assert f"{ABS}sys/joined" in includes
 
 
-def test_parse_entry_resolves_relative_includes() -> None:
+def testparse_entry_resolves_relative_includes() -> None:
     """Relative includes are resolved against the entry's ``directory``."""
     directory = f"{ABS}build/proj"
     entry = _entry(
@@ -69,10 +69,10 @@ def test_parse_entry_resolves_relative_includes() -> None:
         "g++ -Iconfig -I../shared -isystem rel/sys -c x.cpp",
     )
 
-    _, _, includes, _ = idedata._parse_entry(entry)
+    _, _, includes, _ = idedata.parse_entry(entry)
 
     def resolved(rel: str) -> str:
-        # _parse_entry emits forward slashes for consistency (normpath would
+        # parse_entry emits forward slashes for consistency (normpath would
         # yield backslashes on Windows).
         return os.path.normpath(Path(directory) / rel).replace("\\", "/")
 
@@ -83,7 +83,7 @@ def test_parse_entry_resolves_relative_includes() -> None:
     assert all(Path(inc).is_absolute() for inc in includes)
 
 
-def test_parse_entry_skips_dependency_flags() -> None:
+def testparse_entry_skips_dependency_flags() -> None:
     """Dependency-generation flags (and their args) are dropped."""
     entry = _entry(
         "/build",
@@ -91,7 +91,7 @@ def test_parse_entry_skips_dependency_flags() -> None:
         "g++ -MD -MT x.cpp.o -MF x.cpp.o.d -c x.cpp -o x.cpp.o",
     )
 
-    _, _, _, cxx_flags = idedata._parse_entry(entry)
+    _, _, _, cxx_flags = idedata.parse_entry(entry)
 
     for tok in ("-MD", "-MT", "x.cpp.o", "-MF", "x.cpp.o.d", "-c", "-o", "x.cpp"):
         assert tok not in cxx_flags
@@ -198,17 +198,17 @@ def test_idedata_from_build(tmp_path: Path) -> None:
     assert data["includes"]["toolchain"] == ["/tc/inc/c++", "/tc/inc"]
 
 
-def test_get_toolchain_includes_raises_on_probe_failure() -> None:
+def testget_toolchain_includes_raises_on_probe_failure() -> None:
     """A failed compiler probe is a hard error, not a silent empty list."""
     fake_proc = MagicMock(returncode=1, stderr="xtensa-esp32-elf-g++: not found")
     with (
         patch.object(idedata.subprocess, "run", return_value=fake_proc),
         pytest.raises(RuntimeError, match="builtin include dirs"),
     ):
-        idedata._get_toolchain_includes("/bad/compiler")
+        idedata.get_toolchain_includes("/bad/compiler")
 
 
-def test_get_toolchain_includes_raises_when_no_dirs_found() -> None:
+def testget_toolchain_includes_raises_when_no_dirs_found() -> None:
     """Markers present but no dirs (anomalous output) also raises."""
     fake_proc = MagicMock(
         returncode=0,
@@ -218,7 +218,7 @@ def test_get_toolchain_includes_raises_when_no_dirs_found() -> None:
         patch.object(idedata.subprocess, "run", return_value=fake_proc),
         pytest.raises(RuntimeError, match="builtin include dirs"),
     ):
-        idedata._get_toolchain_includes("/some/compiler")
+        idedata.get_toolchain_includes("/some/compiler")
 
 
 # ESP-IDF's compile_commands.json on Windows mixes literal backslash path
@@ -248,7 +248,7 @@ def test_split_command_empty_returns_empty() -> None:
 
 
 @pytest.mark.skipif(os.name != "nt", reason="Windows argv tokenization")
-def test_parse_entry_normalizes_windows_cxx_path() -> None:
+def testparse_entry_normalizes_windows_cxx_path() -> None:
     """A backslash compiler path is emitted forward-slashed; define unescaped."""
     entry = _entry(
         r"C:\b",
@@ -256,7 +256,7 @@ def test_parse_entry_normalizes_windows_cxx_path() -> None:
         r"C:\esp\bin\g++.exe -DVER=\"1.2.3\" -IC:/inc/a -c x.cpp",
     )
 
-    cxx_path, defines, includes, _ = idedata._parse_entry(entry)
+    cxx_path, defines, includes, _ = idedata.parse_entry(entry)
 
     assert cxx_path == "C:/esp/bin/g++.exe"
     assert "\\" not in cxx_path
@@ -264,7 +264,7 @@ def test_parse_entry_normalizes_windows_cxx_path() -> None:
     assert "C:/inc/a" in includes
 
 
-def test_parse_entry_strips_launcher_prefix() -> None:
+def testparse_entry_strips_launcher_prefix() -> None:
     """A launcher-wrapped compile names the compiler second; the exact
     configured launcher is stripped, not anything ccache-shaped."""
     entry = _entry(
@@ -273,18 +273,18 @@ def test_parse_entry_strips_launcher_prefix() -> None:
         "/opt/homebrew/bin/ccache /tools/xtensa-lx106-elf-g++ -DUSE_ESP8266 "
         "-c app.cpp -o app.cpp.o",
     )
-    cxx_path, defines, _, _ = idedata._parse_entry(
+    cxx_path, defines, _, _ = idedata.parse_entry(
         entry, launcher="/opt/homebrew/bin/ccache"
     )
     assert cxx_path == "/tools/xtensa-lx106-elf-g++"
     assert defines == ["USE_ESP8266"]
     # Without a configured launcher nothing is stripped, even a token that
     # happens to be named ccache -- but the surprise is warned about
-    cxx_path, _, _, _ = idedata._parse_entry(entry)
+    cxx_path, _, _, _ = idedata.parse_entry(entry)
     assert cxx_path == "/opt/homebrew/bin/ccache"
 
 
-def test_parse_entry_warns_when_first_token_is_not_a_compiler(
+def testparse_entry_warns_when_first_token_is_not_a_compiler(
     caplog: pytest.LogCaptureFixture,
 ) -> None:
     entry = _entry(
@@ -292,10 +292,10 @@ def test_parse_entry_warns_when_first_token_is_not_a_compiler(
         f"{ABS}build/src/esphome/core/application.cpp",
         "/opt/homebrew/bin/ccache /tools/xtensa-lx106-elf-g++ -c a.cpp -o a.o",
     )
-    idedata._parse_entry(entry)
+    idedata.parse_entry(entry)
     assert "does not start with a compiler" in caplog.text
     caplog.clear()
-    idedata._parse_entry(entry, launcher="/opt/homebrew/bin/ccache")
+    idedata.parse_entry(entry, launcher="/opt/homebrew/bin/ccache")
     assert "does not start with a compiler" not in caplog.text
 
 
@@ -328,7 +328,7 @@ def test_load_or_build_idedata_builds_and_caches(tmp_path: Path) -> None:
     compile_commands = _write_compile_commands(tmp_path)
     cache = tmp_path / "cache" / "test.json"
     with patch.object(
-        idedata, "_get_toolchain_includes", return_value=["/toolchain/include"]
+        idedata, "get_toolchain_includes", return_value=["/toolchain/include"]
     ):
         data = idedata.load_or_build_idedata(
             compile_commands, tmp_path / "firmware.elf", cache
@@ -355,8 +355,39 @@ def test_load_or_build_idedata_rebuilds_bad_cache(tmp_path: Path) -> None:
     for bad in ("not json", json.dumps({"no_cc_path": True})):
         cache.write_text(bad)
         os.utime(cache, (compile_commands.stat().st_mtime + 10,) * 2)
-        with patch.object(idedata, "_get_toolchain_includes", return_value=[]):
+        with patch.object(idedata, "get_toolchain_includes", return_value=[]):
             data = idedata.load_or_build_idedata(
                 compile_commands, tmp_path / "f.elf", cache
             )
         assert "cc_path" in data
+
+
+def test_load_or_build_idedata_rebuilds_when_compile_db_newer(tmp_path: Path) -> None:
+    """A compile DB newer than the cache forces regeneration."""
+    compile_commands = _write_compile_commands(tmp_path)
+    cache = tmp_path / "cache.json"
+    cache.write_text(json.dumps({"cc_path": "stale"}))
+    os.utime(compile_commands, (cache.stat().st_mtime + 10,) * 2)
+    with patch.object(idedata, "get_toolchain_includes", return_value=[]):
+        data = idedata.load_or_build_idedata(
+            compile_commands, tmp_path / "f.elf", cache
+        )
+    assert data["cc_path"] != "stale"
+
+
+def test_load_or_build_idedata_rebuilds_non_dict_cache(tmp_path: Path) -> None:
+    """Valid JSON that is not an object is regenerated, never handed out.
+
+    A bare string would otherwise pass the cc_path check by substring.
+    """
+    compile_commands = _write_compile_commands(tmp_path)
+    cache = tmp_path / "cache.json"
+    for bad in ('"cc_path is a string"', "[]", "42"):
+        cache.write_text(bad)
+        os.utime(cache, (compile_commands.stat().st_mtime + 10,) * 2)
+        with patch.object(idedata, "get_toolchain_includes", return_value=[]):
+            data = idedata.load_or_build_idedata(
+                compile_commands, tmp_path / "f.elf", cache
+            )
+        assert isinstance(data, dict)
+        assert "cc_path" in data

tests/unit_tests/build_helpers/test_size_summary.py

@@ -0,0 +1,10 @@
+"""Tests for the shared PlatformIO-format size bar."""
+
+from __future__ import annotations
+
+from esphome.build_helpers.size_summary import format_bar
+
+
+def test_format_bar_zero_total() -> None:
+    """A zero total must not divide by zero."""
+    assert format_bar(0, 0) == "[          ]   0.0% (used 0 bytes from 0 bytes)"