Merge branch 'esp8266-native-library-backend' into esp8266-native-build-spec

tests/unit_tests/build_gen/test_build_tool.py

@@ -87,3 +87,40 @@ def test_ar_expands_rspfile_without_escaping(tmp_path) -> None:
         "obj/a.o",
         "sub\\b.o",
     ]
+
+
+def test_ar_unquotes_ninja_escaped_paths(tmp_path: Path) -> None:
+    """The shim strips a simple surrounding quote, since ninja shell-
+    quotes special rsp paths, so ar sees the real filename."""
+    rsp = tmp_path / "t.rsp"
+    rsp.write_text("'obj/a b.o'\nobj/c.o\n")
+    with (
+        patch.object(
+            build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)]
+        ),
+        patch.object(build_tool.subprocess, "run") as mock_run,
+    ):
+        mock_run.return_value.returncode = 0
+        rc = build_tool.main()
+    assert rc == 0
+    assert mock_run.call_args.args[0] == [
+        "/usr/bin/ar",
+        "rc",
+        "lib.a",
+        "obj/a b.o",
+        "obj/c.o",
+    ]
+
+
+def test_ar_empty_object_list_fails(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """A lost object list is an error here, not undefined symbols at link."""
+    rsp = tmp_path / "t.rsp"
+    rsp.write_text("\n\n")
+    with patch.object(
+        build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)]
+    ):
+        rc = build_tool.main()
+    assert rc == 1
+    assert "no objects listed" in capsys.readouterr().err

tests/unit_tests/build_helpers/test_ccache.py

@@ -0,0 +1,79 @@
+"""Tests for the shared ccache policy in esphome.build_helpers.ccache."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from esphome.build_helpers import ccache
+
+
+def test_resolve_opt_out() -> None:
+    with patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}):
+        assert ccache.resolve_ccache_path() is None
+
+
+def test_resolve_no_binary(caplog: pytest.LogCaptureFixture) -> None:
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch("shutil.which", return_value=None),
+    ):
+        assert ccache.resolve_ccache_path() is None
+    assert "no ccache binary" not in caplog.text
+
+
+def test_resolve_probe_failure() -> None:
+    with (
+        patch.dict(os.environ, {}, clear=True),
+        patch("shutil.which", return_value="/usr/bin/ccache"),
+        patch(
+            "esphome.build_helpers.ccache.subprocess.run", side_effect=OSError("boom")
+        ),
+    ):
+        assert ccache.resolve_ccache_path() is None
+
+
+def test_resolve_explicit_skips_probe_and_warns_missing(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    with (
+        patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True),
+        patch("shutil.which", return_value="/usr/bin/ccache"),
+        patch.object(ccache, "_ccache_runs", side_effect=AssertionError),
+    ):
+        assert ccache.resolve_ccache_path() == "/usr/bin/ccache"
+    with (
+        patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True),
+        patch("shutil.which", return_value=None),
+    ):
+        assert ccache.resolve_ccache_path() is None
+    assert "no ccache binary is on PATH" in caplog.text
+
+
+def test_probe_spawns_with_close_fds_false() -> None:
+    with patch("esphome.build_helpers.ccache.subprocess.run") as mock_run:
+        assert ccache._ccache_runs("/usr/bin/ccache") is True
+    assert mock_run.call_args.kwargs["close_fds"] is False
+
+
+def test_defaults_env(tmp_path: Path) -> None:
+    with (
+        patch("esphome.core.CORE", SimpleNamespace(build_path=tmp_path / "b")),
+        patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True),
+    ):
+        env = ccache.ccache_defaults_env(tmp_path / "cache")
+    assert env["CCACHE_DIR"] == str(tmp_path / "cache")
+    assert env["CCACHE_DEPEND"] == "1"
+    assert "CCACHE_NOHASHDIR" not in env  # user value respected
+
+
+def test_defaults_env_requires_build_path() -> None:
+    with (
+        patch("esphome.core.CORE", SimpleNamespace(build_path=None)),
+        pytest.raises(ValueError, match="build_path"),
+    ):
+        ccache.ccache_defaults_env(Path("/x"))

tests/unit_tests/build_helpers/test_ninja.py

@@ -76,3 +76,8 @@ def test_shell_token_quotes_shell_metacharacters() -> None:
 def test_quote_path_force_quotes() -> None:
     assert ninja_helper.quote_path(Path("a b")) == '"a b"'
     assert ninja_helper.quote_path("simple") == '"simple"'
+
+
+def test_shell_token_empty_token_is_quoted() -> None:
+    """An empty argv element must survive as an explicit pair of quotes."""
+    assert ninja_helper.shell_token("") == '""'

tests/unit_tests/components/esp8266/test_build_surgery.py

@@ -108,8 +108,8 @@ def test_board_build_covers_every_board() -> None:
 
 
 def test_surgery_fingerprint_covers_module_source() -> None:
-    """The fingerprint hashes the module source, so any surgery edit
-    invalidates linker-script caches stamped with it."""
+    """Pins the mechanism: the fingerprint is the sha256 of the module
+    source (behavioral coverage follows from that, not from this test)."""
     import hashlib
     import inspect
 

tests/unit_tests/core/test_config.py

@@ -1285,6 +1285,7 @@ async def test_add_platformio_options_native_idf(
     await config._add_platformio_options(
         {
             "build_flags": "-DSINGLE_FLAG",  # string and list forms both valid
+            "build_unflags": ["-Os"],
             "lib_deps": ["bblanchon/ArduinoJson@7.4.2"],
             "lib_ignore": "libsodium",
             "upload_speed": "115200",
@@ -1294,6 +1295,7 @@ async def test_add_platformio_options_native_idf(
 
     assert "-DSINGLE_FLAG" in CORE.build_flags
     assert "ArduinoJson" in CORE.platformio_libraries
+    assert "-Os" in CORE.build_unflags
     # lib_ignore is stored (listified) for generate_idf_components to read;
     # nothing else lands in platformio_options on the native toolchain.
     assert CORE.platformio_options == {"lib_ignore": ["libsodium"]}

tests/unit_tests/test_arduino8266_framework.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-import subprocess
 from unittest.mock import patch
 
 import pytest
@@ -23,6 +22,9 @@ def _clear_caches(tmp_path: Path) -> None:
 def test_framework_package_version() -> None:
     assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0"
     assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0"
+    # A future major bump needs its own encoding, not a doomed registry lookup
+    with pytest.raises(EsphomeError, match="only 3.x"):
+        framework.framework_package_version(cv.Version(4, 0, 0))
 
 
 def test_tools_path_default_and_prefix(tmp_path: Path) -> None:
@@ -46,11 +48,25 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None:
     assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION
     assert paths.ninja == tmp_path / "ninja"
     assert mock_install.call_count == 2
-    # The layout checks cover the directories write_project needs, including
-    # the bundled libraries/ tree
-    fw_expect = mock_install.call_args_list[0].kwargs["expect"]
-    assert fw_expect == ("cores/esp8266", "tools/sdk", "libraries")
-    assert mock_install.call_args_list[1].kwargs["expect"] == ("bin",)
+    # Full argument pinning: a copy-paste swap between the two near-identical
+    # calls (mirrors, destination) must not stay green
+    fw_call, tc_call = mock_install.call_args_list
+    assert fw_call.args == (
+        framework.FRAMEWORK_PACKAGE,
+        "3.30102.0",
+        tmp_path / "frameworks" / "3.30102.0",
+        framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
+        tmp_path / "downloads",
+    )
+    assert fw_call.kwargs["expect"] == ("cores/esp8266", "tools/sdk", "libraries")
+    assert tc_call.args == (
+        framework.TOOLCHAIN_PACKAGE,
+        framework.TOOLCHAIN_VERSION,
+        tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION,
+        framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS,
+        tmp_path / "downloads",
+    )
+    assert tc_call.kwargs["expect"] == ("bin", "xtensa-lx106-elf")
 
 
 def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None:
@@ -60,52 +76,18 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None:
     assert env["CCACHE_DIR"] == "x"
 
 
-def test_ccache_path_disabled_by_env() -> None:
-    with patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}):
-        assert framework.ccache_path() is None
-
-
-def test_ccache_path_no_binary(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False)
-    with patch("shutil.which", return_value=None):
-        assert framework.ccache_path() is None
-
-
-def test_ccache_path_probe_failure(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False)
-    with (
-        patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("subprocess.run", side_effect=subprocess.SubprocessError),
-    ):
-        assert framework.ccache_path() is None
-
-
-def test_ccache_path_ok(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False)
-    with (
-        patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("subprocess.run"),
-    ):
-        assert framework.ccache_path() == "/usr/bin/ccache"
-
-
-def test_ccache_path_explicit_missing_binary_warns(
-    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+def test_ccache_path_delegates_and_caches(
+    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1")
-    with patch("shutil.which", return_value=None):
-        assert framework.ccache_path() is None
-    assert "no ccache binary is on PATH" in caplog.text
-
-
-def test_ccache_path_explicit_skips_probe(monkeypatch: pytest.MonkeyPatch) -> None:
-    """An explicit opt-in trusts the binary without the runnability probe."""
-    monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", "1")
-    with (
-        patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.build_helpers.ccache._ccache_runs", side_effect=AssertionError),
-    ):
+    """The wrapper delegates to the shared policy (covered in
+    build_helpers/test_ccache.py) and caches the result."""
+    monkeypatch.delenv("ESPHOME_CCACHE_ENABLE", raising=False)
+    with patch.object(
+        framework, "resolve_ccache_path", return_value="/usr/bin/ccache"
+    ) as mock_resolve:
         assert framework.ccache_path() == "/usr/bin/ccache"
+        assert framework.ccache_path() == "/usr/bin/ccache"
+    mock_resolve.assert_called_once()
 
 
 def test_ccache_env(tmp_path: Path) -> None:
@@ -123,16 +105,6 @@ def test_ccache_env(tmp_path: Path) -> None:
     assert env["CCACHE_DIR"].endswith("ccache")
 
 
-def test_ccache_env_requires_build_path() -> None:
-    """Building the env before preload set build_path fails loudly."""
-    CORE.build_path = None
-    with (
-        patch.object(framework, "ccache_path", return_value="/cc/ccache"),
-        pytest.raises(ValueError, match="build_path"),
-    ):
-        framework.ccache_env()
-
-
 def test_check_and_install_rejects_old_core(tmp_path: Path) -> None:
     """Calling the installer below the floor fails before any download."""
     with pytest.raises(EsphomeError, match=">= 3.1.1"):

tests/unit_tests/test_arduino_library.py

@@ -356,3 +356,56 @@ def test_bundled_library_prefers_library_json(tmp_path: Path) -> None:
     )
     lib = component._bundled_library(framework, "GDBStub")
     assert [s.name for s in lib.sources] == ["gdb.cpp"]
+
+
+def test_library_info_lib_archive_flag(tmp_path: Path) -> None:
+    """Both libArchive (library.json) and dot_a_linkage (properties) reach
+    the generator's contract; default is archive."""
+    read_path = tmp_path / "lib"
+    (read_path / "src").mkdir(parents=True)
+    assert component._library_info("x", read_path, {}).lib_archive is True
+    assert (
+        component._library_info(
+            "x", read_path, {"build": {"libArchive": False}}
+        ).lib_archive
+        is False
+    )
+    assert (
+        component._library_info("x", read_path, {"dot_a_linkage": "false"}).lib_archive
+        is False
+    )
+    assert (
+        component._library_info("x", read_path, {"dot_a_linkage": "true"}).lib_archive
+        is True
+    )
+
+
+def test_resolve_libraries_dep_warnings(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """Nameless and owner-without-version dependencies are dropped loudly."""
+    framework = _make_framework(tmp_path)
+    _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6")
+    lib_dir = tmp_path / "converted" / "webserver"
+    (lib_dir / "src").mkdir(parents=True)
+    converted = _converted(
+        "esp32async__ESPAsyncWebServer",
+        lib_dir,
+        {
+            "build": {},
+            "dependencies": [
+                {"owner": "someone"},
+                {"name": "Orphan", "owner": "someone"},
+            ],
+        },
+    )
+    with _emitting_converter(converted):
+        component.resolve_libraries(
+            framework,
+            pio_platform="espressif8266",
+            board_mcu="esp8266",
+            cache_key="arduino8266",
+        )
+    assert "malformed dependency entry" in caplog.text
+    assert "Orphan" in caplog.text
+    assert "owner but no version" in caplog.text

tests/unit_tests/test_platformio_registry.py

@@ -204,7 +204,7 @@ def test_install_package_skips_when_marker_exists(tmp_path: Path) -> None:
     dest.mkdir()
     (dest / ".esphome_extracted").touch()
     with patch.object(registry, "download_from_mirrors") as mock_download:
-        registry.install_package("pkg", "1.0.0", dest, [], tmp_path / "dl")
+        registry.install_package("pkg", "1.0.0", dest, [], tmp_path / "dl", expect=())
     mock_download.assert_not_called()
 
 
@@ -218,7 +218,9 @@ def test_install_package_downloads_via_mirrors(tmp_path: Path) -> None:
     ):
         # Extraction is expected to create the directory
         mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir()
-        registry.install_package("pkg", "1.0.0", dest, mirrors, tmp_path / "dl")
+        registry.install_package(
+            "pkg", "1.0.0", dest, mirrors, tmp_path / "dl", expect=()
+        )
     assert mock_download.call_args[0][0] is mirrors
     assert mock_download.call_args[0][1] == {
         "VERSION": "1.0.0",
@@ -240,7 +242,7 @@ def test_install_package_downloads_via_registry(tmp_path: Path) -> None:
         ),
     ):
         mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir()
-        registry.install_package("pkg", "1.0.0", dest, [], tmp_path / "dl")
+        registry.install_package("pkg", "1.0.0", dest, [], tmp_path / "dl", expect=())
     assert mock_download.call_args[0][0] == "http://x/pkg.tar.gz"
     assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42}
 
@@ -291,7 +293,9 @@ def test_install_package_marker_rechecked_under_lock(tmp_path: Path) -> None:
         patch.object(registry, "download_from_mirrors") as mock_download,
         patch.object(registry, "rmdir") as mock_rmdir,
     ):
-        registry.install_package("pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl")
+        registry.install_package(
+            "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=()
+        )
     mock_download.assert_not_called()
     mock_rmdir.assert_not_called()
 
@@ -306,5 +310,39 @@ def test_install_package_uses_hard_lock(tmp_path: Path) -> None:
         patch.object(registry, "get_systype", return_value="linux_x86_64"),
     ):
         mock_extract.side_effect = lambda *_a, **_kw: dest.mkdir(exist_ok=True)
-        registry.install_package("pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl")
+        registry.install_package(
+            "pkg", "1.0.0", dest, ["http://m"], tmp_path / "dl", expect=()
+        )
     assert mock_lock.call_args.kwargs["fallback_to_soft"] is False
+
+
+def test_registry_download_empty_system_list_does_not_match() -> None:
+    """An explicitly empty system list must not act as a wildcard."""
+    with (
+        _registry_response([{"system": [], "download_url": "http://x/any"}]),
+        patch.object(registry, "get_systype", return_value="linux_x86_64"),
+        pytest.raises(EsphomeError, match="No pkg 1.0.0 build"),
+    ):
+        registry.registry_download("pkg", "1.0.0")
+
+
+def test_registry_download_unexpected_payload_is_named() -> None:
+    """An error envelope without a versions list is not 'version not found'."""
+
+    def fake_download(mirrors: list[str], substitutions: dict, target) -> str:
+        target.write(json.dumps({"message": "rate limited"}).encode())
+        return "http://x"
+
+    with (
+        patch.object(registry, "download_from_mirrors", side_effect=fake_download),
+        pytest.raises(EsphomeError, match="Unexpected package registry response"),
+    ):
+        registry.registry_download("pkg", "1.0.0")
+
+
+def test_registry_download_missing_system_key_matches_any() -> None:
+    """A file with no system key at all serves every host."""
+    with _registry_response(
+        [{"download_url": "http://x/any", "checksum": {"sha256": "abc"}, "size": 1}]
+    ):
+        assert registry.registry_download("pkg", "1.0.0") == ("http://x/any", "abc", 1)

tests/unit_tests/test_platformio_toolchain.py

@@ -432,7 +432,7 @@ def test_ccache_env_enabled_by_default(setup_core: Path) -> None:
     with (
         patch.dict(os.environ, {}, clear=True),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run"),
+        patch("esphome.build_helpers.ccache.subprocess.run"),
     ):
         env = toolchain._ccache_env()
 
@@ -495,7 +495,7 @@ def test_ccache_env_disabled_when_probe_fails(
     with (
         patch.dict(os.environ, {}, clear=True),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run", side_effect=probe_error),
+        patch("esphome.build_helpers.ccache.subprocess.run", side_effect=probe_error),
     ):
         env = toolchain._ccache_env()
 
@@ -509,7 +509,7 @@ def test_ccache_env_forced_on_skips_probe(setup_core: Path) -> None:
     with (
         patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "1"}, clear=True),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run") as mock_probe,
+        patch("esphome.build_helpers.ccache.subprocess.run") as mock_probe,
     ):
         env = toolchain._ccache_env()
 
@@ -537,9 +537,9 @@ def test_ccache_env_strips_win_long_path_prefix(setup_core: Path) -> None:
         patch.dict(os.environ, {}, clear=True),
         # shutil.which is patched, so the win32 code path of the real
         # implementation (which crashes on a POSIX host) is never reached.
-        patch("esphome.platformio.toolchain.sys.platform", "win32"),
+        patch("esphome.framework_helpers.sys.platform", "win32"),
         patch("shutil.which", return_value=prefixed),
-        patch("esphome.framework_helpers.subprocess.run") as mock_probe,
+        patch("esphome.build_helpers.ccache.subprocess.run") as mock_probe,
     ):
         env = toolchain._ccache_env()
 
@@ -588,7 +588,7 @@ def test_ccache_env_respects_user_values_and_refreshes_basedir(
     with (
         patch.dict(os.environ, user_env, clear=True),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run"),
+        patch("esphome.build_helpers.ccache.subprocess.run"),
     ):
         env = toolchain._ccache_env()
 
@@ -607,7 +607,7 @@ def test_run_platformio_cli_passes_ccache_env_to_subprocess_only(
     with (
         patch.dict(os.environ, {}, clear=False),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run"),
+        patch("esphome.build_helpers.ccache.subprocess.run"),
     ):
         os.environ.pop("ESPHOME_CCACHE_ENABLE", None)
         mock_run_external_process.return_value = 0
@@ -629,7 +629,7 @@ def test_ccache_env_requires_build_path(setup_core: Path) -> None:
     with (
         patch.dict(os.environ, {}, clear=True),
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run"),
+        patch("esphome.build_helpers.ccache.subprocess.run"),
         pytest.raises(ValueError, match="CORE.build_path must be set"),
     ):
         toolchain._ccache_env()
@@ -643,7 +643,7 @@ def test_run_platformio_cli_merges_caller_env(
 
     with (
         patch("shutil.which", return_value="/usr/bin/ccache"),
-        patch("esphome.framework_helpers.subprocess.run"),
+        patch("esphome.build_helpers.ccache.subprocess.run"),
     ):
         mock_run_external_process.return_value = 0
         toolchain.run_platformio_cli(
@@ -871,7 +871,7 @@ def test_strip_win_long_path_prefix(
     platform: str, input_path: str, expected: str
 ) -> None:
     r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32."""
-    with patch("esphome.platformio.toolchain.sys.platform", platform):
+    with patch("esphome.framework_helpers.sys.platform", platform):
         assert toolchain.strip_win_long_path_prefix(input_path) == expected
 
 
@@ -898,7 +898,7 @@ def test_run_platformio_cli_strips_win_long_path_prefix(
         # so the stdlib sees it too) would send shutil.which down the Windows
         # code path, which crashes on a POSIX host.
         patch.dict(os.environ, {"ESPHOME_CCACHE_ENABLE": "0"}, clear=False),
-        patch("esphome.platformio.toolchain.sys.platform", "win32"),
+        patch("esphome.framework_helpers.sys.platform", "win32"),
         patch("esphome.platformio.toolchain.sys.executable", prefixed_exe),
     ):
         # Pop any pre-existing PYTHONEXEPATH so the assertion below reflects
@@ -930,7 +930,7 @@ def test_run_platformio_cli_does_not_set_pythonexepath_without_strip(
 
     with (
         patch.dict(os.environ, {}, clear=False),
-        patch("esphome.platformio.toolchain.sys.platform", "linux"),
+        patch("esphome.framework_helpers.sys.platform", "linux"),
         patch("esphome.platformio.toolchain.sys.executable", plain_exe),
     ):
         os.environ.pop("PYTHONEXEPATH", None)