[core] Support platformio_options on the native ESP-IDF toolchain (#16917)

tests/unit_tests/core/test_config.py

@@ -20,6 +20,9 @@ from esphome.const import (
     CONF_NAME,
     CONF_NAME_ADD_MAC_SUFFIX,
     KEY_CORE,
+    KEY_TARGET_FRAMEWORK,
+    KEY_TARGET_PLATFORM,
+    Toolchain,
 )
 from esphome.core import CORE, config
 from esphome.core.config import (
@@ -1161,3 +1164,123 @@ def test_make_app_name_cpp_special_chars_escaped() -> None:
     cpp_expr, _, _ = make_app_name_cpp('my "device"', "buf", "-", add_mac_suffix=False)
     # cpp_string_escape uses octal escapes for quotes
     assert '"' not in cpp_expr[1:-1]  # no unescaped quotes inside the outer quotes
+
+
+@pytest.mark.parametrize(
+    ("lib", "name", "version", "repository"),
+    [
+        ("ArduinoJson", "ArduinoJson", None, None),
+        ("bblanchon/ArduinoJson@7.4.2", "bblanchon/ArduinoJson", "7.4.2", None),
+        (
+            "noise-c=https://github.com/esphome/noise-c.git",
+            "noise-c",
+            None,
+            "https://github.com/esphome/noise-c.git",
+        ),
+    ],
+)
+def test_add_library_str(
+    lib: str, name: str, version: str | None, repository: str | None
+) -> None:
+    CORE.data[KEY_CORE] = {
+        KEY_TARGET_PLATFORM: "esp32",
+        KEY_TARGET_FRAMEWORK: "esp-idf",
+    }
+
+    config._add_library_str(lib)
+
+    libraries = list(CORE.platformio_libraries.values())
+    assert len(libraries) == 1
+    assert libraries[0].name == name
+    assert libraries[0].version == version
+    assert libraries[0].repository == repository
+
+
+@pytest.mark.asyncio
+async def test_add_platformio_options_native_idf(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """On the native IDF toolchain, build_flags/lib_deps/lib_ignore are
+    honored, upload_speed is silent and everything else warns."""
+    CORE.toolchain = Toolchain.ESP_IDF
+    CORE.data[KEY_CORE] = {
+        KEY_TARGET_PLATFORM: "esp32",
+        KEY_TARGET_FRAMEWORK: "esp-idf",
+    }
+
+    await config._add_platformio_options(
+        {
+            "build_flags": "-DSINGLE_FLAG",  # string and list forms both valid
+            "lib_deps": ["bblanchon/ArduinoJson@7.4.2"],
+            "lib_ignore": "libsodium",
+            "upload_speed": "115200",
+            "board_build.f_flash": "80000000L",
+        }
+    )
+
+    assert "-DSINGLE_FLAG" in CORE.build_flags
+    assert "ArduinoJson" in CORE.platformio_libraries
+    # lib_ignore is stored (listified) for generate_idf_components to read;
+    # nothing else lands in platformio_options on the native toolchain.
+    assert CORE.platformio_options == {"lib_ignore": ["libsodium"]}
+    assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text
+    assert "upload_speed" not in caplog.text
+    # build_flags has a first-class esphome equivalent, so it is deprecated.
+    # lib_deps/lib_ignore are kept as valid platformio_options (no warning).
+    assert (
+        "esphome->platformio_options->build_flags is deprecated; use "
+        "esphome->build_flags instead" in caplog.text
+    )
+    assert "lib_deps is deprecated" not in caplog.text
+    assert "lib_ignore is deprecated" not in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_add_platformio_options_platformio(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """On the PlatformIO toolchain all options pass through to the ini,
+    with build_flags/lib_ignore listified."""
+    CORE.toolchain = Toolchain.PLATFORMIO
+
+    await config._add_platformio_options(
+        {
+            "build_flags": "-DSINGLE_FLAG",
+            "lib_ignore": "libsodium",
+            "upload_speed": "115200",
+        }
+    )
+
+    assert CORE.platformio_options == {
+        "build_flags": ["-DSINGLE_FLAG"],
+        "lib_ignore": ["libsodium"],
+        "upload_speed": "115200",
+    }
+    # platformio_options is the correct mechanism on the PlatformIO toolchain,
+    # so the native-equivalent deprecation must not fire here.
+    assert "deprecated" not in caplog.text
+
+
+def test_add_library_str_bare_url_requires_name() -> None:
+    """A bare repository URL has no library name; CORE.add_library rejects it."""
+    with pytest.raises(ValueError, match="must have a name"):
+        config._add_library_str("https://github.com/esphome/noise-c.git")
+
+
+@pytest.mark.asyncio
+@pytest.mark.filterwarnings("ignore::RuntimeWarning")
+async def test_to_code_adds_libraries(yaml_file: Callable[[str], Path]) -> None:
+    """esphome->libraries entries are parsed and registered via cg.add_library."""
+    result = load_config_from_fixture(yaml_file, "libraries.yaml", FIXTURES_DIR)
+    assert result is not None
+
+    with patch("esphome.core.config.cg") as mock_cg:
+        mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock()
+        mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock()
+        await config.to_code(result[CONF_ESPHOME])
+
+    mock_cg.add_library.assert_any_call("SomeLib", None)
+    mock_cg.add_library.assert_any_call("bblanchon/ArduinoJson", "7.4.2")
+    mock_cg.add_library.assert_any_call(
+        "noise-c", None, "https://github.com/esphome/noise-c.git"
+    )

tests/unit_tests/fixtures/core/config/libraries.yaml

@@ -0,0 +1,8 @@
+esphome:
+  name: test-libraries
+  libraries:
+    - SomeLib
+    - bblanchon/ArduinoJson@7.4.2
+    - noise-c=https://github.com/esphome/noise-c.git
+
+host:

tests/unit_tests/test_core.py

@@ -915,3 +915,21 @@ class TestEsphomeCore:
             mock_enable.assert_called_once_with("Wire")
 
         assert "Wire" in target.platformio_libraries
+
+    def test_add_build_unflag__warns_on_native_idf_toolchain(
+        self, target, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Build unflags are not consumed by the native IDF build generator,
+        so adding one on that toolchain warns; PlatformIO stays silent."""
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.add_build_unflag("-fno-rtti")
+        assert "ignored" not in caplog.text
+
+        target.toolchain = const.Toolchain.ESP_IDF
+        target.add_build_unflag("-fno-exceptions")
+        assert (
+            "Build unflag -fno-exceptions is ignored when building with the "
+            "native ESP-IDF toolchain" in caplog.text
+        )
+        # The unflag is still recorded either way.
+        assert target.build_unflags == {"-fno-rtti", "-fno-exceptions"}

tests/unit_tests/test_espidf_component.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -515,7 +516,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
         "esphome/C": {"name": "C"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -557,6 +558,62 @@ def test_generate_idf_components_dedupes_shared_dependency(
     assert "idf_component_register" in generated
 
 
+def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    esp32_idf_core: None,
+) -> None:
+    # lib_ignore must drop B at the top level and C when it is discovered as a
+    # dependency of A during the graph walk -- neither may be resolved,
+    # downloaded, or wired into a manifest. Matching is by lowercase short name.
+    manifests = {
+        "esphome/A": {
+            "name": "A",
+            "dependencies": [
+                {"owner": "esphome", "name": "C", "version": "==1.10021.0"}
+            ],
+        },
+        "esphome/B": {"name": "B"},
+    }
+
+    download_salts: list[str] = []
+
+    def fake_download(self, force=False, salt=""):
+        download_salts.append(salt)
+        self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
+        (self.path / "src").mkdir(parents=True, exist_ok=True)
+        (self.path / "src" / "x.c").write_text("int x;")
+        (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
+
+    monkeypatch.setattr(IDFComponent, "download", fake_download)
+
+    resolve_calls: list[str] = []
+
+    def fake_resolve(owner, pkgname, requirements):
+        resolve_calls.append(pkgname)
+        return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
+
+    monkeypatch.setattr(
+        esphome.espidf.component, "_resolve_registry_version", fake_resolve
+    )
+    # lib_ignore is read from CORE.platformio_options (stored there by
+    # _add_platformio_options); matched by lowercase short name.
+    monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["B", "esphome/C"]})
+
+    top = generate_idf_components(
+        [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
+    )
+
+    assert [c.name for c in top] == ["esphome/A"]
+    # Ignored libraries were never resolved (and therefore never downloaded).
+    assert resolve_calls == ["A"]
+    # The ignored dependency is not wired into A's manifest.
+    assert top[0].dependencies == []
+    # lib_ignore changes the generated wiring, so the cache path is salted to
+    # keep this conversion separate from ones with a different lib_ignore.
+    assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]]
+
+
 def test_generate_idf_components_handles_dependency_cycle(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
@@ -575,7 +632,7 @@ def test_generate_idf_components_handles_dependency_cycle(
         },
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -632,7 +689,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
         "esphome/shared": {"name": "shared"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -669,7 +726,7 @@ def test_generate_idf_components_missing_manifest_raises(
 ) -> None:
     # A library with neither library.json nor library.properties is invalid;
     # fail loudly rather than silently generating build files for it.
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         # no library.json / library.properties written
@@ -711,7 +768,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
         "owner/shared": {"name": "shared"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -744,7 +801,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
 ) -> None:
     # A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
     # not be silently dropped.
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "library.json").write_text(
@@ -782,7 +839,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
         "esphome/B": {"name": "B", "platforms": ["espressif8266"]},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
@@ -804,3 +861,54 @@ def test_generate_idf_components_incompatible_dependency_skipped(
     assert [c.name for c in top] == ["esphome/A"]
     # The incompatible dependency was dropped, not wired in.
     assert top[0].dependencies == []
+
+
+def test_url_source_salt_changes_cache_path(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The salt is mixed into the URL hash so salted conversions get their own
+    cache tree. Pre-created extraction markers keep this network-free."""
+    monkeypatch.setattr(CORE, "config_path", tmp_path / "test.yaml")
+    url = "http://example.com/lib.tar.gz"
+    base = tmp_path / ".esphome" / "pio_components"
+    expected = {}
+    for salt in ("", "abcd1234"):
+        digest = hashlib.sha256((url + salt).encode()).hexdigest()[:8]
+        expected[salt] = base / digest / "lib"
+        expected[salt].mkdir(parents=True)
+        (expected[salt] / ".esphome_extracted").touch()
+
+    source = URLSource(url)
+    assert source.download("lib") == expected[""]
+    assert source.download("lib", salt="abcd1234") == expected["abcd1234"]
+
+
+def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The salt becomes a subdirectory of the git clone domain."""
+    domains: list[str] = []
+
+    def fake_clone_or_update(**kwargs):
+        domains.append(kwargs["domain"])
+        return Path("/cloned"), None
+
+    monkeypatch.setattr(
+        esphome.espidf.component.git, "clone_or_update", fake_clone_or_update
+    )
+
+    source = GitSource("https://github.com/esphome/noise-c.git", "v1.0")
+    source.download("noise-c")
+    source.download("noise-c", salt="abcd1234")
+    assert domains == ["pio_components", "pio_components/abcd1234"]
+
+
+def test_idf_component_download_passes_salt() -> None:
+    """IDFComponent.download forwards the sanitized name and salt to the
+    source and records the returned path."""
+    source = MagicMock()
+    source.download.return_value = Path("/converted/owner/name")
+
+    c = IDFComponent("owner/name", "1.0", source=source)
+    c.download(force=True, salt="abcd1234")
+
+    source.download.assert_called_once_with("owner/name", force=True, salt="abcd1234")
+    assert c.path == Path("/converted/owner/name")