[core] Support platformio_options on the native ESP-IDF toolchain (#16917)

esphome/core/__init__.py

@@ -958,6 +958,13 @@ class EsphomeCore:
         return build_flag
 
     def add_build_unflag(self, build_unflag: str) -> None:
+        if self.using_toolchain_esp_idf:
+            # The native ESP-IDF build generator does not consume build_unflags
+            _LOGGER.warning(
+                "Build unflag %s is ignored when building with the native "
+                "ESP-IDF toolchain",
+                build_unflag,
+            )
         self.build_unflags.add(build_unflag)
         _LOGGER.debug("Adding build unflag: %s", build_unflag)
 

esphome/core/config.py

@@ -503,8 +503,58 @@ async def add_includes(includes: list[str], is_c_header: bool = False) -> None:
             include_file(path, basename, is_c_header)
 
 
+def _add_library_str(lib: str) -> None:
+    if "@" in lib:
+        name, vers = lib.split("@", 1)
+        cg.add_library(name, vers)
+    elif "://" in lib:
+        # Repository...
+        if "=" in lib:
+            name, repo = lib.split("=", 1)
+            cg.add_library(name, None, repo)
+        else:
+            cg.add_library(None, None, lib)
+    else:
+        cg.add_library(lib, None)
+
+
 @coroutine_with_priority(CoroPriority.FINAL)
-async def _add_platformio_options(pio_options):
+async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None:
+    if CORE.using_toolchain_esp_idf:
+        # The native ESP-IDF build doesn't read platformio.ini; honor the
+        # options with a native equivalent and warn about the rest, which
+        # would otherwise be silently ignored.
+        for key, val in pio_options.items():
+            vals = [val] if isinstance(val, str) else val
+            if key == CONF_BUILD_FLAGS:
+                # Deprecated: esphome->build_flags is the native equivalent.
+                # Remove before 2026.12.0
+                _LOGGER.warning(
+                    "esphome->platformio_options->build_flags is deprecated; use "
+                    "esphome->build_flags instead. Support for it will be removed "
+                    "in 2026.12.0."
+                )
+                for flag in vals:
+                    cg.add_build_flag(flag)
+            elif key == "lib_deps":
+                # Routed through the regular library mechanism so the libraries
+                # are converted to IDF components like any other PIO library
+                for lib in vals:
+                    _add_library_str(lib)
+            elif key == "lib_ignore":
+                # Read by the PIO-library-to-IDF-component conversion
+                # (generate_idf_components); filters both top-level libraries
+                # and dependencies discovered during conversion
+                cg.add_platformio_option(key, vals)
+            elif key != "upload_speed":
+                # upload_speed needs no handling: it is read from the raw
+                # config at upload time (upload_using_esptool)
+                _LOGGER.warning(
+                    "esphome->platformio_options->%s is ignored when building with "
+                    "the native ESP-IDF toolchain",
+                    key,
+                )
+        return
     # Add includes at the very end, so that they override everything
     for key, val in pio_options.items():
         if key in ["build_flags", "lib_ignore"] and not isinstance(val, list):
@@ -655,19 +705,7 @@ async def to_code(config: ConfigType) -> None:
 
     # Libraries
     for lib in config[CONF_LIBRARIES]:
-        if "@" in lib:
+        _add_library_str(lib)
-            name, vers = lib.split("@", 1)
-            cg.add_library(name, vers)
-        elif "://" in lib:
-            # Repository...
-            if "=" in lib:
-                name, repo = lib.split("=", 1)
-                cg.add_library(name, None, repo)
-            else:
-                cg.add_library(None, None, lib)
-
-        else:
-            cg.add_library(lib, None)
 
     cg.add_build_flag("-Wno-unused-variable")
     cg.add_build_flag("-Wno-unused-but-set-variable")

esphome/espidf/component.py

@@ -56,7 +56,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE"
 
 
 class Source:
-    def download(self, dir_suffix: str, force: bool = False) -> Path:
+    def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
         raise NotImplementedError
 
 
@@ -64,10 +64,12 @@ class URLSource(Source):
     def __init__(self, url: str):
         self.url = url
 
-    def download(self, dir_suffix: str, force: bool = False) -> Path:
+    def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
         base_dir = Path(CORE.data_dir) / DOMAIN
         h = hashlib.new("sha256")
         h.update(self.url.encode())
+        if salt:
+            h.update(salt.encode())
         path = base_dir / h.hexdigest()[:8] / dir_suffix
         # Marker file written last to signal a complete extraction. Using a
         # marker (instead of just `path.is_dir()`) means an interrupted
@@ -99,12 +101,12 @@ class GitSource(Source):
         self.url = url
         self.ref = ref
 
-    def download(self, dir_suffix: str, force: bool = False) -> Path:
+    def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
         path, _ = git.clone_or_update(
             url=self.url,
             ref=self.ref,
             refresh=git.NEVER_REFRESH if not force else None,
-            domain=DOMAIN,
+            domain=f"{DOMAIN}/{salt}" if salt else DOMAIN,
             submodules=[],
             subpath=Path(dir_suffix),
         )
@@ -146,14 +148,16 @@ class IDFComponent:
     def get_require_name(self):
         return self.get_sanitized_name().replace("/", "__")
 
-    def download(self, force: bool = False):
+    def download(self, force: bool = False, salt: str = ""):
         """
         The dependency name should match the directory name at the end of the override path.
         The ESP-IDF build system uses the directory name as the component name, so the directory of the override_path should match the component name.
         If you want to specify the full name of the component with the namespace, replace / in the component name with __.
         @see https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html
         """
-        self.path = self.source.download(self.get_sanitized_name(), force=force)
+        self.path = self.source.download(
+            self.get_sanitized_name(), force=force, salt=salt
+        )
 
 
 def _apply_extra_script(component: IDFComponent) -> None:
@@ -699,9 +703,33 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
     The returned list holds the top-level components (those directly requested);
     transitive dependencies are converted too and wired into each component's
     generated manifest.
+
+    ``lib_ignore`` from ``esphome->platformio_options`` excludes libraries by
+    short name (part after the ``/``), matched against both the top-level
+    libraries and every dependency discovered during the graph walk.
     """
     nodes: dict[str, _LibNode] = {}
 
+    lib_ignore = {
+        name.split("/")[-1].lower()
+        for name in CORE.platformio_options.get("lib_ignore", [])
+    }
+
+    # The generated CMakeLists.txt/idf_component.yml inside the shared cache
+    # bake in the dependency wiring, which lib_ignore changes; salt the cache
+    # path so configs with different lib_ignore values don't fight over (and
+    # constantly rewrite) the same converted component files.
+    salt = (
+        hashlib.sha256(",".join(sorted(lib_ignore)).encode()).hexdigest()[:8]
+        if lib_ignore
+        else ""
+    )
+
+    def is_ignored(name: str | None) -> bool:
+        if not lib_ignore or name is None:
+            return False
+        return name.split("/")[-1].lower() in lib_ignore
+
     def add_spec(name: str | None, version: str | None, repository: str | None) -> str:
         key, is_git, locator = _node_key(name, version, repository)
         node = nodes.get(key) or _LibNode(key=key, is_git=is_git)
@@ -718,6 +746,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
     top_level = [
         add_spec(library.name, library.version, library.repository)
         for library in libraries
+        if not is_ignored(library.name)
     ]
 
     # Collect + resolve to a fixpoint: a node is (re)resolved whenever its
@@ -749,7 +778,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
             component = IDFComponent(
                 _owner_pkgname_to_name(owner, name), version, URLSource(url)
             )
-        component.download()
+        component.download(salt=salt)
 
         library_json_path = component.path / "library.json"
         library_properties_path = component.path / "library.properties"
@@ -787,6 +816,12 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
             except InvalidIDFComponent as e:
                 _LOGGER.debug("Skip dependency %s: %s", dependency.get("name"), str(e))
                 continue
+            dep_name = _owner_pkgname_to_name(
+                dependency.get("owner"), dependency.get("name")
+            )
+            if is_ignored(dep_name):
+                _LOGGER.debug("Skip ignored dependency %s", dep_name)
+                continue
             # The version field may actually be a URL (git/archive dependency).
             dep_version = dependency["version"]
             dep_url = None
@@ -796,11 +831,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
                     dep_url, dep_version = dep_version, None
             except (TypeError, ValueError):
                 pass
-            dep_key = add_spec(
+            dep_key = add_spec(dep_name, dep_version, dep_url)
-                _owner_pkgname_to_name(dependency.get("owner"), dependency.get("name")),
-                dep_version,
-                dep_url,
-            )
             node.edges.add(dep_key)
             worklist.append(dep_key)
 

tests/unit_tests/core/test_config.py

@@ -20,6 +20,9 @@ from esphome.const import (
     CONF_NAME,
     CONF_NAME_ADD_MAC_SUFFIX,
     KEY_CORE,
+    KEY_TARGET_FRAMEWORK,
+    KEY_TARGET_PLATFORM,
+    Toolchain,
 )
 from esphome.core import CORE, config
 from esphome.core.config import (
@@ -1161,3 +1164,123 @@ def test_make_app_name_cpp_special_chars_escaped() -> None:
     cpp_expr, _, _ = make_app_name_cpp('my "device"', "buf", "-", add_mac_suffix=False)
     # cpp_string_escape uses octal escapes for quotes
     assert '"' not in cpp_expr[1:-1]  # no unescaped quotes inside the outer quotes
+
+
+@pytest.mark.parametrize(
+    ("lib", "name", "version", "repository"),
+    [
+        ("ArduinoJson", "ArduinoJson", None, None),
+        ("bblanchon/ArduinoJson@7.4.2", "bblanchon/ArduinoJson", "7.4.2", None),
+        (
+            "noise-c=https://github.com/esphome/noise-c.git",
+            "noise-c",
+            None,
+            "https://github.com/esphome/noise-c.git",
+        ),
+    ],
+)
+def test_add_library_str(
+    lib: str, name: str, version: str | None, repository: str | None
+) -> None:
+    CORE.data[KEY_CORE] = {
+        KEY_TARGET_PLATFORM: "esp32",
+        KEY_TARGET_FRAMEWORK: "esp-idf",
+    }
+
+    config._add_library_str(lib)
+
+    libraries = list(CORE.platformio_libraries.values())
+    assert len(libraries) == 1
+    assert libraries[0].name == name
+    assert libraries[0].version == version
+    assert libraries[0].repository == repository
+
+
+@pytest.mark.asyncio
+async def test_add_platformio_options_native_idf(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """On the native IDF toolchain, build_flags/lib_deps/lib_ignore are
+    honored, upload_speed is silent and everything else warns."""
+    CORE.toolchain = Toolchain.ESP_IDF
+    CORE.data[KEY_CORE] = {
+        KEY_TARGET_PLATFORM: "esp32",
+        KEY_TARGET_FRAMEWORK: "esp-idf",
+    }
+
+    await config._add_platformio_options(
+        {
+            "build_flags": "-DSINGLE_FLAG",  # string and list forms both valid
+            "lib_deps": ["bblanchon/ArduinoJson@7.4.2"],
+            "lib_ignore": "libsodium",
+            "upload_speed": "115200",
+            "board_build.f_flash": "80000000L",
+        }
+    )
+
+    assert "-DSINGLE_FLAG" in CORE.build_flags
+    assert "ArduinoJson" in CORE.platformio_libraries
+    # lib_ignore is stored (listified) for generate_idf_components to read;
+    # nothing else lands in platformio_options on the native toolchain.
+    assert CORE.platformio_options == {"lib_ignore": ["libsodium"]}
+    assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text
+    assert "upload_speed" not in caplog.text
+    # build_flags has a first-class esphome equivalent, so it is deprecated.
+    # lib_deps/lib_ignore are kept as valid platformio_options (no warning).
+    assert (
+        "esphome->platformio_options->build_flags is deprecated; use "
+        "esphome->build_flags instead" in caplog.text
+    )
+    assert "lib_deps is deprecated" not in caplog.text
+    assert "lib_ignore is deprecated" not in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_add_platformio_options_platformio(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """On the PlatformIO toolchain all options pass through to the ini,
+    with build_flags/lib_ignore listified."""
+    CORE.toolchain = Toolchain.PLATFORMIO
+
+    await config._add_platformio_options(
+        {
+            "build_flags": "-DSINGLE_FLAG",
+            "lib_ignore": "libsodium",
+            "upload_speed": "115200",
+        }
+    )
+
+    assert CORE.platformio_options == {
+        "build_flags": ["-DSINGLE_FLAG"],
+        "lib_ignore": ["libsodium"],
+        "upload_speed": "115200",
+    }
+    # platformio_options is the correct mechanism on the PlatformIO toolchain,
+    # so the native-equivalent deprecation must not fire here.
+    assert "deprecated" not in caplog.text
+
+
+def test_add_library_str_bare_url_requires_name() -> None:
+    """A bare repository URL has no library name; CORE.add_library rejects it."""
+    with pytest.raises(ValueError, match="must have a name"):
+        config._add_library_str("https://github.com/esphome/noise-c.git")
+
+
+@pytest.mark.asyncio
+@pytest.mark.filterwarnings("ignore::RuntimeWarning")
+async def test_to_code_adds_libraries(yaml_file: Callable[[str], Path]) -> None:
+    """esphome->libraries entries are parsed and registered via cg.add_library."""
+    result = load_config_from_fixture(yaml_file, "libraries.yaml", FIXTURES_DIR)
+    assert result is not None
+
+    with patch("esphome.core.config.cg") as mock_cg:
+        mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock()
+        mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock()
+        await config.to_code(result[CONF_ESPHOME])
+
+    mock_cg.add_library.assert_any_call("SomeLib", None)
+    mock_cg.add_library.assert_any_call("bblanchon/ArduinoJson", "7.4.2")
+    mock_cg.add_library.assert_any_call(
+        "noise-c", None, "https://github.com/esphome/noise-c.git"
+    )

tests/unit_tests/fixtures/core/config/libraries.yaml

@@ -0,0 +1,8 @@
+esphome:
+  name: test-libraries
+  libraries:
+    - SomeLib
+    - bblanchon/ArduinoJson@7.4.2
+    - noise-c=https://github.com/esphome/noise-c.git
+
+host:

tests/unit_tests/test_core.py

@@ -915,3 +915,21 @@ class TestEsphomeCore:
             mock_enable.assert_called_once_with("Wire")
 
         assert "Wire" in target.platformio_libraries
+
+    def test_add_build_unflag__warns_on_native_idf_toolchain(
+        self, target, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Build unflags are not consumed by the native IDF build generator,
+        so adding one on that toolchain warns; PlatformIO stays silent."""
+        target.toolchain = const.Toolchain.PLATFORMIO
+        target.add_build_unflag("-fno-rtti")
+        assert "ignored" not in caplog.text
+
+        target.toolchain = const.Toolchain.ESP_IDF
+        target.add_build_unflag("-fno-exceptions")
+        assert (
+            "Build unflag -fno-exceptions is ignored when building with the "
+            "native ESP-IDF toolchain" in caplog.text
+        )
+        # The unflag is still recorded either way.
+        assert target.build_unflags == {"-fno-rtti", "-fno-exceptions"}

tests/unit_tests/test_espidf_component.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -515,7 +516,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
         "esphome/C": {"name": "C"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -557,6 +558,62 @@ def test_generate_idf_components_dedupes_shared_dependency(
     assert "idf_component_register" in generated
 
 
+def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    esp32_idf_core: None,
+) -> None:
+    # lib_ignore must drop B at the top level and C when it is discovered as a
+    # dependency of A during the graph walk -- neither may be resolved,
+    # downloaded, or wired into a manifest. Matching is by lowercase short name.
+    manifests = {
+        "esphome/A": {
+            "name": "A",
+            "dependencies": [
+                {"owner": "esphome", "name": "C", "version": "==1.10021.0"}
+            ],
+        },
+        "esphome/B": {"name": "B"},
+    }
+
+    download_salts: list[str] = []
+
+    def fake_download(self, force=False, salt=""):
+        download_salts.append(salt)
+        self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
+        (self.path / "src").mkdir(parents=True, exist_ok=True)
+        (self.path / "src" / "x.c").write_text("int x;")
+        (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
+
+    monkeypatch.setattr(IDFComponent, "download", fake_download)
+
+    resolve_calls: list[str] = []
+
+    def fake_resolve(owner, pkgname, requirements):
+        resolve_calls.append(pkgname)
+        return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
+
+    monkeypatch.setattr(
+        esphome.espidf.component, "_resolve_registry_version", fake_resolve
+    )
+    # lib_ignore is read from CORE.platformio_options (stored there by
+    # _add_platformio_options); matched by lowercase short name.
+    monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["B", "esphome/C"]})
+
+    top = generate_idf_components(
+        [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
+    )
+
+    assert [c.name for c in top] == ["esphome/A"]
+    # Ignored libraries were never resolved (and therefore never downloaded).
+    assert resolve_calls == ["A"]
+    # The ignored dependency is not wired into A's manifest.
+    assert top[0].dependencies == []
+    # lib_ignore changes the generated wiring, so the cache path is salted to
+    # keep this conversion separate from ones with a different lib_ignore.
+    assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]]
+
+
 def test_generate_idf_components_handles_dependency_cycle(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
@@ -575,7 +632,7 @@ def test_generate_idf_components_handles_dependency_cycle(
         },
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -632,7 +689,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
         "esphome/shared": {"name": "shared"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -669,7 +726,7 @@ def test_generate_idf_components_missing_manifest_raises(
 ) -> None:
     # A library with neither library.json nor library.properties is invalid;
     # fail loudly rather than silently generating build files for it.
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         # no library.json / library.properties written
@@ -711,7 +768,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
         "owner/shared": {"name": "shared"},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "src" / "x.c").write_text("int x;")
@@ -744,7 +801,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
 ) -> None:
     # A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
     # not be silently dropped.
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "library.json").write_text(
@@ -782,7 +839,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
         "esphome/B": {"name": "B", "platforms": ["espressif8266"]},
     }
 
-    def fake_download(self, force=False):
+    def fake_download(self, force=False, salt=""):
         self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
         (self.path / "src").mkdir(parents=True, exist_ok=True)
         (self.path / "library.json").write_text(json.dumps(manifests[self.name]))
@@ -804,3 +861,54 @@ def test_generate_idf_components_incompatible_dependency_skipped(
     assert [c.name for c in top] == ["esphome/A"]
     # The incompatible dependency was dropped, not wired in.
     assert top[0].dependencies == []
+
+
+def test_url_source_salt_changes_cache_path(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The salt is mixed into the URL hash so salted conversions get their own
+    cache tree. Pre-created extraction markers keep this network-free."""
+    monkeypatch.setattr(CORE, "config_path", tmp_path / "test.yaml")
+    url = "http://example.com/lib.tar.gz"
+    base = tmp_path / ".esphome" / "pio_components"
+    expected = {}
+    for salt in ("", "abcd1234"):
+        digest = hashlib.sha256((url + salt).encode()).hexdigest()[:8]
+        expected[salt] = base / digest / "lib"
+        expected[salt].mkdir(parents=True)
+        (expected[salt] / ".esphome_extracted").touch()
+
+    source = URLSource(url)
+    assert source.download("lib") == expected[""]
+    assert source.download("lib", salt="abcd1234") == expected["abcd1234"]
+
+
+def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The salt becomes a subdirectory of the git clone domain."""
+    domains: list[str] = []
+
+    def fake_clone_or_update(**kwargs):
+        domains.append(kwargs["domain"])
+        return Path("/cloned"), None
+
+    monkeypatch.setattr(
+        esphome.espidf.component.git, "clone_or_update", fake_clone_or_update
+    )
+
+    source = GitSource("https://github.com/esphome/noise-c.git", "v1.0")
+    source.download("noise-c")
+    source.download("noise-c", salt="abcd1234")
+    assert domains == ["pio_components", "pio_components/abcd1234"]
+
+
+def test_idf_component_download_passes_salt() -> None:
+    """IDFComponent.download forwards the sanitized name and salt to the
+    source and records the returned path."""
+    source = MagicMock()
+    source.download.return_value = Path("/converted/owner/name")
+
+    c = IDFComponent("owner/name", "1.0", source=source)
+    c.download(force=True, salt="abcd1234")
+
+    source.download.assert_called_once_with("owner/name", force=True, salt="abcd1234")
+    assert c.path == Path("/converted/owner/name")