From e191fc5d47284c2e0609c4fe368847d1fb33e79f Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:56:03 -0400 Subject: [PATCH] [core] Support platformio_options on the native ESP-IDF toolchain (#16917) --- esphome/core/__init__.py | 7 + esphome/core/config.py | 66 ++++++++-- esphome/espidf/component.py | 55 ++++++-- tests/unit_tests/core/test_config.py | 123 ++++++++++++++++++ .../fixtures/core/config/libraries.yaml | 8 ++ tests/unit_tests/test_core.py | 18 +++ tests/unit_tests/test_espidf_component.py | 122 ++++++++++++++++- 7 files changed, 366 insertions(+), 33 deletions(-) create mode 100644 tests/unit_tests/fixtures/core/config/libraries.yaml diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 4289cdf3e5..21ff7ef07c 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -958,6 +958,13 @@ class EsphomeCore: return build_flag def add_build_unflag(self, build_unflag: str) -> None: + if self.using_toolchain_esp_idf: + # The native ESP-IDF build generator does not consume build_unflags + _LOGGER.warning( + "Build unflag %s is ignored when building with the native " + "ESP-IDF toolchain", + build_unflag, + ) self.build_unflags.add(build_unflag) _LOGGER.debug("Adding build unflag: %s", build_unflag) diff --git a/esphome/core/config.py b/esphome/core/config.py index 8214fcf80c..b925f0b7d9 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -503,8 +503,58 @@ async def add_includes(includes: list[str], is_c_header: bool = False) -> None: include_file(path, basename, is_c_header) +def _add_library_str(lib: str) -> None: + if "@" in lib: + name, vers = lib.split("@", 1) + cg.add_library(name, vers) + elif "://" in lib: + # Repository... + if "=" in lib: + name, repo = lib.split("=", 1) + cg.add_library(name, None, repo) + else: + cg.add_library(None, None, lib) + else: + cg.add_library(lib, None) + + @coroutine_with_priority(CoroPriority.FINAL) -async def _add_platformio_options(pio_options): +async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None: + if CORE.using_toolchain_esp_idf: + # The native ESP-IDF build doesn't read platformio.ini; honor the + # options with a native equivalent and warn about the rest, which + # would otherwise be silently ignored. + for key, val in pio_options.items(): + vals = [val] if isinstance(val, str) else val + if key == CONF_BUILD_FLAGS: + # Deprecated: esphome->build_flags is the native equivalent. + # Remove before 2026.12.0 + _LOGGER.warning( + "esphome->platformio_options->build_flags is deprecated; use " + "esphome->build_flags instead. Support for it will be removed " + "in 2026.12.0." + ) + for flag in vals: + cg.add_build_flag(flag) + elif key == "lib_deps": + # Routed through the regular library mechanism so the libraries + # are converted to IDF components like any other PIO library + for lib in vals: + _add_library_str(lib) + elif key == "lib_ignore": + # Read by the PIO-library-to-IDF-component conversion + # (generate_idf_components); filters both top-level libraries + # and dependencies discovered during conversion + cg.add_platformio_option(key, vals) + elif key != "upload_speed": + # upload_speed needs no handling: it is read from the raw + # config at upload time (upload_using_esptool) + _LOGGER.warning( + "esphome->platformio_options->%s is ignored when building with " + "the native ESP-IDF toolchain", + key, + ) + return # Add includes at the very end, so that they override everything for key, val in pio_options.items(): if key in ["build_flags", "lib_ignore"] and not isinstance(val, list): @@ -655,19 +705,7 @@ async def to_code(config: ConfigType) -> None: # Libraries for lib in config[CONF_LIBRARIES]: - if "@" in lib: - name, vers = lib.split("@", 1) - cg.add_library(name, vers) - elif "://" in lib: - # Repository... - if "=" in lib: - name, repo = lib.split("=", 1) - cg.add_library(name, None, repo) - else: - cg.add_library(None, None, lib) - - else: - cg.add_library(lib, None) + _add_library_str(lib) cg.add_build_flag("-Wno-unused-variable") cg.add_build_flag("-Wno-unused-but-set-variable") diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 7398a91c36..cfd42916b2 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -56,7 +56,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE" class Source: - def download(self, dir_suffix: str, force: bool = False) -> Path: + def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path: raise NotImplementedError @@ -64,10 +64,12 @@ class URLSource(Source): def __init__(self, url: str): self.url = url - def download(self, dir_suffix: str, force: bool = False) -> Path: + def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path: base_dir = Path(CORE.data_dir) / DOMAIN h = hashlib.new("sha256") h.update(self.url.encode()) + if salt: + h.update(salt.encode()) path = base_dir / h.hexdigest()[:8] / dir_suffix # Marker file written last to signal a complete extraction. Using a # marker (instead of just `path.is_dir()`) means an interrupted @@ -99,12 +101,12 @@ class GitSource(Source): self.url = url self.ref = ref - def download(self, dir_suffix: str, force: bool = False) -> Path: + def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path: path, _ = git.clone_or_update( url=self.url, ref=self.ref, refresh=git.NEVER_REFRESH if not force else None, - domain=DOMAIN, + domain=f"{DOMAIN}/{salt}" if salt else DOMAIN, submodules=[], subpath=Path(dir_suffix), ) @@ -146,14 +148,16 @@ class IDFComponent: def get_require_name(self): return self.get_sanitized_name().replace("/", "__") - def download(self, force: bool = False): + def download(self, force: bool = False, salt: str = ""): """ The dependency name should match the directory name at the end of the override path. The ESP-IDF build system uses the directory name as the component name, so the directory of the override_path should match the component name. If you want to specify the full name of the component with the namespace, replace / in the component name with __. @see https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html """ - self.path = self.source.download(self.get_sanitized_name(), force=force) + self.path = self.source.download( + self.get_sanitized_name(), force=force, salt=salt + ) def _apply_extra_script(component: IDFComponent) -> None: @@ -699,9 +703,33 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: The returned list holds the top-level components (those directly requested); transitive dependencies are converted too and wired into each component's generated manifest. + + ``lib_ignore`` from ``esphome->platformio_options`` excludes libraries by + short name (part after the ``/``), matched against both the top-level + libraries and every dependency discovered during the graph walk. """ nodes: dict[str, _LibNode] = {} + lib_ignore = { + name.split("/")[-1].lower() + for name in CORE.platformio_options.get("lib_ignore", []) + } + + # The generated CMakeLists.txt/idf_component.yml inside the shared cache + # bake in the dependency wiring, which lib_ignore changes; salt the cache + # path so configs with different lib_ignore values don't fight over (and + # constantly rewrite) the same converted component files. + salt = ( + hashlib.sha256(",".join(sorted(lib_ignore)).encode()).hexdigest()[:8] + if lib_ignore + else "" + ) + + def is_ignored(name: str | None) -> bool: + if not lib_ignore or name is None: + return False + return name.split("/")[-1].lower() in lib_ignore + def add_spec(name: str | None, version: str | None, repository: str | None) -> str: key, is_git, locator = _node_key(name, version, repository) node = nodes.get(key) or _LibNode(key=key, is_git=is_git) @@ -718,6 +746,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: top_level = [ add_spec(library.name, library.version, library.repository) for library in libraries + if not is_ignored(library.name) ] # Collect + resolve to a fixpoint: a node is (re)resolved whenever its @@ -749,7 +778,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: component = IDFComponent( _owner_pkgname_to_name(owner, name), version, URLSource(url) ) - component.download() + component.download(salt=salt) library_json_path = component.path / "library.json" library_properties_path = component.path / "library.properties" @@ -787,6 +816,12 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: except InvalidIDFComponent as e: _LOGGER.debug("Skip dependency %s: %s", dependency.get("name"), str(e)) continue + dep_name = _owner_pkgname_to_name( + dependency.get("owner"), dependency.get("name") + ) + if is_ignored(dep_name): + _LOGGER.debug("Skip ignored dependency %s", dep_name) + continue # The version field may actually be a URL (git/archive dependency). dep_version = dependency["version"] dep_url = None @@ -796,11 +831,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: dep_url, dep_version = dep_version, None except (TypeError, ValueError): pass - dep_key = add_spec( - _owner_pkgname_to_name(dependency.get("owner"), dependency.get("name")), - dep_version, - dep_url, - ) + dep_key = add_spec(dep_name, dep_version, dep_url) node.edges.add(dep_key) worklist.append(dep_key) diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index ff150f2540..e2b34d92d8 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -20,6 +20,9 @@ from esphome.const import ( CONF_NAME, CONF_NAME_ADD_MAC_SUFFIX, KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Toolchain, ) from esphome.core import CORE, config from esphome.core.config import ( @@ -1161,3 +1164,123 @@ def test_make_app_name_cpp_special_chars_escaped() -> None: cpp_expr, _, _ = make_app_name_cpp('my "device"', "buf", "-", add_mac_suffix=False) # cpp_string_escape uses octal escapes for quotes assert '"' not in cpp_expr[1:-1] # no unescaped quotes inside the outer quotes + + +@pytest.mark.parametrize( + ("lib", "name", "version", "repository"), + [ + ("ArduinoJson", "ArduinoJson", None, None), + ("bblanchon/ArduinoJson@7.4.2", "bblanchon/ArduinoJson", "7.4.2", None), + ( + "noise-c=https://github.com/esphome/noise-c.git", + "noise-c", + None, + "https://github.com/esphome/noise-c.git", + ), + ], +) +def test_add_library_str( + lib: str, name: str, version: str | None, repository: str | None +) -> None: + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: "esp-idf", + } + + config._add_library_str(lib) + + libraries = list(CORE.platformio_libraries.values()) + assert len(libraries) == 1 + assert libraries[0].name == name + assert libraries[0].version == version + assert libraries[0].repository == repository + + +@pytest.mark.asyncio +async def test_add_platformio_options_native_idf( + caplog: pytest.LogCaptureFixture, +) -> None: + """On the native IDF toolchain, build_flags/lib_deps/lib_ignore are + honored, upload_speed is silent and everything else warns.""" + CORE.toolchain = Toolchain.ESP_IDF + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: "esp-idf", + } + + await config._add_platformio_options( + { + "build_flags": "-DSINGLE_FLAG", # string and list forms both valid + "lib_deps": ["bblanchon/ArduinoJson@7.4.2"], + "lib_ignore": "libsodium", + "upload_speed": "115200", + "board_build.f_flash": "80000000L", + } + ) + + assert "-DSINGLE_FLAG" in CORE.build_flags + assert "ArduinoJson" in CORE.platformio_libraries + # lib_ignore is stored (listified) for generate_idf_components to read; + # nothing else lands in platformio_options on the native toolchain. + assert CORE.platformio_options == {"lib_ignore": ["libsodium"]} + assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text + assert "upload_speed" not in caplog.text + # build_flags has a first-class esphome equivalent, so it is deprecated. + # lib_deps/lib_ignore are kept as valid platformio_options (no warning). + assert ( + "esphome->platformio_options->build_flags is deprecated; use " + "esphome->build_flags instead" in caplog.text + ) + assert "lib_deps is deprecated" not in caplog.text + assert "lib_ignore is deprecated" not in caplog.text + + +@pytest.mark.asyncio +async def test_add_platformio_options_platformio( + caplog: pytest.LogCaptureFixture, +) -> None: + """On the PlatformIO toolchain all options pass through to the ini, + with build_flags/lib_ignore listified.""" + CORE.toolchain = Toolchain.PLATFORMIO + + await config._add_platformio_options( + { + "build_flags": "-DSINGLE_FLAG", + "lib_ignore": "libsodium", + "upload_speed": "115200", + } + ) + + assert CORE.platformio_options == { + "build_flags": ["-DSINGLE_FLAG"], + "lib_ignore": ["libsodium"], + "upload_speed": "115200", + } + # platformio_options is the correct mechanism on the PlatformIO toolchain, + # so the native-equivalent deprecation must not fire here. + assert "deprecated" not in caplog.text + + +def test_add_library_str_bare_url_requires_name() -> None: + """A bare repository URL has no library name; CORE.add_library rejects it.""" + with pytest.raises(ValueError, match="must have a name"): + config._add_library_str("https://github.com/esphome/noise-c.git") + + +@pytest.mark.asyncio +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +async def test_to_code_adds_libraries(yaml_file: Callable[[str], Path]) -> None: + """esphome->libraries entries are parsed and registered via cg.add_library.""" + result = load_config_from_fixture(yaml_file, "libraries.yaml", FIXTURES_DIR) + assert result is not None + + with patch("esphome.core.config.cg") as mock_cg: + mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock() + mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock() + await config.to_code(result[CONF_ESPHOME]) + + mock_cg.add_library.assert_any_call("SomeLib", None) + mock_cg.add_library.assert_any_call("bblanchon/ArduinoJson", "7.4.2") + mock_cg.add_library.assert_any_call( + "noise-c", None, "https://github.com/esphome/noise-c.git" + ) diff --git a/tests/unit_tests/fixtures/core/config/libraries.yaml b/tests/unit_tests/fixtures/core/config/libraries.yaml new file mode 100644 index 0000000000..c93e828f31 --- /dev/null +++ b/tests/unit_tests/fixtures/core/config/libraries.yaml @@ -0,0 +1,8 @@ +esphome: + name: test-libraries + libraries: + - SomeLib + - bblanchon/ArduinoJson@7.4.2 + - noise-c=https://github.com/esphome/noise-c.git + +host: diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py index cc371ee1f9..a61b6ae7ae 100644 --- a/tests/unit_tests/test_core.py +++ b/tests/unit_tests/test_core.py @@ -915,3 +915,21 @@ class TestEsphomeCore: mock_enable.assert_called_once_with("Wire") assert "Wire" in target.platformio_libraries + + def test_add_build_unflag__warns_on_native_idf_toolchain( + self, target, caplog: pytest.LogCaptureFixture + ) -> None: + """Build unflags are not consumed by the native IDF build generator, + so adding one on that toolchain warns; PlatformIO stays silent.""" + target.toolchain = const.Toolchain.PLATFORMIO + target.add_build_unflag("-fno-rtti") + assert "ignored" not in caplog.text + + target.toolchain = const.Toolchain.ESP_IDF + target.add_build_unflag("-fno-exceptions") + assert ( + "Build unflag -fno-exceptions is ignored when building with the " + "native ESP-IDF toolchain" in caplog.text + ) + # The unflag is still recorded either way. + assert target.build_unflags == {"-fno-rtti", "-fno-exceptions"} diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 602ff03942..87e168dc94 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -1,3 +1,4 @@ +import hashlib import json import os from pathlib import Path @@ -515,7 +516,7 @@ def test_generate_idf_components_dedupes_shared_dependency( "esphome/C": {"name": "C"}, } - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "src" / "x.c").write_text("int x;") @@ -557,6 +558,62 @@ def test_generate_idf_components_dedupes_shared_dependency( assert "idf_component_register" in generated +def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # lib_ignore must drop B at the top level and C when it is discovered as a + # dependency of A during the graph walk -- neither may be resolved, + # downloaded, or wired into a manifest. Matching is by lowercase short name. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [ + {"owner": "esphome", "name": "C", "version": "==1.10021.0"} + ], + }, + "esphome/B": {"name": "B"}, + } + + download_salts: list[str] = [] + + def fake_download(self, force=False, salt=""): + download_salts.append(salt) + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz" + + monkeypatch.setattr( + esphome.espidf.component, "_resolve_registry_version", fake_resolve + ) + # lib_ignore is read from CORE.platformio_options (stored there by + # _add_platformio_options); matched by lowercase short name. + monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["B", "esphome/C"]}) + + top = generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)] + ) + + assert [c.name for c in top] == ["esphome/A"] + # Ignored libraries were never resolved (and therefore never downloaded). + assert resolve_calls == ["A"] + # The ignored dependency is not wired into A's manifest. + assert top[0].dependencies == [] + # lib_ignore changes the generated wiring, so the cache path is salted to + # keep this conversion separate from ones with a different lib_ignore. + assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]] + + def test_generate_idf_components_handles_dependency_cycle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -575,7 +632,7 @@ def test_generate_idf_components_handles_dependency_cycle( }, } - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "src" / "x.c").write_text("int x;") @@ -632,7 +689,7 @@ def test_generate_idf_components_git_overrides_registry_warns( "esphome/shared": {"name": "shared"}, } - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "src" / "x.c").write_text("int x;") @@ -669,7 +726,7 @@ def test_generate_idf_components_missing_manifest_raises( ) -> None: # A library with neither library.json nor library.properties is invalid; # fail loudly rather than silently generating build files for it. - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) # no library.json / library.properties written @@ -711,7 +768,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate( "owner/shared": {"name": "shared"}, } - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "src" / "x.c").write_text("int x;") @@ -744,7 +801,7 @@ def test_generate_idf_components_incompatible_top_level_raises( ) -> None: # A top-level library that isn't ESP-IDF/esp32 compatible must fail fast, # not be silently dropped. - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "library.json").write_text( @@ -782,7 +839,7 @@ def test_generate_idf_components_incompatible_dependency_skipped( "esphome/B": {"name": "B", "platforms": ["espressif8266"]}, } - def fake_download(self, force=False): + def fake_download(self, force=False, salt=""): self.path = tmp_path / self.get_sanitized_name().replace("/", "__") (self.path / "src").mkdir(parents=True, exist_ok=True) (self.path / "library.json").write_text(json.dumps(manifests[self.name])) @@ -804,3 +861,54 @@ def test_generate_idf_components_incompatible_dependency_skipped( assert [c.name for c in top] == ["esphome/A"] # The incompatible dependency was dropped, not wired in. assert top[0].dependencies == [] + + +def test_url_source_salt_changes_cache_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The salt is mixed into the URL hash so salted conversions get their own + cache tree. Pre-created extraction markers keep this network-free.""" + monkeypatch.setattr(CORE, "config_path", tmp_path / "test.yaml") + url = "http://example.com/lib.tar.gz" + base = tmp_path / ".esphome" / "pio_components" + expected = {} + for salt in ("", "abcd1234"): + digest = hashlib.sha256((url + salt).encode()).hexdigest()[:8] + expected[salt] = base / digest / "lib" + expected[salt].mkdir(parents=True) + (expected[salt] / ".esphome_extracted").touch() + + source = URLSource(url) + assert source.download("lib") == expected[""] + assert source.download("lib", salt="abcd1234") == expected["abcd1234"] + + +def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None: + """The salt becomes a subdirectory of the git clone domain.""" + domains: list[str] = [] + + def fake_clone_or_update(**kwargs): + domains.append(kwargs["domain"]) + return Path("/cloned"), None + + monkeypatch.setattr( + esphome.espidf.component.git, "clone_or_update", fake_clone_or_update + ) + + source = GitSource("https://github.com/esphome/noise-c.git", "v1.0") + source.download("noise-c") + source.download("noise-c", salt="abcd1234") + assert domains == ["pio_components", "pio_components/abcd1234"] + + +def test_idf_component_download_passes_salt() -> None: + """IDFComponent.download forwards the sanitized name and salt to the + source and records the returned path.""" + source = MagicMock() + source.download.return_value = Path("/converted/owner/name") + + c = IDFComponent("owner/name", "1.0", source=source) + c.download(force=True, salt="abcd1234") + + source.download.assert_called_once_with("owner/name", force=True, salt="abcd1234") + assert c.path == Path("/converted/owner/name")