github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-06-24 13:45:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[core] Support platformio_options on the native ESP-IDF toolchain (#16917)

Browse Source
This commit is contained in:
Jonathan Swoboda
2026-06-14 18:56:03 -04:00
committed by Jesse Hills
parent c768e2eabc
commit f83e3ad6a6
7 changed files with 366 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
esphome/core/__init__.py
View File

@@ -958,6 +958,13 @@ class EsphomeCore:
return build_flag
def add_build_unflag(self, build_unflag: str) -> None:
if self.using_toolchain_esp_idf:
# The native ESP-IDF build generator does not consume build_unflags
_LOGGER.warning(
"Build unflag %s is ignored when building with the native "
"ESP-IDF toolchain",
build_unflag,
)
self.build_unflags.add(build_unflag)
_LOGGER.debug("Adding build unflag: %s", build_unflag)

66
esphome/core/config.py
View File

@@ -503,8 +503,58 @@ async def add_includes(includes: list[str], is_c_header: bool = False) -> None:
include_file(path, basename, is_c_header)
def _add_library_str(lib: str) -> None:
if "@" in lib:
name, vers = lib.split("@", 1)
cg.add_library(name, vers)
elif "://" in lib:
# Repository...
if "=" in lib:
name, repo = lib.split("=", 1)
cg.add_library(name, None, repo)
else:
cg.add_library(None, None, lib)
else:
cg.add_library(lib, None)
@coroutine_with_priority(CoroPriority.FINAL)
async def _add_platformio_options(pio_options):
async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> None:
if CORE.using_toolchain_esp_idf:
# The native ESP-IDF build doesn't read platformio.ini; honor the
# options with a native equivalent and warn about the rest, which
# would otherwise be silently ignored.
for key, val in pio_options.items():
vals = [val] if isinstance(val, str) else val
if key == CONF_BUILD_FLAGS:
# Deprecated: esphome->build_flags is the native equivalent.
# Remove before 2026.12.0
_LOGGER.warning(
"esphome->platformio_options->build_flags is deprecated; use "
"esphome->build_flags instead. Support for it will be removed "
"in 2026.12.0."
)
for flag in vals:
cg.add_build_flag(flag)
elif key == "lib_deps":
# Routed through the regular library mechanism so the libraries
# are converted to IDF components like any other PIO library
for lib in vals:
_add_library_str(lib)
elif key == "lib_ignore":
# Read by the PIO-library-to-IDF-component conversion
# (generate_idf_components); filters both top-level libraries
# and dependencies discovered during conversion
cg.add_platformio_option(key, vals)
elif key != "upload_speed":
# upload_speed needs no handling: it is read from the raw
# config at upload time (upload_using_esptool)
_LOGGER.warning(
"esphome->platformio_options->%s is ignored when building with "
"the native ESP-IDF toolchain",
key,
)
return
# Add includes at the very end, so that they override everything
for key, val in pio_options.items():
if key in ["build_flags", "lib_ignore"] and not isinstance(val, list):
@@ -655,19 +705,7 @@ async def to_code(config: ConfigType) -> None:
# Libraries
for lib in config[CONF_LIBRARIES]:
if "@" in lib:
name, vers = lib.split("@", 1)
cg.add_library(name, vers)
elif "://" in lib:
# Repository...
if "=" in lib:
name, repo = lib.split("=", 1)
cg.add_library(name, None, repo)
else:
cg.add_library(None, None, lib)
else:
cg.add_library(lib, None)
_add_library_str(lib)
cg.add_build_flag("-Wno-unused-variable")
cg.add_build_flag("-Wno-unused-but-set-variable")

55
esphome/espidf/component.py
View File

@@ -56,7 +56,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE"
class Source:
def download(self, dir_suffix: str, force: bool = False) -> Path:
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
raise NotImplementedError
@@ -64,10 +64,12 @@ class URLSource(Source):
def __init__(self, url: str):
self.url = url
def download(self, dir_suffix: str, force: bool = False) -> Path:
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
base_dir = Path(CORE.data_dir) / DOMAIN
h = hashlib.new("sha256")
h.update(self.url.encode())
if salt:
h.update(salt.encode())
path = base_dir / h.hexdigest()[:8] / dir_suffix
# Marker file written last to signal a complete extraction. Using a
# marker (instead of just `path.is_dir()`) means an interrupted
@@ -99,12 +101,12 @@ class GitSource(Source):
self.url = url
self.ref = ref
def download(self, dir_suffix: str, force: bool = False) -> Path:
def download(self, dir_suffix: str, force: bool = False, salt: str = "") -> Path:
path, _ = git.clone_or_update(
url=self.url,
ref=self.ref,
refresh=git.NEVER_REFRESH if not force else None,
domain=DOMAIN,
domain=f"{DOMAIN}/{salt}" if salt else DOMAIN,
submodules=[],
subpath=Path(dir_suffix),
)
@@ -146,14 +148,16 @@ class IDFComponent:
def get_require_name(self):
return self.get_sanitized_name().replace("/", "__")
def download(self, force: bool = False):
def download(self, force: bool = False, salt: str = ""):
"""
The dependency name should match the directory name at the end of the override path.
The ESP-IDF build system uses the directory name as the component name, so the directory of the override_path should match the component name.
If you want to specify the full name of the component with the namespace, replace / in the component name with __.
@see https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html
"""
self.path = self.source.download(self.get_sanitized_name(), force=force)
self.path = self.source.download(
self.get_sanitized_name(), force=force, salt=salt
)
def _apply_extra_script(component: IDFComponent) -> None:
@@ -699,9 +703,33 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
The returned list holds the top-level components (those directly requested);
transitive dependencies are converted too and wired into each component's
generated manifest.
``lib_ignore`` from ``esphome->platformio_options`` excludes libraries by
short name (part after the ``/``), matched against both the top-level
libraries and every dependency discovered during the graph walk.
"""
nodes: dict[str, _LibNode] = {}
lib_ignore = {
name.split("/")[-1].lower()
for name in CORE.platformio_options.get("lib_ignore", [])
}
# The generated CMakeLists.txt/idf_component.yml inside the shared cache
# bake in the dependency wiring, which lib_ignore changes; salt the cache
# path so configs with different lib_ignore values don't fight over (and
# constantly rewrite) the same converted component files.
salt = (
hashlib.sha256(",".join(sorted(lib_ignore)).encode()).hexdigest()[:8]
if lib_ignore
else ""
)
def is_ignored(name: str | None) -> bool:
if not lib_ignore or name is None:
return False
return name.split("/")[-1].lower() in lib_ignore
def add_spec(name: str | None, version: str | None, repository: str | None) -> str:
key, is_git, locator = _node_key(name, version, repository)
node = nodes.get(key) or _LibNode(key=key, is_git=is_git)
@@ -718,6 +746,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
top_level = [
add_spec(library.name, library.version, library.repository)
for library in libraries
if not is_ignored(library.name)
]
# Collect + resolve to a fixpoint: a node is (re)resolved whenever its
@@ -749,7 +778,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
component = IDFComponent(
_owner_pkgname_to_name(owner, name), version, URLSource(url)
)
component.download()
component.download(salt=salt)
library_json_path = component.path / "library.json"
library_properties_path = component.path / "library.properties"
@@ -787,6 +816,12 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
except InvalidIDFComponent as e:
_LOGGER.debug("Skip dependency %s: %s", dependency.get("name"), str(e))
continue
dep_name = _owner_pkgname_to_name(
dependency.get("owner"), dependency.get("name")
)
if is_ignored(dep_name):
_LOGGER.debug("Skip ignored dependency %s", dep_name)
continue
# The version field may actually be a URL (git/archive dependency).
dep_version = dependency["version"]
dep_url = None
@@ -796,11 +831,7 @@ def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]:
dep_url, dep_version = dep_version, None
except (TypeError, ValueError):
pass
dep_key = add_spec(
_owner_pkgname_to_name(dependency.get("owner"), dependency.get("name")),
dep_version,
dep_url,
)
dep_key = add_spec(dep_name, dep_version, dep_url)
node.edges.add(dep_key)
worklist.append(dep_key)

123
tests/unit_tests/core/test_config.py
View File

@@ -20,6 +20,9 @@ from esphome.const import (
CONF_NAME,
CONF_NAME_ADD_MAC_SUFFIX,
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
Toolchain,
)
from esphome.core import CORE, config
from esphome.core.config import (
@@ -1161,3 +1164,123 @@ def test_make_app_name_cpp_special_chars_escaped() -> None:
cpp_expr, _, _ = make_app_name_cpp('my "device"', "buf", "-", add_mac_suffix=False)
# cpp_string_escape uses octal escapes for quotes
assert '"' not in cpp_expr[1:-1] # no unescaped quotes inside the outer quotes
@pytest.mark.parametrize(
("lib", "name", "version", "repository"),
[
("ArduinoJson", "ArduinoJson", None, None),
("bblanchon/ArduinoJson@7.4.2", "bblanchon/ArduinoJson", "7.4.2", None),
(
"noise-c=https://github.com/esphome/noise-c.git",
"noise-c",
None,
"https://github.com/esphome/noise-c.git",
),
],
)
def test_add_library_str(
lib: str, name: str, version: str | None, repository: str | None
) -> None:
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp32",
KEY_TARGET_FRAMEWORK: "esp-idf",
}
config._add_library_str(lib)
libraries = list(CORE.platformio_libraries.values())
assert len(libraries) == 1
assert libraries[0].name == name
assert libraries[0].version == version
assert libraries[0].repository == repository
@pytest.mark.asyncio
async def test_add_platformio_options_native_idf(
caplog: pytest.LogCaptureFixture,
) -> None:
"""On the native IDF toolchain, build_flags/lib_deps/lib_ignore are
honored, upload_speed is silent and everything else warns."""
CORE.toolchain = Toolchain.ESP_IDF
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp32",
KEY_TARGET_FRAMEWORK: "esp-idf",
}
await config._add_platformio_options(
{
"build_flags": "-DSINGLE_FLAG", # string and list forms both valid
"lib_deps": ["bblanchon/ArduinoJson@7.4.2"],
"lib_ignore": "libsodium",
"upload_speed": "115200",
"board_build.f_flash": "80000000L",
}
)
assert "-DSINGLE_FLAG" in CORE.build_flags
assert "ArduinoJson" in CORE.platformio_libraries
# lib_ignore is stored (listified) for generate_idf_components to read;
# nothing else lands in platformio_options on the native toolchain.
assert CORE.platformio_options == {"lib_ignore": ["libsodium"]}
assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text
assert "upload_speed" not in caplog.text
# build_flags has a first-class esphome equivalent, so it is deprecated.
# lib_deps/lib_ignore are kept as valid platformio_options (no warning).
assert (
"esphome->platformio_options->build_flags is deprecated; use "
"esphome->build_flags instead" in caplog.text
)
assert "lib_deps is deprecated" not in caplog.text
assert "lib_ignore is deprecated" not in caplog.text
@pytest.mark.asyncio
async def test_add_platformio_options_platformio(
caplog: pytest.LogCaptureFixture,
) -> None:
"""On the PlatformIO toolchain all options pass through to the ini,
with build_flags/lib_ignore listified."""
CORE.toolchain = Toolchain.PLATFORMIO
await config._add_platformio_options(
{
"build_flags": "-DSINGLE_FLAG",
"lib_ignore": "libsodium",
"upload_speed": "115200",
}
)
assert CORE.platformio_options == {
"build_flags": ["-DSINGLE_FLAG"],
"lib_ignore": ["libsodium"],
"upload_speed": "115200",
}
# platformio_options is the correct mechanism on the PlatformIO toolchain,
# so the native-equivalent deprecation must not fire here.
assert "deprecated" not in caplog.text
def test_add_library_str_bare_url_requires_name() -> None:
"""A bare repository URL has no library name; CORE.add_library rejects it."""
with pytest.raises(ValueError, match="must have a name"):
config._add_library_str("https://github.com/esphome/noise-c.git")
@pytest.mark.asyncio
@pytest.mark.filterwarnings("ignore::RuntimeWarning")
async def test_to_code_adds_libraries(yaml_file: Callable[[str], Path]) -> None:
"""esphome->libraries entries are parsed and registered via cg.add_library."""
result = load_config_from_fixture(yaml_file, "libraries.yaml", FIXTURES_DIR)
assert result is not None
with patch("esphome.core.config.cg") as mock_cg:
mock_cg.RawStatement.side_effect = lambda *args, **kwargs: MagicMock()
mock_cg.RawExpression.side_effect = lambda *args, **kwargs: MagicMock()
await config.to_code(result[CONF_ESPHOME])
mock_cg.add_library.assert_any_call("SomeLib", None)
mock_cg.add_library.assert_any_call("bblanchon/ArduinoJson", "7.4.2")
mock_cg.add_library.assert_any_call(
"noise-c", None, "https://github.com/esphome/noise-c.git"
)

8
tests/unit_tests/fixtures/core/config/libraries.yaml Normal file
View File

@@ -0,0 +1,8 @@
esphome:
name: test-libraries
libraries:
- SomeLib
- bblanchon/ArduinoJson@7.4.2
- noise-c=https://github.com/esphome/noise-c.git
host:

18
tests/unit_tests/test_core.py
View File

@@ -915,3 +915,21 @@ class TestEsphomeCore:
mock_enable.assert_called_once_with("Wire")
assert "Wire" in target.platformio_libraries
def test_add_build_unflag__warns_on_native_idf_toolchain(
self, target, caplog: pytest.LogCaptureFixture
) -> None:
"""Build unflags are not consumed by the native IDF build generator,
so adding one on that toolchain warns; PlatformIO stays silent."""
target.toolchain = const.Toolchain.PLATFORMIO
target.add_build_unflag("-fno-rtti")
assert "ignored" not in caplog.text
target.toolchain = const.Toolchain.ESP_IDF
target.add_build_unflag("-fno-exceptions")
assert (
"Build unflag -fno-exceptions is ignored when building with the "
"native ESP-IDF toolchain" in caplog.text
)
# The unflag is still recorded either way.
assert target.build_unflags == {"-fno-rtti", "-fno-exceptions"}

122
tests/unit_tests/test_espidf_component.py
View File

@@ -1,3 +1,4 @@
import hashlib
import json
import os
from pathlib import Path
@@ -515,7 +516,7 @@ def test_generate_idf_components_dedupes_shared_dependency(
"esphome/C": {"name": "C"},
}
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -557,6 +558,62 @@ def test_generate_idf_components_dedupes_shared_dependency(
assert "idf_component_register" in generated
def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
esp32_idf_core: None,
) -> None:
# lib_ignore must drop B at the top level and C when it is discovered as a
# dependency of A during the graph walk -- neither may be resolved,
# downloaded, or wired into a manifest. Matching is by lowercase short name.
manifests = {
"esphome/A": {
"name": "A",
"dependencies": [
{"owner": "esphome", "name": "C", "version": "==1.10021.0"}
],
},
"esphome/B": {"name": "B"},
}
download_salts: list[str] = []
def fake_download(self, force=False, salt=""):
download_salts.append(salt)
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
monkeypatch.setattr(IDFComponent, "download", fake_download)
resolve_calls: list[str] = []
def fake_resolve(owner, pkgname, requirements):
resolve_calls.append(pkgname)
return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz"
monkeypatch.setattr(
esphome.espidf.component, "_resolve_registry_version", fake_resolve
)
# lib_ignore is read from CORE.platformio_options (stored there by
# _add_platformio_options); matched by lowercase short name.
monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["B", "esphome/C"]})
top = generate_idf_components(
[Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)]
)
assert [c.name for c in top] == ["esphome/A"]
# Ignored libraries were never resolved (and therefore never downloaded).
assert resolve_calls == ["A"]
# The ignored dependency is not wired into A's manifest.
assert top[0].dependencies == []
# lib_ignore changes the generated wiring, so the cache path is salted to
# keep this conversion separate from ones with a different lib_ignore.
assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]]
def test_generate_idf_components_handles_dependency_cycle(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -575,7 +632,7 @@ def test_generate_idf_components_handles_dependency_cycle(
},
}
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -632,7 +689,7 @@ def test_generate_idf_components_git_overrides_registry_warns(
"esphome/shared": {"name": "shared"},
}
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -669,7 +726,7 @@ def test_generate_idf_components_missing_manifest_raises(
) -> None:
# A library with neither library.json nor library.properties is invalid;
# fail loudly rather than silently generating build files for it.
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
# no library.json / library.properties written
@@ -711,7 +768,7 @@ def test_generate_idf_components_warns_on_noncanonical_duplicate(
"owner/shared": {"name": "shared"},
}
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "src" / "x.c").write_text("int x;")
@@ -744,7 +801,7 @@ def test_generate_idf_components_incompatible_top_level_raises(
) -> None:
# A top-level library that isn't ESP-IDF/esp32 compatible must fail fast,
# not be silently dropped.
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(
@@ -782,7 +839,7 @@ def test_generate_idf_components_incompatible_dependency_skipped(
"esphome/B": {"name": "B", "platforms": ["espressif8266"]},
}
def fake_download(self, force=False):
def fake_download(self, force=False, salt=""):
self.path = tmp_path / self.get_sanitized_name().replace("/", "__")
(self.path / "src").mkdir(parents=True, exist_ok=True)
(self.path / "library.json").write_text(json.dumps(manifests[self.name]))
@@ -804,3 +861,54 @@ def test_generate_idf_components_incompatible_dependency_skipped(
assert [c.name for c in top] == ["esphome/A"]
# The incompatible dependency was dropped, not wired in.
assert top[0].dependencies == []
def test_url_source_salt_changes_cache_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The salt is mixed into the URL hash so salted conversions get their own
cache tree. Pre-created extraction markers keep this network-free."""
monkeypatch.setattr(CORE, "config_path", tmp_path / "test.yaml")
url = "http://example.com/lib.tar.gz"
base = tmp_path / ".esphome" / "pio_components"
expected = {}
for salt in ("", "abcd1234"):
digest = hashlib.sha256((url + salt).encode()).hexdigest()[:8]
expected[salt] = base / digest / "lib"
expected[salt].mkdir(parents=True)
(expected[salt] / ".esphome_extracted").touch()
source = URLSource(url)
assert source.download("lib") == expected[""]
assert source.download("lib", salt="abcd1234") == expected["abcd1234"]
def test_git_source_salt_scopes_domain(monkeypatch: pytest.MonkeyPatch) -> None:
"""The salt becomes a subdirectory of the git clone domain."""
domains: list[str] = []
def fake_clone_or_update(**kwargs):
domains.append(kwargs["domain"])
return Path("/cloned"), None
monkeypatch.setattr(
esphome.espidf.component.git, "clone_or_update", fake_clone_or_update
)
source = GitSource("https://github.com/esphome/noise-c.git", "v1.0")
source.download("noise-c")
source.download("noise-c", salt="abcd1234")
assert domains == ["pio_components", "pio_components/abcd1234"]
def test_idf_component_download_passes_salt() -> None:
"""IDFComponent.download forwards the sanitized name and salt to the
source and records the returned path."""
source = MagicMock()
source.download.return_value = Path("/converted/owner/name")
c = IDFComponent("owner/name", "1.0", source=source)
c.download(force=True, salt="abcd1234")
source.download.assert_called_once_with("owner/name", force=True, salt="abcd1234")
assert c.path == Path("/converted/owner/name")