github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-07 03:16:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

[espidf] Cache the discovered component list to skip the discovery configure (#18752)

Browse Source 
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
This commit is contained in:
J. Nick Koston
2026-08-25 12:36:57 -05:00
committed by GitHub
co-authored by pre-commit-ci-lite[bot]
parent 0b78800e3d
commit 19b434f99c
4 changed files with 473 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
esphome/build_gen/espidf.py
+38 -33
View File
@@ -1,6 +1,7 @@
"""ESP-IDF direct build generator for ESPHome."""
import json
import logging
from pathlib import Path
from esphome.components.esp32 import (
@@ -11,6 +12,7 @@ from esphome.components.esp32 import (
)
import esphome.config_validation as cv
from esphome.core import CORE
from esphome.espidf import variant_to_idf_target
from esphome.framework_helpers import (
get_project_compile_flags,
get_project_cxx_compile_flags,
@@ -18,6 +20,8 @@ from esphome.framework_helpers import (
)
from esphome.helpers import mkdir_p, write_file_if_changed
_LOGGER = logging.getLogger(__name__)
# Replaces the IDF default C++ standard (-std=gnu++2b appended to
# CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via
# cg.set_cpp_standard(). Emitted between include(project.cmake) and project(),
@@ -31,11 +35,12 @@ idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")""
def get_available_components() -> list[str] | None:
"""Get list of built-in ESP-IDF components from project_description.json.
"""List the built-in ESP-IDF components from ``project_description.json``.
Excludes ``src``, IDF-managed components (``managed_components/``), and
converted PIO libs (``pio_components/``). Returns ``None`` if the build
dir or ``project_description.json`` isn't ready yet.
Only components below its ``idf_path/components`` count, which leaves out
``src``, IDF-managed components, converted PIO libs and project local
ones such as the Arduino ``component_stubs``. Returns ``None`` if the
build dir or ``project_description.json`` isn't ready yet.
"""
if CORE.build_path is None:
return None
@@ -46,30 +51,24 @@ def get_available_components() -> list[str] | None:
try:
with project_desc.open(encoding="utf-8") as f:
data = json.load(f)
component_info = data.get("build_component_info", {})
result = []
for name, info in component_info.items():
# Exclude our own src component
if name == "src":
continue
# Exclude IDF-managed and converted-PIO components (external).
comp_dir = info.get("dir", "")
if "managed_components" in comp_dir or "pio_components" in comp_dir:
continue
result.append(name)
return result
except (json.JSONDecodeError, OSError):
root = (Path(data["idf_path"]) / "components").resolve()
result = [
name
for name, info in data.get("build_component_info", {}).items()
if (comp_dir := info.get("dir"))
and Path(comp_dir).resolve().is_relative_to(root)
]
except (json.JSONDecodeError, KeyError, OSError) as err:
_LOGGER.debug("Could not read %s: %s", project_desc, err)
return None
if not result:
_LOGGER.warning("No ESP-IDF components found under %s", root)
return result
def has_discovered_components() -> bool:
"""Check if we have discovered components from a previous configure."""
return get_available_components() is not None
"""Check if a previous configure discovered any built-in components."""
return bool(get_available_components())
def _cmake_quote(value: str) -> str:
@@ -79,15 +78,17 @@ def _cmake_quote(value: str) -> str:
return f'"{escaped}"'
def get_project_cmakelists(minimal: bool = False) -> str:
def get_project_cmakelists(
minimal: bool = False, builtin_components: list[str] | None = None
) -> str:
"""Generate the top-level CMakeLists.txt for ESP-IDF project.
When ``minimal`` is true, omit ``ESPHOME_PROJECT_BUILTIN_COMPONENTS``
since ``project_description.json`` may be stale on the first write.
``builtin_components`` supplies the discovered list (from the cache)
instead of reading it from ``project_description.json``.
"""
# Get IDF target from ESP32 variant (e.g., ESP32S3 -> esp32s3)
variant = get_esp32_variant()
idf_target = variant.lower().replace("-", "")
idf_target = variant_to_idf_target(get_esp32_variant())
# esp_idf_size 2.x (bundled with IDF >=6.0) made NG the default and
# removed the --ng flag; on 1.x (IDF 5.5) --ng is required to get
@@ -162,9 +163,11 @@ def get_project_cmakelists(minimal: bool = False) -> str:
else "\n".join(
f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)"
for name in sorted(
set(get_available_components() or []).difference(
CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";")
)
set(
builtin_components
if builtin_components is not None
else get_available_components() or []
).difference(CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";"))
)
)
)
@@ -279,7 +282,9 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC
"""
def write_project(minimal: bool = False) -> None:
def write_project(
minimal: bool = False, builtin_components: list[str] | None = None
) -> None:
"""Write ESP-IDF project files."""
mkdir_p(CORE.build_path)
mkdir_p(CORE.relative_src_path())
@@ -287,7 +292,7 @@ def write_project(minimal: bool = False) -> None:
# Write top-level CMakeLists.txt
write_file_if_changed(
CORE.relative_build_path("CMakeLists.txt"),
get_project_cmakelists(minimal=minimal),
get_project_cmakelists(minimal=minimal, builtin_components=builtin_components),
)
# Write component CMakeLists.txt in src/
esphome/espidf/toolchain.py
+109 -20
View File
@@ -1,6 +1,7 @@
"""ESP-IDF direct build API for ESPHome."""
from dataclasses import dataclass, field
import hashlib
import json
import logging
import os
@@ -23,7 +24,7 @@ from esphome.core import CORE, EsphomeError
from esphome.espidf import variant_to_idf_target
from esphome.espidf.framework import check_esp_idf_install, get_framework_env
from esphome.espidf.size_summary import print_summary
from esphome.helpers import add_git_ceiling_directory
from esphome.helpers import add_git_ceiling_directory, write_file
_LOGGER = logging.getLogger(__name__)
@@ -256,6 +257,106 @@ def run_reconfigure() -> int:
return run_idf_py(*_get_sdkconfig_args(), "reconfigure")
def _builtin_component_cache_path() -> Path | None:
"""Cache file for this build's built-in component list.
The file lives inside the extracted framework directory so it is
discarded together with that exact checkout (re-extract, source
override, clean-all); the target and the EXCLUDE_COMPONENTS set name it.
The sdkconfig is not part of the key: IDF components register regardless
of CONFIG_* options and only gate their sources on them. A checkout
supplied through IDF_PATH is not managed by ESPHome and is never cached.
"""
if "IDF_PATH" in os.environ:
return None
target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "")
excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12]
return (
_get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json"
)
def load_cached_builtin_components() -> list[str] | None:
"""Return the cached built-in component list for this build, if valid.
Every name must still exist under ``$IDF_PATH/components`` so a stale
entry is treated as a miss instead of failing the configure.
"""
if (path := _builtin_component_cache_path()) is None:
return None
try:
components = json.loads(path.read_text(encoding="utf-8"))
present = {
entry.name
for entry in (path.parents[1] / "components").iterdir()
if entry.is_dir()
}
except (OSError, ValueError):
return None
if (
isinstance(components, list)
and all(isinstance(c, str) for c in components)
and present.issuperset(components)
):
return components
return None
def save_cached_builtin_components(components: list[str]) -> None:
"""Store a built-in component list that just configured successfully."""
if not components or (path := _builtin_component_cache_path()) is None:
return
try:
write_file(path, json.dumps(components, separators=(",", ":")))
except EsphomeError as err:
_LOGGER.warning("Could not write component list cache %s: %s", path, err)
def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int:
"""Write the full CMakeLists.txt and run the configure for it."""
from esphome.build_gen.espidf import write_project
_LOGGER.info("Writing CMakeLists.txt with the built-in component list...")
write_project(minimal=False, builtin_components=builtin_components)
# Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt
# is strictly newer than build.ninja, which fails on coarse-mtime
# filesystems (#18682). Also keeps idf.py from regenerating memory.ld
# in testing mode.
return run_reconfigure()
def _configure_project() -> int:
"""Configure the project, discovering the built-in components if needed.
A cached component list skips the discovery configure. If the configure
with a cached list fails the entry is dropped and discovery runs once; a
list is only cached after it configured successfully.
"""
from esphome.build_gen.espidf import get_available_components, write_project
if (cached := load_cached_builtin_components()) is not None:
_LOGGER.info("Using cached ESP-IDF component list")
if _write_project_and_reconfigure(cached) == 0:
return 0
_LOGGER.warning("Cached component list failed; rediscovering")
_builtin_component_cache_path().unlink(missing_ok=True)
_LOGGER.info("Discovering available ESP-IDF components...")
write_project(minimal=True)
if (rc := run_reconfigure()) != 0:
_LOGGER.error("Component discovery failed")
return rc
discovered = get_available_components()
if not discovered:
_LOGGER.error("Component discovery found no built-in ESP-IDF components")
return 1
if (rc := _write_project_and_reconfigure(discovered)) != 0:
_LOGGER.error("Reconfigure with discovered components failed")
return rc
save_cached_builtin_components(discovered)
return 0
def has_outdated_files():
"""Check if the build configuration is stale.
@@ -382,29 +483,17 @@ def run_compile(config, verbose: bool) -> int:
"""Compile the ESP-IDF project.
Uses two-phase configure to auto-discover available components:
1. If no previous build, configure with minimal REQUIRES to discover components
1. If no previous build, configure with minimal REQUIRES to discover
components (skipped when a cached list for this IDF/target/exclusion
set exists)
2. Regenerate CMakeLists.txt with discovered components
3. Run full build
"""
from esphome.build_gen.espidf import write_project
# Check if we need to do discovery phase
if need_reconfigure():
_LOGGER.info("Discovering available ESP-IDF components...")
write_project(minimal=True)
rc = run_reconfigure()
if rc != 0:
_LOGGER.error("Component discovery failed")
return rc
_LOGGER.info("Regenerating CMakeLists.txt with discovered components...")
write_project(minimal=False)
# Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt
# is strictly newer than build.ninja, which fails on coarse-mtime
# filesystems (#18682). Also keeps idf.py from regenerating memory.ld
# in testing mode.
rc = run_reconfigure()
if rc != 0:
_LOGGER.error("Reconfigure with discovered components failed")
if not need_reconfigure():
_LOGGER.info("Build configuration is up to date")
else:
if (rc := _configure_project()) != 0:
return rc
# cmake does not rewrite CMakeCache.txt when only properties change,
# so restamp it or every build repeats discovery. Only after success,