[espidf] Derive idedata from the native ESP-IDF compile_commands.json (#16742)

This commit is contained in:
Jonathan Swoboda
2026-05-31 17:44:12 -04:00
committed by GitHub
parent 48844a68ba
commit 6116d10ab1
5 changed files with 495 additions and 0 deletions
+1
View File
@@ -760,6 +760,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
toolchain.create_factory_bin()
toolchain.create_ota_bin()
toolchain.create_elf_copy()
toolchain.get_idedata()
else:
from esphome.platformio import toolchain
+178
View File
@@ -0,0 +1,178 @@
"""Derive idedata from an ESP-IDF native-toolchain ``compile_commands.json``.
PlatformIO exposes a curated ``pio run -t idedata`` JSON; the native ESP-IDF
toolchain has no such command, but its CMake build emits
``build/compile_commands.json`` (CMAKE_EXPORT_COMPILE_COMMANDS). This module
turns that file into the same fields consumers (IDE integration, clang-tidy)
expect:
{cxx_path, cxx_flags, defines, includes: {build, toolchain}}
"""
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
import shlex
import subprocess
_LOGGER = logging.getLogger(__name__)
# C++ translation-unit suffixes used to identify ESPHome source files.
_CXX_SUFFIXES = (".cpp", ".cc")
# Suffixes of input/output files that appear bare on the command line (and so
# must not be mistaken for compiler flags).
_INPUT_FILE_SUFFIXES = (*_CXX_SUFFIXES, ".c", ".o", ".S", ".s")
# Path marker identifying an ESPHome source translation unit.
_ESPHOME_SRC_MARKER = "/src/esphome/"
def _expand_response_files(tokens: list[str], directory: Path) -> list[str]:
"""Inline any ``@response-file`` arguments (paths relative to ``directory``).
GCC response files embed flags that must be expanded so GCC-only flags
inside them (e.g. ``-mlongcalls``) can be filtered downstream; left as
``@file`` clang would read them and choke.
"""
out: list[str] = []
for tok in tokens:
if tok.startswith("@"):
rf = Path(tok[1:])
if not rf.is_absolute():
rf = directory / rf
try:
out.extend(
_expand_response_files(
shlex.split(rf.read_text(encoding="utf-8")), directory
)
)
continue
except OSError as err:
# Keep the literal token if the file can't be read, but log it
# so the (otherwise opaque) downstream clang failure is traceable.
_LOGGER.warning("Could not read response file %s: %s", rf, err)
out.append(tok)
return out
def _pick_entry(entries: list[dict]) -> dict:
"""Pick a representative ESPHome C++ translation unit.
All ESPHome sources share the same component flags/defines, so any one of
them yields the cxx_path / cxx_flags / defines we need.
"""
for entry in entries:
f = entry["file"]
if _ESPHOME_SRC_MARKER in f and f.endswith(_CXX_SUFFIXES):
return entry
for entry in entries:
if entry["file"].endswith(_CXX_SUFFIXES):
return entry
raise ValueError("no C++ translation unit found in compile_commands.json")
def _parse_entry(entry: dict) -> tuple[str, list[str], list[str], list[str]]:
"""Parse one compile_commands entry -> (cxx_path, defines, includes, cxx_flags)."""
directory = Path(entry["directory"])
tokens = _expand_response_files(shlex.split(entry["command"]), directory)
def _include(raw: str) -> str:
# Include paths in compile_commands are interpreted relative to the
# entry's ``directory`` (e.g. build-local ``-Iconfig``); resolve them
# so the cached idedata is usable regardless of the consumer's cwd.
raw = raw.strip()
if raw and not Path(raw).is_absolute():
raw = os.path.normpath(directory / raw)
return raw
cxx_path = tokens[0]
defines: list[str] = []
includes: list[str] = []
cxx_flags: list[str] = []
it = iter(tokens[1:])
for tok in it:
if tok in ("-c", "-o"):
next(it, None) # drop the flag and its argument (input/output)
elif tok.startswith("-D"):
# ``.strip()`` handles tokens like ``-D CONFIGURED=1`` (a single
# quoted arg with a space after -D) that some flags arrive as.
defines.append(tok[2:].strip() if len(tok) > 2 else next(it, "").strip())
elif tok.startswith("-I"):
includes.append(_include(tok[2:] if len(tok) > 2 else next(it, "")))
elif tok == "-isystem":
includes.append(_include(next(it, "")))
elif tok.startswith("-isystem"):
includes.append(_include(tok[len("-isystem") :]))
elif tok in ("-MT", "-MF", "-MQ"):
next(it, None) # dependency-file flag + its argument
elif tok.startswith(("-MD", "-MMD", "-MP", "-MM")):
pass # dependency-generation flags, no argument
elif tok.endswith(_INPUT_FILE_SUFFIXES):
pass # input/output files
else:
cxx_flags.append(tok)
return cxx_path, defines, includes, cxx_flags
def _get_toolchain_includes(cxx_path: str) -> list[str]:
"""Query the compiler for its builtin ``#include <...>`` search dirs."""
result = subprocess.run(
[cxx_path, "-E", "-x", "c++", "-", "-v"],
input="",
text=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
check=False,
close_fds=False,
)
includes: list[str] = []
capture = False
for line in result.stderr.splitlines():
if "#include <...> search starts here:" in line:
capture = True
continue
if "End of search list." in line:
break
if capture:
includes.append(line.strip())
if result.returncode != 0 or not includes:
raise RuntimeError(
f"Could not query builtin include dirs from {cxx_path} "
f"(return code {result.returncode}); stderr:\n{result.stderr.strip()}"
)
return includes
def idedata_from_build(compile_commands: Path) -> dict:
"""Parse compile_commands.json into the idedata fields consumers expect.
A single ESP-IDF compile entry only carries its own component's REQUIRES
include set, but consumers (clang-tidy) analyze ESPHome headers that
transitively pull in other components. So take cxx_path / cxx_flags /
defines from a representative ESPHome TU, but union the include dirs across
all ESPHome TUs to get a project-wide superset (as PlatformIO's idedata
provides).
"""
entries = json.loads(Path(compile_commands).read_text(encoding="utf-8"))
cxx_path, defines, _, cxx_flags = _parse_entry(_pick_entry(entries))
build_includes: dict[str, None] = {}
for entry in entries:
f = entry["file"]
if _ESPHOME_SRC_MARKER not in f or not f.endswith(_CXX_SUFFIXES):
continue
for inc in _parse_entry(entry)[2]:
build_includes.setdefault(inc, None)
return {
"cxx_path": cxx_path,
"cxx_flags": cxx_flags,
"defines": defines,
"includes": {
"build": list(build_includes),
"toolchain": _get_toolchain_includes(cxx_path),
},
}
+28
View File
@@ -443,6 +443,34 @@ def get_addr2line_path() -> Path:
return _get_cmake_tool_path("CMAKE_ADDR2LINE")
def get_idedata() -> dict | None:
"""Derive idedata from the build's compile_commands.json.
The native ESP-IDF toolchain has no ``pio run -t idedata`` equivalent, but
its CMake build emits ``build/compile_commands.json``. Parse that into the
idedata fields IDE integrations and clang-tidy expect, cached alongside the
PlatformIO idedata path. Returns None if the compile DB doesn't exist yet.
"""
from esphome.espidf.idedata import idedata_from_build
compile_commands = CORE.relative_build_path("build", "compile_commands.json")
if not compile_commands.is_file():
_LOGGER.debug("No %s yet; skipping idedata generation", compile_commands)
return None
cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json")
if cache.is_file() and cache.stat().st_mtime >= compile_commands.stat().st_mtime:
try:
return json.loads(cache.read_text(encoding="utf-8"))
except ValueError:
pass
data = idedata_from_build(compile_commands)
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
return data
def create_factory_bin() -> bool:
"""Create factory.bin by merging bootloader, partition table, and app."""
build_dir = CORE.relative_build_path("build")
+196
View File
@@ -0,0 +1,196 @@
"""Tests for esphome.espidf.idedata (compile_commands.json -> idedata)."""
# pylint: disable=protected-access
import json
import os
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from esphome.espidf import idedata
# An absolute, forward-slash (shlex-safe) path prefix valid on the host OS, so
# tests exercise the same is-absolute / normalize behavior as a real compile DB
# (a drive-qualified path on Windows, a leading slash elsewhere).
ABS = "C:/" if os.name == "nt" else "/"
def _entry(directory: str, file: str, command: str) -> dict:
return {"directory": directory, "file": file, "command": command}
def test_parse_entry_extracts_fields() -> None:
"""cxx_path, defines, includes and remaining flags are split apart."""
entry = _entry(
f"{ABS}build",
f"{ABS}build/src/esphome/core/application.cpp",
f"/tools/xtensa-esp32-elf-g++ -DUSE_ESP32 -DESPHOME_LOG_LEVEL=5 "
f"-I{ABS}inc/a -isystem {ABS}sys/b -std=gnu++20 -c app.cpp -o app.cpp.o",
)
cxx_path, defines, includes, cxx_flags = idedata._parse_entry(entry)
assert cxx_path == "/tools/xtensa-esp32-elf-g++"
assert "USE_ESP32" in defines
assert "ESPHOME_LOG_LEVEL=5" in defines
assert f"{ABS}inc/a" in includes
assert f"{ABS}sys/b" in includes
assert "-std=gnu++20" in cxx_flags
# input/output files and their flags are not treated as flags
assert "-c" not in cxx_flags
assert "-o" not in cxx_flags
assert "app.cpp" not in cxx_flags
assert "app.cpp.o" not in cxx_flags
def test_parse_entry_space_separated_args() -> None:
"""``-D X`` / ``-I path`` (separate arg) and ``-isystem<path>`` (joined)."""
entry = _entry(
f"{ABS}build",
f"{ABS}build/src/esphome/x.cpp",
f"g++ -D FOO=1 -I {ABS}inc/sep -isystem{ABS}sys/joined -c x.cpp",
)
_, defines, includes, _ = idedata._parse_entry(entry)
assert "FOO=1" in defines
assert f"{ABS}inc/sep" in includes
assert f"{ABS}sys/joined" in includes
def test_parse_entry_resolves_relative_includes() -> None:
"""Relative includes are resolved against the entry's ``directory``."""
directory = f"{ABS}build/proj"
entry = _entry(
directory,
f"{directory}/src/esphome/x.cpp",
"g++ -Iconfig -I../shared -isystem rel/sys -c x.cpp",
)
_, _, includes, _ = idedata._parse_entry(entry)
def resolved(rel: str) -> str:
return os.path.normpath(Path(directory) / rel)
assert resolved("config") in includes
assert resolved("../shared") in includes # ../ normalized away
assert resolved("rel/sys") in includes
# nothing is left relative
assert all(Path(inc).is_absolute() for inc in includes)
def test_parse_entry_skips_dependency_flags() -> None:
"""Dependency-generation flags (and their args) are dropped."""
entry = _entry(
"/build",
"/build/src/esphome/x.cpp",
"g++ -MD -MT x.cpp.o -MF x.cpp.o.d -c x.cpp -o x.cpp.o",
)
_, _, _, cxx_flags = idedata._parse_entry(entry)
for tok in ("-MD", "-MT", "x.cpp.o", "-MF", "x.cpp.o.d", "-c", "-o", "x.cpp"):
assert tok not in cxx_flags
def test_expand_response_files(tmp_path: Path) -> None:
"""``@file`` arguments are inlined relative to the directory."""
rsp = tmp_path / "flags.rsp"
rsp.write_text("-DFROM_RSP -I/rsp/inc")
tokens = idedata._expand_response_files(
["g++", f"@{rsp.name}", "-c", "x.cpp"], tmp_path
)
assert "-DFROM_RSP" in tokens
assert "-I/rsp/inc" in tokens
assert not any(t.startswith("@") for t in tokens)
def test_expand_response_files_keeps_literal_when_missing(tmp_path: Path) -> None:
"""An unreadable ``@file`` token is kept verbatim rather than dropped."""
tokens = idedata._expand_response_files(["g++", "@nope.rsp"], tmp_path)
assert "@nope.rsp" in tokens
def test_pick_entry_prefers_esphome_tu() -> None:
"""A ``/src/esphome/`` C++ TU is picked over other compile entries."""
entries = [
_entry("/b", "/b/managed_components/foo/foo.c", "gcc -c foo.c"),
_entry("/b", "/b/src/esphome/core/app.cpp", "g++ -c app.cpp"),
]
assert idedata._pick_entry(entries)["file"].endswith("app.cpp")
def test_idedata_from_build(tmp_path: Path) -> None:
"""Full transform: representative entry + include union + toolchain dirs."""
compile_commands = tmp_path / "compile_commands.json"
entries = [
_entry(
f"{ABS}b",
f"{ABS}b/src/esphome/core/app.cpp",
f"g++ -DUSE_ESP32 -I{ABS}inc/core -std=gnu++20 -c app.cpp -o app.cpp.o",
),
_entry(
f"{ABS}b",
f"{ABS}b/src/esphome/sensor/s.cpp",
f"g++ -DUSE_ESP32 -I{ABS}inc/sensor -c s.cpp -o s.cpp.o",
),
# non-esphome TU: its includes must not leak into the union
_entry(
f"{ABS}b",
f"{ABS}b/managed_components/x/x.c",
f"gcc -I{ABS}inc/managed -c x.c",
),
]
compile_commands.write_text(json.dumps(entries))
fake_proc = MagicMock(
returncode=0,
stderr=(
"ignored\n"
"#include <...> search starts here:\n"
" /tc/inc/c++\n"
" /tc/inc\n"
"End of search list.\n"
"more ignored\n"
),
)
with patch.object(idedata.subprocess, "run", return_value=fake_proc):
data = idedata.idedata_from_build(compile_commands)
assert data["cxx_path"] == "g++"
assert "USE_ESP32" in data["defines"]
assert "-std=gnu++20" in data["cxx_flags"]
# include dirs unioned across all esphome TUs
assert f"{ABS}inc/core" in data["includes"]["build"]
assert f"{ABS}inc/sensor" in data["includes"]["build"]
# the non-esphome TU is excluded from the union
assert f"{ABS}inc/managed" not in data["includes"]["build"]
# toolchain search dirs parsed from the compiler's -v output
assert data["includes"]["toolchain"] == ["/tc/inc/c++", "/tc/inc"]
def test_get_toolchain_includes_raises_on_probe_failure() -> None:
"""A failed compiler probe is a hard error, not a silent empty list."""
fake_proc = MagicMock(returncode=1, stderr="xtensa-esp32-elf-g++: not found")
with (
patch.object(idedata.subprocess, "run", return_value=fake_proc),
pytest.raises(RuntimeError, match="builtin include dirs"),
):
idedata._get_toolchain_includes("/bad/compiler")
def test_get_toolchain_includes_raises_when_no_dirs_found() -> None:
"""Markers present but no dirs (anomalous output) also raises."""
fake_proc = MagicMock(
returncode=0,
stderr="#include <...> search starts here:\nEnd of search list.\n",
)
with (
patch.object(idedata.subprocess, "run", return_value=fake_proc),
pytest.raises(RuntimeError, match="builtin include dirs"),
):
idedata._get_toolchain_includes("/some/compiler")
+92
View File
@@ -2,6 +2,9 @@
# pylint: disable=protected-access
import json
import os
from pathlib import Path
from unittest.mock import patch
from esphome.const import CONF_FRAMEWORK, CONF_SOURCE
@@ -56,3 +59,92 @@ def test_get_esphome_esp_idf_paths_no_override():
) as mock_install:
toolchain._get_esphome_esp_idf_paths("5.5.4")
mock_install.assert_called_once_with("5.5.4", source_url=None)
def _setup_build(setup_core: Path) -> tuple[Path, Path]:
"""Point CORE at a build dir; return (compile_commands, idedata cache) paths."""
CORE.name = "test"
CORE.build_path = setup_core / "build" / "test"
compile_commands = CORE.relative_build_path("build", "compile_commands.json")
cache = CORE.relative_internal_path("idedata", "test.json")
return compile_commands, cache
def test_get_idedata_returns_none_without_compile_commands(setup_core: Path) -> None:
"""No compile DB yet -> None (rather than an error)."""
_setup_build(setup_core)
assert toolchain.get_idedata() is None
def test_get_idedata_generates_and_caches(setup_core: Path) -> None:
"""Generates from the compile DB and writes the cache."""
compile_commands, cache = _setup_build(setup_core)
compile_commands.parent.mkdir(parents=True, exist_ok=True)
compile_commands.write_text("[]")
with patch(
"esphome.espidf.idedata.idedata_from_build",
return_value={"cxx_path": "g++"},
) as mock_transform:
result = toolchain.get_idedata()
mock_transform.assert_called_once()
assert result == {"cxx_path": "g++"}
assert json.loads(cache.read_text()) == {"cxx_path": "g++"}
def test_get_idedata_uses_cache_when_valid(setup_core: Path) -> None:
"""A cache at least as new as the compile DB is reused without regenerating."""
compile_commands, cache = _setup_build(setup_core)
compile_commands.parent.mkdir(parents=True, exist_ok=True)
compile_commands.write_text("[]")
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text('{"cxx_path": "cached"}')
cc_mtime = compile_commands.stat().st_mtime
os.utime(cache, (cc_mtime + 1, cc_mtime + 1))
with patch("esphome.espidf.idedata.idedata_from_build") as mock_transform:
result = toolchain.get_idedata()
mock_transform.assert_not_called()
assert result == {"cxx_path": "cached"}
def test_get_idedata_regenerates_when_compile_commands_newer(setup_core: Path) -> None:
"""A compile DB newer than the cache forces regeneration."""
compile_commands, cache = _setup_build(setup_core)
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text('{"cxx_path": "stale"}')
compile_commands.parent.mkdir(parents=True, exist_ok=True)
compile_commands.write_text("[]")
cache_mtime = cache.stat().st_mtime
os.utime(compile_commands, (cache_mtime + 1, cache_mtime + 1))
with patch(
"esphome.espidf.idedata.idedata_from_build",
return_value={"cxx_path": "fresh"},
) as mock_transform:
result = toolchain.get_idedata()
mock_transform.assert_called_once()
assert result == {"cxx_path": "fresh"}
def test_get_idedata_regenerates_on_corrupted_cache(setup_core: Path) -> None:
"""An unparseable (but newer) cache falls back to regeneration."""
compile_commands, cache = _setup_build(setup_core)
compile_commands.parent.mkdir(parents=True, exist_ok=True)
compile_commands.write_text("[]")
cache.parent.mkdir(parents=True, exist_ok=True)
cache.write_text("{not json")
cc_mtime = compile_commands.stat().st_mtime
os.utime(cache, (cc_mtime + 1, cc_mtime + 1))
with patch(
"esphome.espidf.idedata.idedata_from_build",
return_value={"cxx_path": "regen"},
) as mock_transform:
result = toolchain.get_idedata()
mock_transform.assert_called_once()
assert result == {"cxx_path": "regen"}