github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-01 00:40:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

[host] Build natively with ninja instead of PlatformIO (#19214)

Browse Source 
Co-authored-by: J. Nick Koston <nick@home-assistant.io>
This commit is contained in:
Jesse Hills
2026-09-28 11:29:59 +02:00
committed by GitHub
co-authored by J. Nick Koston
parent bb7af14012
commit 32e11c4f4a
47 changed files with 2532 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+15 -21
View File
@@ -400,14 +400,9 @@ jobs:
matrix:
bucket: ${{ fromJson(needs.determine-jobs.outputs.integration-test-buckets) }}
env:
# What the cache steps persist; libdeps is excluded (keyed per xdist
# worker and env, it never crosses runs).
INTEGRATION_PIO_CACHE_PATH: |
~/.esphome-integration-tests/platformio/platforms
~/.esphome-integration-tests/platformio/packages
~/.esphome-integration-tests/platformio/appstate.json
~/.esphome-integration-tests/platformio/.cache
~/.esphome-integration-tests/platformio/.esphome.pio.stamp.json
# Registry libraries (noise-c, libsodium, ArduinoJson, lvgl) the host builds
# download, shared per xdist worker by tests/integration/conftest.py
INTEGRATION_LIBRARY_CACHE_PATH: ~/.esphome-integration-tests/pio_components
steps:
- name: Check out code from GitHub
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -427,14 +422,13 @@ jobs:
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.13"
- name: Restore integration PlatformIO cache
# Native platform + toolchain installed by shared_platformio_cache in
# tests/integration/conftest.py; a miss self-heals, so no restore-keys.
id: pio-cache
- name: Restore integration library cache
# A miss or a changed pin self-heals with a download, so no restore-keys
id: library-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.INTEGRATION_PIO_CACHE_PATH }}
key: integration-pio-v1-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements.txt', 'tests/integration/fixtures/cache_init.yaml', 'esphome/components/host/__init__.py') }}
path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
key: integration-libraries-v1-${{ runner.os }}-${{ hashFiles('esphome/components/json/__init__.py', 'esphome/components/noise/__init__.py', 'esphome/components/lvgl/__init__.py', 'esphome/components/improv_base/__init__.py') }}
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -502,16 +496,16 @@ jobs:
# A full cron period of margin for the weekly refresh
retention-days: 14
- name: Print ccache statistics
# esphome stores the PlatformIO ccache under the machine-global cache
# dir (see _ccache_env() in esphome/platformio/toolchain.py).
run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s
- name: Save integration PlatformIO cache
# esphome stores the host build's ccache under the machine-global
# cache dir (see get_build_env() in esphome/host/toolchain.py).
run: CCACHE_DIR="$HOME/.cache/esphome/host/ccache" ccache -s
- name: Save integration library cache
# Bucket 0 only; the others would race the same immutable key.
if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.pio-cache.outputs.cache-hit != 'true'
if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.library-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.INTEGRATION_PIO_CACHE_PATH }}
key: ${{ steps.pio-cache.outputs.cache-primary-key }}
path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
key: ${{ steps.library-cache.outputs.cache-primary-key }}
import-time:
name: Check import esphome.__main__ time
esphome/__main__.py
+7 -16
View File
@@ -1714,26 +1714,12 @@ def command_compile(args: ArgsProtocol, config: ConfigType) -> int | None:
if exit_code != 0:
return exit_code
if CORE.is_host:
_LOGGER.info(
"Successfully compiled program to path '%s'", _host_program_path(config)
)
_LOGGER.info("Successfully compiled program to path '%s'", CORE.firmware_bin)
else:
_LOGGER.info("Successfully compiled program.")
return 0
def _host_program_path(config: ConfigType) -> str:
"""Return the compiled host ELF path."""
if CORE.using_toolchain_esp_idf:
from esphome.espidf import toolchain
return str(toolchain.get_elf_path())
from esphome.platformio.toolchain import get_idedata
# Memoized by compile_program's own call; this is a dict lookup
return str(get_idedata(config).firmware_elf_path)
def command_upload(args: ArgsProtocol, config: ConfigType) -> int | None:
# Get devices, resolving special identifiers like OTA
devices = choose_upload_log_host(
@@ -1778,7 +1764,7 @@ def command_run(args: ArgsProtocol, config: ConfigType) -> int | None:
return exit_code
_LOGGER.info("Successfully compiled program.")
if CORE.is_host:
program_path = _host_program_path(config)
program_path = str(CORE.firmware_bin)
_LOGGER.info("Running program from path '%s'", program_path)
return run_external_process(program_path)
@@ -2024,6 +2010,11 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int:
CORE.target_platform,
)
return 1
if (
check_supported := getattr(analysis_toolchain, "check_analysis_supported", None)
) is not None:
# Raises with the reason; before the compile, not after it
check_supported()
# Always compile to ensure fresh data (fast if no changes - just relinks)
exit_code = write_cpp(config)
esphome/arduino/library.py
+33 -15
View File
@@ -2,7 +2,9 @@
Bundled names build straight from the framework tree; everything else goes
through ``esphome.platformio.library``. Mirrors ``lib_ldf_mode=off``: each
library builds its own archive; all include dirs join one global path.
library builds its own archive; all include dirs join one global path. The
host build reuses it without a framework tree: nothing is bundled there and
every name resolves from the registry.
Deviations from PlatformIO: flat-layout libraries get the recursive default
source filter; ``dot_a_linkage`` is honored; bundled libraries never run a
@@ -342,13 +344,24 @@ def _check_unfulfilled_provides(
def resolve_libraries(
framework_path: Path, *, pio_platform: str, board_mcu: str, cache_key: str
framework_path: Path | None,
*,
pio_platform: str,
board_mcu: str,
cache_key: str,
framework: str | None = "arduino",
manifest_optional: bool = False,
) -> list[ArduinoLibrary]:
"""Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`.
``pio_platform``/``board_mcu`` filter manifests the way PlatformIO would
for that core (e.g. ``espressif8266``/``esp8266``); ``cache_key`` keys the
shared converter's download cache.
shared converter's download cache. ``framework`` is the manifest
framework token the compatibility check warns about; None skips it.
A None ``framework_path`` means no core-bundled libraries exist (the
host build): every name resolves from the registry.
``manifest_optional`` accepts libraries without a manifest, built with
PlatformIO's default layout.
The returned list is not topologically sorted, so the caller must link
the archives inside one ``--start-group``/``--end-group`` pair (the
@@ -359,18 +372,22 @@ def resolve_libraries(
# PlatformIO's lib_ignore covers framework-bundled libraries too; the
# shared converter only filters the registry/git ones.
lib_ignore = lib_ignore_set()
# Exact directory names keep membership case-sensitive everywhere
# (an is_dir() probe would match "wire" on macOS/Windows and build
# the bundled Wire twice)
libraries_dir = framework_path / "libraries"
if not libraries_dir.is_dir():
# A registry fallback would fail later with a misleading
# package-not-found error per bundled name
raise EsphomeError(
f"{libraries_dir} is missing; the framework install may be "
"incomplete (run 'esphome clean-all')"
bundled_dir_names: frozenset[str] = frozenset()
if framework_path is not None:
# Exact directory names keep membership case-sensitive everywhere
# (an is_dir() probe would match "wire" on macOS/Windows and build
# the bundled Wire twice)
libraries_dir = framework_path / "libraries"
if not libraries_dir.is_dir():
# A registry fallback would fail later with a misleading
# package-not-found error per bundled name
raise EsphomeError(
f"{libraries_dir} is missing; the framework install may be "
"incomplete (run 'esphome clean-all')"
)
bundled_dir_names = frozenset(
p.name for p in libraries_dir.iterdir() if p.is_dir()
)
bundled_dir_names = frozenset(p.name for p in libraries_dir.iterdir() if p.is_dir())
def _provided(name: object) -> bool:
return _is_safe_library_name(name) and name in bundled_dir_names
@@ -497,12 +514,13 @@ def resolve_libraries(
backend = LibraryBackend(
platform=pio_platform,
framework="arduino",
framework=framework,
emit=_emit,
cache_key=cache_key,
# The walk must not resolve bundled names from the registry;
# _add_bundled_dependencies adds them after emit
provides=_provided,
manifest_optional=manifest_optional,
)
if external:
convert_libraries(external, backend)
esphome/arduino8266/framework.py
+2 -14
View File
@@ -17,7 +17,7 @@ import os
from pathlib import Path
from typing import NamedTuple
from esphome.build_helpers.ccache import ccache_defaults_env
from esphome.build_helpers.ccache import ccache_env
from esphome.build_helpers.ninja import find_ninja
from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path
from esphome.core import EsphomeError, Version
@@ -143,17 +143,5 @@ def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]:
*filter(None, env.get("PATH", "").split(os.pathsep)),
]
env["PATH"] = os.pathsep.join(parts)
env.update(ccache_env(ccache))
env.update(ccache_env(ccache, ARDUINO8266_TOOLS_CACHE))
return env
def ccache_env(ccache: str | None) -> dict[str, str]:
"""Return ccache settings for the build subprocess (not os.environ).
``ccache`` is the pre-resolved binary (resolve_ccache_path), or None
when disabled. Values the user already set in the environment are
respected.
"""
if ccache is None:
return {}
return ccache_defaults_env(get_arduino8266_tools_path() / "ccache")
esphome/arduino8266/toolchain.py
+10 -64
View File
@@ -2,21 +2,21 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
import subprocess
from typing import TYPE_CHECKING
from esphome.build_helpers.ccache import resolve_ccache_path
from esphome.build_helpers.ccache import resolve_absolute_ccache_path
from esphome.build_helpers.native import warn_ignored_platformio_options
from esphome.build_helpers.ninja import refresh_compile_commands
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
CONF_ESPHOME,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
)
from esphome.core import CORE, EsphomeError
from esphome.helpers import write_file
from esphome.core import CORE
from esphome.types import ConfigType
if TYPE_CHECKING:
@@ -28,20 +28,6 @@ _LOGGER = logging.getLogger(__name__)
_MAX_RAM_SIZE = 81920
def _warn_ignored_platformio_options() -> None:
"""Warn for component-added platformio options the native build drops."""
from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS
consumed = NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS
for key in sorted(CORE.platformio_options or {}):
if key not in consumed:
_LOGGER.warning(
"platformio_options->%s is ignored when building with the "
"native 'arduino' toolchain",
key,
)
_RAM_SECTIONS = (".data", ".rodata", ".bss")
_FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata")
@@ -84,21 +70,17 @@ def get_readelf_path() -> Path:
def run_compile(config: ConfigType, verbose: bool) -> int:
from esphome.arduino8266 import framework
from esphome.build_gen import arduino8266 as build_gen
from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS
_warn_ignored_platformio_options()
warn_ignored_platformio_options(NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS)
paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION])
# Resolved once: the probe is not free and three consumers need it
ccache = resolve_ccache_path()
build_gen.write_project(paths, ccache)
ccache = resolve_absolute_ccache_path()
ninja_changed = build_gen.write_project(paths, ccache)
build_dir = get_build_dir()
env = framework.get_build_env(paths.toolchain, ccache)
# The DB is a pure function of build.ninja; regenerate when it is older
compdb = build_dir / "compile_commands.json"
ninja_file = build_dir / "build.ninja"
if not compdb.is_file() or compdb.stat().st_mtime < ninja_file.stat().st_mtime:
_write_compile_commands(paths.ninja, build_dir, env)
refresh_compile_commands(paths.ninja, build_dir, env, ninja_changed)
cmd = [str(paths.ninja)]
if verbose:
@@ -140,42 +122,6 @@ def run_compile(config: ConfigType, verbose: bool) -> int:
return 0
def _write_compile_commands(
ninja_path: Path, build_dir: Path, env: dict[str, str]
) -> None:
compdb = build_dir / "compile_commands.json"
result = subprocess.run(
[str(ninja_path), "-C", str(build_dir), "-t", "compdb", "c", "cxx", "asm"],
env=env,
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if result.returncode != 0:
# Drop any stale database so consumers (IDE integration, clang-tidy,
# the memory analyzer) can't silently read outdated data.
compdb.unlink(missing_ok=True)
raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}")
try:
entries = json.loads(result.stdout)
except ValueError as err:
compdb.unlink(missing_ok=True)
raise EsphomeError(
f"ninja produced an unparsable compile database: {err} "
f"(output starts {result.stdout[:120]!r})"
) from err
if not entries:
# compdb exits 0 with [] for unknown rule names; a renamed compile
# rule must fail the build, not silently strand every consumer
compdb.unlink(missing_ok=True)
raise EsphomeError(
"ninja produced an empty compile database; the generator's rule "
"names no longer match"
)
write_file(compdb, result.stdout)
def _parse_app_size(build_dir: Path, paths: InstalledPaths) -> int | None:
"""Read the app flash budget (irom0_0_seg length) from the linker script."""
from esphome.build_gen.arduino8266 import get_flash_ld_path
@@ -253,7 +199,7 @@ def get_idedata(ccache: str | None = None) -> dict | None:
# A disabled ccache resolves to None without spawning anything, so
# re-resolving here costs nothing when the caller has no answer
launcher = ccache or resolve_ccache_path()
launcher = ccache or resolve_absolute_ccache_path()
return load_or_build_idedata(
get_build_dir() / "compile_commands.json",
get_elf_path(),
esphome/build_gen/arduino8266.py
+25 -117
View File
@@ -14,7 +14,6 @@ from the build flags with the same precedence as the PlatformIO builder.
from __future__ import annotations
from collections.abc import Collection
from dataclasses import dataclass
import hashlib
import logging
@@ -23,7 +22,6 @@ from pathlib import Path
import re
import shlex
import subprocess
import sys
from typing import TYPE_CHECKING, NamedTuple
from esphome.arduino8266.framework import toolchain_tool
@@ -32,6 +30,14 @@ from esphome.build_helpers.ninja import (
quote_path as _q,
shell_token as _shell_token,
)
from esphome.build_helpers.ninja_gen import (
ar_rule_lines,
collect_sources,
compile_edges,
compile_rule_lines,
library_edges,
tool_lines,
)
from esphome.components.esp8266 import build_surgery
from esphome.components.esp8266.boards import (
BOARDS,
@@ -46,12 +52,9 @@ from esphome.components.esp8266.const import (
KEY_SCANF_FLOAT,
)
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import (
get_project_cxx_compile_flags,
strip_win_long_path_prefix,
)
from esphome.framework_helpers import get_project_cxx_compile_flags
from esphome.helpers import mkdir_p, write_file_if_changed
from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX, lex_build_flags
from esphome.platformio.library import lex_build_flags
if TYPE_CHECKING:
from esphome.arduino8266.framework import InstalledPaths
@@ -643,14 +646,6 @@ _PLAIN_LINKER_PREFIXES = ("-T", "-Xlinker")
_PLAIN_DRIVER_LINK_PREFIXES = ("-fuse-ld=", "--specs=", "-specs=")
def _collect_sources(root: Path, exclude: Collection[str] = frozenset()) -> list[Path]:
return sorted(
p
for p in root.rglob("*")
if p.suffix in SOURCE_KIND_FOR_SUFFIX and p.name not in exclude
)
def _stat_sig(path: Path) -> str:
"""Size and mtime cache-stamp signature for one input file.
@@ -868,34 +863,6 @@ def _generate_testing_flash_ld(
)
def _ninja_compile_edges(
lines: list[str],
sources: list[Path],
root: Path,
group: str,
flags: str = "",
) -> list[str]:
"""Emit compile edges for ``sources``; return the object paths."""
objects = []
for src in sources:
rel = src.relative_to(root).as_posix()
obj = f"obj/{group}/{rel}.o"
escaped_obj = _e(obj)
lines.append(
f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}"
)
if flags:
lines.append(f" flags = {flags}")
# Escaped once here: the returned paths only ever appear in build
# statements (archive/link inputs), which use ninja escaping.
objects.append(escaped_obj)
return objects
def _common_parent(paths: list[Path]) -> Path:
return Path(os.path.commonpath([str(p.parent) for p in paths]))
class _FlagSets(NamedTuple):
cflags: list[str]
cxxflags: list[str]
@@ -1082,48 +1049,14 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
+ ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"]
)
build_tool = Path(__file__).parent / "build_tool.py"
# $in/$out stay unquoted: ninja escapes its built-in path variables
# itself; only literal paths need _q().
lines = [
"# Auto-generated by ESPHome",
"ninja_required_version = 1.5",
f"cc = {_q(toolchain_tool(paths.toolchain, 'gcc'))}",
f"cxx = {_q(toolchain_tool(paths.toolchain, 'g++'))}",
# The NSIS launcher starts Python with a \\?\ extended-length path
# that cmd.exe cannot spawn; same strip every other emitted binary
# path gets
f"python = {_q(strip_win_long_path_prefix(sys.executable))}",
f"buildtool = {_q(build_tool)}",
f"ccache = {_q(ccache) if ccache else ''}",
"",
# Rule names match SOURCE_KIND_FOR_SUFFIX values (c, cxx, asm, aspp)
"rule c",
" command = $ccache $cc -MMD -MF $out.d $cflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CC $out",
"rule cxx",
" command = $ccache $cxx -MMD -MF $out.d $cxxflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CXX $out",
"rule aspp",
" command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $asflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = AS $out",
# Plain assembler, as SCons's ASCOM: no preprocessor, so no
# depfile and no $flags (defines/includes) either
"rule asm",
" command = $ccache $cc -x assembler $asflags -c $in -o $out",
" description = AS $out",
"rule ar",
f" command = $python $buildtool ar {_q(toolchain_tool(paths.toolchain, 'ar'))} $out $out.rsp",
" rspfile = $out.rsp",
" rspfile_content = $in_newline",
" description = AR $out",
*tool_lines(
(toolchain_tool(paths.toolchain, "gcc"),),
(toolchain_tool(paths.toolchain, "g++"),),
ccache,
),
*compile_rule_lines(),
*ar_rule_lines(toolchain_tool(paths.toolchain, "ar")),
"rule link",
" command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group",
" rspfile = $out.rsp",
@@ -1153,16 +1086,15 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
core_exclude |= _CORE_EXCLUDE_WAVEFORM
archives = []
direct_objs: list[str] = []
# variant_dir existence was already enforced with the include dirs
variant_sources = _collect_sources(variant_dir)
variant_sources = collect_sources(variant_dir)
if variant_sources:
objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant")
objs = compile_edges(lines, variant_sources, variant_dir, "variant")
lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}")
archives.append("libFrameworkArduinoVariant.a")
core_objs = _ninja_compile_edges(
lines, _collect_sources(core_dir, core_exclude), core_dir, "core"
core_objs = compile_edges(
lines, collect_sources(core_dir, core_exclude), core_dir, "core"
)
if not core_objs:
# An empty archive would link into a wall of undefined references
@@ -1173,32 +1105,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
lines.append(f"build libFrameworkArduino.a: ar {' '.join(core_objs)}")
archives.append("libFrameworkArduino.a")
for lib in libraries:
if not lib.sources:
# Header-only libraries are legitimate; the log makes an empty
# srcFilter or broken tree traceable before link errors do.
_LOGGER.debug(
"Library %s has no source files; contributing includes only",
lib.name,
)
continue
lib_root = _common_parent(lib.sources)
objs = _ninja_compile_edges(
lines,
lib.sources,
lib_root,
f"lib/{lib.name}",
flags=" ".join(_shell_token(f) for f in lib.flags),
)
if not lib.lib_archive:
# libArchive: false / dot_a_linkage=false: hand the objects to
# the linker directly so unreferenced-but-required symbols
# (exception handlers, weak overrides) survive
direct_objs.extend(objs)
continue
archive = f"lib{lib.name}.a"
lines.append(f"build {_e(archive)}: ar {' '.join(objs)}")
archives.append(archive)
lib_archives, direct_objs = library_edges(lines, libraries)
archives += lib_archives
# One source of truth with the PlatformIO path: esp8266/__init__ pins
# build_src_flags (the throw_stubs force-include); -include paths
@@ -1221,8 +1129,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
# One shared variable instead of repeating the flags line on every src
# edge (hundreds of edges in a real project)
lines.append(f"srcflags = {src_extra}")
src_objs = _ninja_compile_edges(
lines, _collect_sources(src_dir), src_dir, "src", flags="$srcflags"
src_objs = compile_edges(
lines, collect_sources(src_dir), src_dir, "src", flags="$srcflags"
)
ld_deps = [f"ld/{_COMMON_LD_NAME}"]
esphome/build_gen/host.py
+272
View File
@@ -0,0 +1,272 @@
"""Native ninja build generator for the host platform.
Emits ``build.ninja`` under ``.pioenvs/<name>/``: every source in the
generated ``src/`` tree plus the resolved registry libraries compiles with
the machine's compiler and links into ``program``, the name PlatformIO's
native platform produced. Build flags route the way SCons's ``ParseFlags``
did under PlatformIO: ``-D``/``-I``/``-std=``/``-W`` shapes reach the
compile lines only, ``-l``/``-L``/``-Wl,`` the link line only, everything
else both.
"""
from __future__ import annotations
from collections.abc import Iterable
import logging
import os
from pathlib import Path
import sys
from typing import TYPE_CHECKING
from esphome.build_helpers.ninja import escape as _e, quote_path as _q, shell_token
from esphome.build_helpers.ninja_gen import (
PATH_ARG_FLAGS,
Flag,
anchor_path_flag,
ar_rule_lines,
collect_sources,
compile_edges,
compile_rule_lines,
library_edges,
tool_lines,
)
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import get_project_cxx_compile_flags
from esphome.helpers import mkdir_p, write_file_if_changed
from esphome.host.toolchain import PROGRAM_NAME, HostCompilers, find_tool, get_build_dir
from esphome.platformio.library import lex_build_flags
if TYPE_CHECKING:
from esphome.arduino.library import ArduinoLibrary
_LOGGER = logging.getLogger(__name__)
# The PlatformIO platform the host built under; registry manifests declare
# compatibility against it, as lib_compat_mode=strict checked before
PIO_PLATFORM = "native"
# Namespaces the shared library download cache (pio_components/host/)
LIBRARY_CACHE_KEY = "host"
# Flag shapes that only the compiler understands; dropped from the link line
_COMPILE_ONLY_PREFIXES = ("-D", "-U", "-I", "-std=", "-W", *PATH_ARG_FLAGS)
# Flag shapes that only the linker consumes; inert on a -c compile line
_LINK_ONLY_PREFIXES = ("-l", "-L", "-Wl,")
# Link-only flags whose argument is the next token
_LINK_ONLY_ARG_FLAGS = ("-framework", "-Xlinker", "-z")
def parse_flags(entries: Iterable[str], owner: str) -> list[Flag]:
"""Lex build flag entries into flags, each with its argument.
Entries are a set, so their order is not the user's: a flag and the
argument it takes as the next token must share one entry.
"""
flags: list[Flag] = []
for entry in entries:
it = iter(lex_build_flags(entry, owner))
for tok in it:
if tok not in PATH_ARG_FLAGS and tok not in _LINK_ONLY_ARG_FLAGS:
flags.append((tok,))
continue
arg = next(it, None)
# A path never starts with "-"; that is the next flag
if arg is None or (tok in PATH_ARG_FLAGS and arg.startswith("-")):
raise EsphomeError(
f"{owner} build flags have '{tok}' with no argument; write "
f"the flag and its argument as one entry"
)
flags.append((tok, arg))
return flags
def split_flags(flags: list[Flag]) -> tuple[list[Flag], list[Flag]]:
"""Route build flags to the compile and link lines."""
compile_flags: list[Flag] = []
link_flags: list[Flag] = []
for flag in flags:
name = flag[0]
if len(flag) > 1:
(compile_flags if name in PATH_ARG_FLAGS else link_flags).append(flag)
elif name.startswith(_LINK_ONLY_PREFIXES):
# Checked before the compile prefixes: -Wl, would match -W
link_flags.append(flag)
elif name.startswith(_COMPILE_ONLY_PREFIXES):
compile_flags.append(flag)
else:
# -g, -O, -f*, -m*, -pthread, --coverage: both lines, as SCons
compile_flags.append(flag)
link_flags.append(flag)
return compile_flags, link_flags
def _is_std(flag: Flag) -> bool:
return flag[0].startswith("-std=")
def _is_cxx_std(flag: Flag) -> bool:
return _is_std(flag) and "++" in flag[0]
def _anchored_flags(entries: Iterable[str], owner: str) -> list[Flag]:
build_path = Path(CORE.build_path)
return [
anchor_path_flag(flag, build_path)
for flag in parse_flags(sorted(entries), owner)
]
def _flag_lists() -> tuple[list[str], list[str], list[str]]:
"""The C, C++, and link flag lists (raw tokens), build_unflags applied.
``cg.set_cpp_standard`` wins over any ``-std=`` in the build flags for
C++ compiles, as PlatformIO's unflag of every other standard did; C
compiles never see a C++ standard.
"""
# The funnel warns and drops empty glued arguments (-D "") itself
compile_flags, link_flags = split_flags(
_anchored_flags(CORE.build_flags, "esphome")
)
cflags = [f for f in compile_flags if not _is_cxx_std(f)]
cxx_std = CORE.cpp_standard
cxxflags = [f for f in compile_flags if not (cxx_std and _is_std(f))]
if cxx_std:
cxxflags.insert(0, (f"-std={cxx_std}",))
cxxflags += [(tok,) for tok in get_project_cxx_compile_flags()]
# A flag is removed whole, with its argument, as PlatformIO did
unflags = set(_anchored_flags(CORE.build_unflags, "esphome build_unflags"))
# An unflag that hits nothing (a typo, or -DUSE_FOO against
# -DUSE_FOO=1) must be visible, since the user believes the flag is
# gone while it still drives the build
if unmatched := sorted(unflags - set(cflags) - set(cxxflags) - set(link_flags)):
_LOGGER.warning(
"build_unflags entries matched no build flag: %s",
", ".join(" ".join(flag) for flag in unmatched),
)
def keep(flags: list[Flag]) -> list[str]:
return [tok for flag in flags if flag not in unflags for tok in flag]
return keep(cflags), keep(cxxflags), keep(link_flags)
def _resolve_host_libraries() -> list[ArduinoLibrary]:
"""Every ``cg.add_library()`` entry, fetched from the registry.
The host has no framework, so nothing is bundled and no framework
compatibility check applies; the platform check keeps the strict
manifest gate PlatformIO's native platform enforced. Manifest-less
libraries (a bare git checkout) build with PlatformIO's default
layout, as they did under its native platform.
"""
if not CORE.platformio_libraries:
return []
from esphome.arduino.library import resolve_libraries
return resolve_libraries(
None,
pio_platform=PIO_PLATFORM,
board_mcu="host",
cache_key=LIBRARY_CACHE_KEY,
framework=None,
manifest_optional=True,
)
def _file_macro_maps(build_dir: Path) -> list[str]:
"""Flags that keep ``__FILE__`` relative to the build path.
PlatformIO compiled ``src/x.cpp`` from the build path, and tools name
things after that spelling (CodSpeed's benchmark ids). Here a source
reaches the compiler by its absolute path, or relative to the build
directory when ccache rewrites it.
"""
build_path = Path(CORE.build_path)
prefixes = (build_path, Path(os.path.relpath(build_path, build_dir)))
return [
shell_token(f"-fmacro-prefix-map={prefix}{os.sep}=", force=True)
for prefix in prefixes
]
def write_project(compilers: HostCompilers, ccache: str | None) -> bool:
"""Write the ninja build for the current configuration.
``ccache`` is the caller's already-resolved binary (None when disabled).
Returns True when ``build.ninja`` changed, so the caller can skip work
derived purely from it (the compile database) on unchanged builds.
"""
build_dir = get_build_dir()
mkdir_p(build_dir)
src_dir = CORE.relative_src_path()
if not src_dir.is_dir():
# Generated project state, not install state: clean-all would not help
raise EsphomeError(f"Generated source directory {src_dir} is missing")
cflags, cxxflags, link_flags = _flag_lists()
libraries = _resolve_host_libraries()
include_dirs = [src_dir]
for lib in libraries:
include_dirs += lib.include_dirs
includes = [f"-I{_q(d)}" for d in include_dirs]
includes += _file_macro_maps(build_dir)
# SCons's link line: $LINKFLAGS $SOURCES $_LIBDIRFLAGS $_LIBFLAGS, so
# -L and -l trail the objects while every other link token leads
lib_dirs = [Path(t[2:]) for t in link_flags if t.startswith("-L")]
libs = [t for t in link_flags if t.startswith("-l")]
linkflags = [shell_token(t) for t in link_flags if not t.startswith(("-L", "-l"))]
for lib in libraries:
lib_dirs += lib.link_dirs
libs += [f"-l{name}" for name in lib.link_libs]
linkflags += [shell_token(f) for f in lib.link_flags]
# PlatformIO's ASPPCOM passes only -D/-I user flags to assembly
asflags = [t for t in cflags if t.startswith(("-D", "-I"))]
lines = [
*tool_lines(compilers.cc, compilers.cxx, ccache),
*compile_rule_lines(),
"rule link",
" command = $cxx -o $out $linkflags @$out.rsp $archives $libdirflags $libflags",
" rspfile = $out.rsp",
" rspfile_content = $in_newline",
" description = LINK $out",
]
if any(lib.sources and lib.lib_archive for lib in libraries):
# Resolved only when an archive is built, so a system without
# binutils still links a library-free configuration
lines += ar_rule_lines(find_tool("AR", ("ar",)))
lines += [
"",
f"cflags = {' '.join([*map(shell_token, cflags), *includes])}",
f"cxxflags = {' '.join([*map(shell_token, cxxflags), *includes])}",
f"asflags = {' '.join([*map(shell_token, asflags), *includes])}",
f"linkflags = {' '.join(linkflags)}",
f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}",
f"libflags = {' '.join(shell_token(lib) for lib in libs)}",
"",
]
archives, direct_objs = library_edges(lines, libraries)
src_objs = compile_edges(lines, collect_sources(src_dir), src_dir, "src")
if not src_objs:
raise EsphomeError(f"No source files found under {src_dir}")
# Archives are not topologically sorted; GNU ld needs the group to
# resolve references between them. ld64 loads archives iteratively and
# rejects the option, so macOS lists them bare.
archive_tokens = [shell_token(a) for a in archives]
if archive_tokens and sys.platform != "darwin":
archive_tokens = ["-Wl,--start-group", *archive_tokens, "-Wl,--end-group"]
lines.append(
f"build {PROGRAM_NAME}: link {' '.join(src_objs + direct_objs)} | "
f"{' '.join(_e(a) for a in archives)}"
)
lines.append(f" archives = {' '.join(archive_tokens)}")
lines.append(f"default {PROGRAM_NAME}")
lines.append("")
return write_file_if_changed(build_dir / "build.ninja", "\n".join(lines))
esphome/build_helpers/ccache.py
+26 -2
View File
@@ -68,6 +68,15 @@ def resolve_ccache_path() -> str | None:
return ccache
def resolve_absolute_ccache_path() -> str | None:
"""``resolve_ccache_path`` for the ninja backends, which run their
commands from the build directory, where a relative path is lost."""
from esphome.build_helpers.ninja import absolute_tool
ccache = resolve_ccache_path()
return absolute_tool(ccache) if ccache else None
def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
"""Default ``CCACHE_*`` values for a build subprocess (not os.environ).
@@ -84,9 +93,24 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
"CORE.build_path must be set before constructing the build environment"
)
defaults = {
"CCACHE_DIR": str(cache_dir),
# ccache expands $VAR in its settings; $$ is a literal $
"CCACHE_DIR": str(cache_dir).replace("$", "$$"),
"CCACHE_NOHASHDIR": "true",
"CCACHE_DEPEND": "1",
"CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
"CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()).replace("$", "$$"),
}
return {k: v for k, v in defaults.items() if k not in os.environ}
def ccache_env(ccache: str | None, tools_cache: tuple[str, str]) -> dict[str, str]:
"""The ccache settings for a build subprocess (not os.environ).
``ccache`` is the pre-resolved binary (resolve_ccache_path), or None when
disabled; ``tools_cache`` is the backend's tools cache spec, which holds
its ccache dir.
"""
if ccache is None:
return {}
from esphome.build_helpers.tools_cache import tools_cache_path
return ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache")
esphome/build_helpers/native.py
+22
View File
@@ -7,16 +7,21 @@ platform component package or the backend itself.
from __future__ import annotations
from collections.abc import Collection
import importlib
import logging
from types import ModuleType
from esphome.const import Toolchain
from esphome.core import CORE, EsphomeError
_LOGGER = logging.getLogger(__name__)
# Native build backend per (target platform, toolchain)
NATIVE_TOOLCHAIN_MODULES = {
("esp32", Toolchain.ESP_IDF): "esphome.espidf.toolchain",
("esp8266", Toolchain.ARDUINO): "esphome.arduino8266.toolchain",
("host", Toolchain.HOST): "esphome.host.toolchain",
}
@@ -50,3 +55,20 @@ def analysis_backend() -> ModuleType | None:
return native
module_path = ANALYSIS_TOOLCHAIN_MODULES.get((CORE.target_platform, CORE.toolchain))
return importlib.import_module(module_path) if module_path else None
def warn_ignored_platformio_options(consumed: Collection[str]) -> None:
"""Warn for component-added platformio options a native build drops.
User-supplied keys were already routed or warned about by
``core/config.py``; what survives into ``CORE.platformio_options`` came
from ``cg.add_platformio_option`` calls in components.
"""
for key in sorted(CORE.platformio_options or {}):
if key not in consumed:
_LOGGER.warning(
"platformio_options->%s is ignored when building with the "
"native '%s' toolchain",
key,
CORE.toolchain.value,
)
esphome/build_helpers/ninja.py
+93 -3
View File
@@ -2,14 +2,17 @@
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
import re
import shutil
import subprocess
from esphome.core import EsphomeError
from esphome.framework_helpers import strip_win_long_path_prefix, tool_version_runs
from esphome.helpers import write_file_if_changed
_LOGGER = logging.getLogger(__name__)
@@ -23,11 +26,26 @@ def _ninja_runs(binary: str) -> bool:
)
# Compile rule names the generators emit; ninja's compdb tool is asked for
# exactly these, so a renamed rule fails the build instead of stranding idedata
COMPILE_RULES = ("c", "cxx", "aspp", "asm")
def absolute_tool(tool: str | Path) -> str:
"""A tool path that still resolves from the build directory.
``shutil.which`` returns a relative path for a relative PATH entry, and
ninja runs the commands from ``.pioenvs/<name>``. Symlinks are kept:
ccache's compiler links depend on the name they are called by.
"""
return strip_win_long_path_prefix(str(Path(tool).absolute()))
def find_ninja() -> Path:
"""Locate the ninja binary: a runnable PATH hit first, else the ninja
PyPI wheel."""
if binary := shutil.which("ninja"):
binary = strip_win_long_path_prefix(binary)
binary = absolute_tool(binary)
if _ninja_runs(binary):
return Path(binary)
import_error: ImportError | None = None
@@ -49,8 +67,18 @@ def find_ninja() -> Path:
def escape(value: Path | str) -> str:
"""Escape a path or token for a ninja file."""
return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ")
"""Escape a path or token for a ninja file.
ninja has no escape for ``|`` or a line break in a path, so those fail
here by name instead of producing a build file ninja misreads.
"""
text = str(value)
if bad := next((c for c in "|\r\n" if c in text), None):
raise EsphomeError(
f"Path {text!r} contains {bad!r}, which a ninja build file cannot "
"express; rename the file or directory"
)
return text.replace("$", "$$").replace(":", "$:").replace(" ", "$ ")
def quote_arg(tok: str) -> str:
@@ -90,3 +118,65 @@ def shell_token(tok: str, force: bool = False) -> str:
def quote_path(value: Path | str) -> str:
"""Force-quote a path for the ninja command line (shell/CreateProcess)."""
return shell_token(str(value), force=True)
def refresh_compile_commands(
ninja_path: Path, build_dir: Path, env: dict[str, str], ninja_changed: bool
) -> None:
"""Regenerate the compile DB (a pure function of build.ninja) when stale.
Freshness rides a stamp: the DB itself is written through
write_file_if_changed (its mtime feeds the idedata cache), so a
regeneration with identical content would stay "stale" forever. An
interrupted previous run may have rewritten the manifest without
regenerating the DB, hence the mtime comparison.
"""
compdb = build_dir / "compile_commands.json"
compdb_stamp = build_dir / ".compile_commands.stamp"
ninja_file = build_dir / "build.ninja"
if (
ninja_changed
or not compdb.is_file()
or not compdb_stamp.is_file()
or compdb_stamp.stat().st_mtime < ninja_file.stat().st_mtime
):
write_compile_commands(ninja_path, build_dir, env)
compdb_stamp.touch()
def write_compile_commands(
ninja_path: Path, build_dir: Path, env: dict[str, str]
) -> None:
compdb = build_dir / "compile_commands.json"
result = subprocess.run(
[str(ninja_path), "-C", str(build_dir), "-t", "compdb", *COMPILE_RULES],
env=env,
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if result.returncode != 0:
# Drop any stale database so consumers (IDE integration, clang-tidy,
# the memory analyzer) can't silently read outdated data
compdb.unlink(missing_ok=True)
raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}")
try:
entries = json.loads(result.stdout)
except ValueError as err:
compdb.unlink(missing_ok=True)
raise EsphomeError(
f"ninja produced an unparsable compile database: {err} "
f"(output starts {result.stdout[:120]!r})"
) from err
if not entries:
# compdb exits 0 with [] for unknown rule names; a renamed compile
# rule must fail the build, not silently strand every consumer
compdb.unlink(missing_ok=True)
raise EsphomeError(
"ninja produced an empty compile database; the generator's rule "
"names no longer match"
)
# write_file_if_changed keeps the mtime stable on no-op builds so the
# idedata cache stays valid
write_file_if_changed(compdb, result.stdout)
esphome/build_helpers/ninja_gen.py
+204
View File
@@ -0,0 +1,204 @@
"""The parts of a ``build.ninja`` every native backend emits the same way.
Rule names match ``SOURCE_KIND_FOR_SUFFIX`` values (c, cxx, asm, aspp) and
``ninja.COMPILE_RULES``, which the compile database is asked for.
"""
from __future__ import annotations
from collections.abc import Collection, Sequence
import logging
import os
from pathlib import Path
import sys
from typing import TYPE_CHECKING
from esphome.build_helpers.ninja import (
escape as _e,
quote_path as _q,
shell_token as _shell_token,
)
from esphome.framework_helpers import strip_win_long_path_prefix
from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX
if TYPE_CHECKING:
from esphome.arduino.library import ArduinoLibrary
_LOGGER = logging.getLogger(__name__)
_BUILD_TOOL = Path(__file__).parent.parent / "build_gen" / "build_tool.py"
def collect_sources(root: Path, exclude: Collection[str] = frozenset()) -> list[Path]:
return sorted(
p
for p in root.rglob("*")
if p.suffix in SOURCE_KIND_FOR_SUFFIX and p.name not in exclude
)
def common_parent(paths: list[Path]) -> Path:
return Path(os.path.commonpath([str(p.parent) for p in paths]))
def _command(words: Sequence[Path | str]) -> str:
program, *args = words
return " ".join([_q(program), *(_shell_token(str(a)) for a in args)])
def tool_lines(
cc: Sequence[Path | str], cxx: Sequence[Path | str], ccache: str | None
) -> list[str]:
"""The file header: the compilers and the helper tools as variables.
A compiler is its program followed by any arguments it always takes.
"""
return [
"# Auto-generated by ESPHome",
"ninja_required_version = 1.5",
f"cc = {_command(cc)}",
f"cxx = {_command(cxx)}",
# The NSIS launcher starts Python with a \\?\ extended-length path
# that cmd.exe cannot spawn; same strip every other emitted binary
# path gets
f"python = {_q(strip_win_long_path_prefix(sys.executable))}",
f"buildtool = {_q(_BUILD_TOOL)}",
f"ccache = {_q(ccache) if ccache else ''}",
"",
]
def compile_rule_lines() -> list[str]:
"""The compile rules; ``$own_includes`` is empty unless an edge sets it."""
return [
"rule c",
" command = $ccache $cc -MMD -MF $out.d $own_includes $cflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CC $out",
"rule cxx",
" command = $ccache $cxx -MMD -MF $out.d $own_includes $cxxflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = CXX $out",
"rule aspp",
" command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $own_includes $asflags $flags -c $in -o $out",
" depfile = $out.d",
" deps = gcc",
" description = AS $out",
# Plain assembler, as SCons's ASCOM: no preprocessor, so no
# depfile and no $flags (defines/includes) either
"rule asm",
" command = $ccache $cc -x assembler $asflags -c $in -o $out",
" description = AS $out",
]
def ar_rule_lines(ar: Path | str) -> list[str]:
return [
"rule ar",
f" command = $python $buildtool ar {_q(ar)} $out $out.rsp",
" rspfile = $out.rsp",
" rspfile_content = $in_newline",
" description = AR $out",
]
def compile_edges(
lines: list[str],
sources: list[Path],
root: Path,
group: str,
flags: str = "",
own_includes: str = "",
) -> list[str]:
"""Emit compile edges for ``sources``; return the object paths."""
objects = []
for src in sources:
rel = src.relative_to(root).as_posix()
obj = f"obj/{group}/{rel}.o"
escaped_obj = _e(obj)
lines.append(
f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}"
)
if own_includes:
lines.append(f" own_includes = {own_includes}")
if flags:
lines.append(f" flags = {flags}")
# Escaped once here: the returned paths only ever appear in build
# statements (archive/link inputs), which use ninja escaping
objects.append(escaped_obj)
return objects
def library_edges(
lines: list[str], libraries: list[ArduinoLibrary]
) -> tuple[list[str], list[str]]:
"""Emit every library's compile and archive edges.
Returns the archive names and the objects that link directly. A
library's own include dirs lead its compile lines, as PlatformIO searched
them first: the include path is one global list, so another library's
header of the same name would shadow them.
"""
archives: list[str] = []
direct_objs: list[str] = []
for lib in libraries:
if not lib.sources:
# Header-only libraries are legitimate; the log makes an empty
# srcFilter or broken tree traceable before link errors do
_LOGGER.debug(
"Library %s has no source files; contributing includes only",
lib.name,
)
continue
objs = compile_edges(
lines,
lib.sources,
common_parent(lib.sources),
f"lib/{lib.name}",
flags=" ".join(_shell_token(f) for f in lib.flags),
own_includes=" ".join(f"-I{_q(d)}" for d in lib.include_dirs),
)
if not lib.lib_archive:
# libArchive: false / dot_a_linkage=false: hand the objects to
# the linker directly so unreferenced-but-required symbols
# (exception handlers, weak overrides) survive
direct_objs.extend(objs)
continue
archive = f"lib{lib.name}.a"
lines.append(f"build {_e(archive)}: ar {' '.join(objs)}")
archives.append(archive)
return archives, direct_objs
# One build flag: the flag, plus its argument when that is a separate token
Flag = tuple[str, ...]
# Flags whose path operand is the next token; gcc also takes it glued on
PATH_ARG_FLAGS = ("-include", "-imacros", "-isystem", "-iquote", "-idirafter")
# Flags whose path operand is glued on
PATH_PREFIXES = ("-I", "-L", *PATH_ARG_FLAGS)
def _anchor(path: str, base: Path) -> str:
if not path or Path(path).is_absolute():
return path
return str(base / path)
def anchor_path_flag(flag: Flag, base: Path) -> Flag:
"""Anchor a flag's relative path operand at ``base``.
PlatformIO ran the compiler from the build path; ninja runs it from
``.pioenvs/<name>``, where a relative operand would point elsewhere.
"""
name, *args = flag
if args:
if name in PATH_ARG_FLAGS:
return (name, _anchor(args[0], base))
return flag
for prefix in PATH_PREFIXES:
if name.startswith(prefix):
return (prefix + _anchor(name[len(prefix) :], base),)
return flag
esphome/build_helpers/tools_cache.py
+8 -1
View File
@@ -33,4 +33,11 @@ def tools_cache_path(env_var: str, subdir: str) -> Path:
IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf")
SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf")
ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266")
TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE)
# The host backend installs nothing; the entry only holds its ccache dir
HOST_TOOLS_CACHE = ("ESPHOME_HOST_PREFIX", "host")
TOOLS_CACHE_SPECS = (
IDF_TOOLS_CACHE,
SDK_NRF_TOOLS_CACHE,
ARDUINO8266_TOOLS_CACHE,
HOST_TOOLS_CACHE,
)
esphome/components/host/__init__.py
+22 -11
View File
@@ -1,3 +1,5 @@
import sys
import esphome.codegen as cg
import esphome.config_validation as cv
from esphome.const import (
@@ -8,9 +10,9 @@ from esphome.const import (
KEY_TARGET_PLATFORM,
PLATFORM_HOST,
ThreadModel,
Toolchain,
)
from esphome.core import CORE
from esphome.platformio.toolchain import copy_ccache_script
from esphome.core import CORE, EsphomeError
from esphome.types import ConfigType
from .const import KEY_HOST
@@ -37,7 +39,9 @@ CONFIG_SCHEMA = cv.All(
cv.Optional(CONF_MAC_ADDRESS, default="98:35:69:ab:f6:79"): cv.mac_address,
}
),
cv.require_platformio_toolchain("host"),
# The host builds with the machine's own compiler through ninja; there
# is no PlatformIO backend, so a CLI --toolchain must name this one
cv.resolve_toolchain("host", (Toolchain.HOST,), Toolchain.HOST),
set_core_data,
)
@@ -48,16 +52,23 @@ async def to_code(config: ConfigType) -> None:
# The prefs file finds stored preferences by key, so key migration is possible
cg.add_define("USE_PREFERENCE_KEY_LOOKUP")
cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts)
cg.add_build_flag("-std=gnu++20")
cg.set_cpp_standard("gnu++20")
cg.add_define("ESPHOME_BOARD", "host")
cg.add_define("ESPHOME_VARIANT", "HOST")
cg.add_define(ThreadModel.MULTI_ATOMICS)
cg.add_platformio_option("platform", "platformio/native")
cg.add_platformio_option("lib_ldf_mode", "off")
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("extra_scripts", ["pre:ccache.py"])
# Called by writer.py
def copy_files() -> None:
copy_ccache_script()
# Called by __main__.compile_program; True means this platform built the
# program itself instead of falling through to the PlatformIO toolchain.
def run_compile(args: object, config: ConfigType) -> bool:
from esphome.host import toolchain
if sys.platform == "win32":
# Here, not in validation: `esphome config` must keep working
raise EsphomeError(
"The host platform uses POSIX interfaces and does not build on "
"Windows; use WSL or a Linux or macOS machine"
)
if toolchain.run_compile(config, CORE.verbose) != 0:
raise EsphomeError("Host build failed")
return True
esphome/const.py
+3 -1
View File
@@ -23,12 +23,14 @@ class Toolchain(StrEnum):
SDK_NRF = "sdk-nrf"
# ESP8266: the Arduino core built directly (no PlatformIO)
ARDUINO = "arduino"
# host: the system C/C++ compiler driven by ninja (no PlatformIO)
HOST = "host"
# Toolchains that drive their build natively and never read platformio.ini.
# SDK_NRF is absent on purpose: the zephyr backend keeps consuming
# platformio_options.
NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO})
NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO, Toolchain.HOST})
class Platform(StrEnum):
esphome/core/__init__.py
+6
View File
@@ -996,6 +996,12 @@ class EsphomeCore:
``using_arduino``, which is the target framework)."""
return self.toolchain == Toolchain.ARDUINO
@property
def using_toolchain_host(self):
"""The native host build toolchain: the system compiler driven by
ninja (the only toolchain the host platform serves)."""
return self.toolchain == Toolchain.HOST
@property
def using_native_toolchain(self):
"""Whether the selected toolchain builds natively, without reading
esphome/host/__init__.py
+10
View File
@@ -0,0 +1,10 @@
"""Native (PlatformIO-free) build support for the host platform.
Builds the generated sources with the machine's own C/C++ compiler through a
ninja file and drives the build directly -- the host equivalent of
``esphome.espidf``. Nothing is downloaded: the compiler comes from PATH and
registry libraries go through the shared library converter.
Deliberately importable without the host component to avoid circular
imports; the component wires these modules in via lazy imports.
"""
esphome/host/toolchain.py
+196
View File
@@ -0,0 +1,196 @@
"""Native host build driver (the PlatformIO ``run`` equivalent).
The compiler and binutils come from PATH (``CC``/``CXX``/``AR``/``OBJDUMP``/
``READELF`` override the lookup, like make and CMake), ninja from PATH or
the ninja PyPI wheel, and ccache is used when found. The build lives under
``.pioenvs/<name>/`` so ``CORE.firmware_bin`` and the clean paths stay the
ones the PlatformIO build used.
"""
from __future__ import annotations
import logging
import os
from pathlib import Path
import shlex
import shutil
import subprocess
import sys
from typing import NamedTuple
from esphome.build_helpers.ccache import ccache_env, resolve_absolute_ccache_path
from esphome.build_helpers.native import warn_ignored_platformio_options
from esphome.build_helpers.ninja import (
absolute_tool,
find_ninja,
refresh_compile_commands,
)
from esphome.build_helpers.tools_cache import HOST_TOOLS_CACHE
from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME
from esphome.core import CORE, EsphomeError
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
# The output name PlatformIO's native platform produced; CORE.firmware_bin
# and the integration-test harness resolve it by this name
PROGRAM_NAME = "program"
# platformio_options keys the host build reads (lib_ignore feeds the library
# converter); anything else has no native equivalent and is warned about
CONSUMED_PIO_OPTIONS = frozenset({"lib_ignore"})
class HostCompilers(NamedTuple):
"""The resolved C and C++ compiler commands: the program, then any
arguments its override carried (``CC="gcc -m32"``)."""
cc: tuple[str, ...]
cxx: tuple[str, ...]
def find_command(env_var: str, candidates: tuple[str, ...]) -> tuple[str, ...]:
"""Resolve a build tool: ``env_var`` when set, else the first candidate
found on PATH. Returns the program and the override's arguments.
An override that does not resolve fails by name rather than falling back
silently to a different compiler than the user asked for.
"""
if override := os.environ.get(env_var, "").strip():
program, *args = shlex.split(override)
if Path(program).stem == "ccache" and args:
# The build adds ccache itself; twice would hide the compiler
program, *args = args
# which() accepts an absolute path as well as a bare program name
if (resolved := shutil.which(program)) is None:
raise EsphomeError(
f"{env_var}={override!r} does not name a runnable program"
)
return (absolute_tool(resolved), *args)
for name in candidates:
if (found := shutil.which(name)) is not None:
return (absolute_tool(found),)
raise EsphomeError(
f"{candidates[0]} not found on PATH (tried {', '.join(candidates)}); "
f"install it or set {env_var}"
)
def find_tool(env_var: str, candidates: tuple[str, ...]) -> str:
"""Resolve a build tool that is called as a bare program."""
program, *args = find_command(env_var, candidates)
if args:
raise EsphomeError(f"{env_var} must name a program without arguments")
return program
def find_compilers() -> HostCompilers:
"""The C and C++ compilers the build uses (gcc first, as PlatformIO did)."""
return HostCompilers(
cc=find_command("CC", ("gcc", "clang", "cc")),
cxx=find_command("CXX", ("g++", "clang++", "c++")),
)
def get_build_dir() -> Path:
return CORE.relative_pioenvs_path(CORE.name)
def get_elf_path() -> Path:
return get_build_dir() / PROGRAM_NAME
def get_objdump_path() -> Path:
return Path(find_tool("OBJDUMP", ("objdump",)))
def get_readelf_path() -> Path:
return Path(find_tool("READELF", ("readelf",)))
def check_analysis_supported() -> None:
"""Refuse analyze-memory where the program is not an ELF file.
Called before the compile, so an unsupported machine fails at once.
"""
if sys.platform != "linux":
raise EsphomeError(
"analyze-memory reads ELF files; the host build on "
f"{sys.platform} produces a different format"
)
def get_build_env(ccache: str | None) -> dict[str, str]:
return {**os.environ, **ccache_env(ccache, HOST_TOOLS_CACHE)}
def run_compile(config: ConfigType, verbose: bool) -> int:
from esphome.build_gen import host as build_gen
warn_ignored_platformio_options(CONSUMED_PIO_OPTIONS)
# Probe the cheap local dependencies before resolving libraries
ninja_path = find_ninja()
compilers = find_compilers()
# Resolved once per build: the resolution probes PATH and spawns the
# runnability check, and three consumers need the same answer
ccache = resolve_absolute_ccache_path()
ninja_changed = build_gen.write_project(compilers, ccache)
build_dir = get_build_dir()
env = get_build_env(ccache)
refresh_compile_commands(ninja_path, build_dir, env, ninja_changed)
cmd = [str(ninja_path)]
if verbose:
cmd.append("-v")
if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT):
cmd += ["-j", str(jobs)]
# The explicit target, not the default statement: a generator defect
# that drops it fails loudly with "unknown target" instead of a green
# no-op run that leaves a stale program in place
cmd.append(PROGRAM_NAME)
_LOGGER.debug("Running: %s", " ".join(cmd))
# cwd instead of -C also drops the "Entering directory" banner
rc = subprocess.run(
cmd, cwd=build_dir, env=env, check=False, close_fds=False
).returncode
if rc != 0:
return rc
elf = get_elf_path()
if not elf.is_file():
# ninja refused a manifest missing the target above; this covers a
# rule that ran but wrote elsewhere
_LOGGER.error("Build produced no %s", elf)
return 1
from esphome.build_helpers.idedata import warn_if_idedata_missing
warn_if_idedata_missing(lambda: _load_idedata(ccache))
return 0
def get_idedata() -> dict | None:
"""Derive idedata from the build's compile_commands.json.
Same contract as ``espidf.toolchain.get_idedata``: the fields IDE
integrations, clang-tidy, and the memory analyzer expect. Returns None
when nothing has been built yet.
"""
# Deliberately uncached: env/PATH can change between builds in a
# long-lived host process
return _load_idedata(resolve_absolute_ccache_path())
def _load_idedata(ccache: str | None) -> dict | None:
from esphome.build_helpers.idedata import load_or_build_idedata
return load_or_build_idedata(
get_build_dir() / "compile_commands.json",
get_elf_path(),
CORE.relative_internal_path("idedata", f"{CORE.name}.json"),
# The compile DB's commands carry the same ccache prefix the ninja
# rules were generated with
launcher=ccache,
)
esphome/platformio/library.py
+32 -10
View File
@@ -360,7 +360,7 @@ class LibraryBackend:
"""
platform: str | None
framework: str
framework: str | None
emit: Callable[["ConvertedLibrary"], None]
cache_key: str
# Owner-less names this returns True for are skipped by the walk;
@@ -368,6 +368,10 @@ class LibraryBackend:
# reconciles provided_requests after resolving
provides: Callable[[str], bool] | None = None
provided_requests: set[str] = field(default_factory=set)
# Accept a library without library.json/library.properties, as
# PlatformIO does (its defaults: src/ or the root, plus include/). Off
# for backends whose emitted build files need the manifest.
manifest_optional: bool = False
def ensure_list[T](obj: T | list[T]) -> list[T]:
@@ -859,6 +863,11 @@ def _url_or_none(value: Any) -> str | None:
return value if parsed.scheme and parsed.netloc else None
# URL schemes that name a local library folder; symlink:// is PlatformIO's
# spelling for one it links instead of copying, which is the same to us
_LOCAL_SCHEMES = ("file", "symlink")
def _node_key(
name: str | None, version: str | None, repository: str | None
) -> tuple[str, str, tuple[str | None, str | None]]:
@@ -896,7 +905,7 @@ def _node_key(
scheme = urlsplit(candidate).scheme
except ValueError:
scheme = ""
if scheme == "file" or _url_or_none(candidate):
if scheme in _LOCAL_SCHEMES or _url_or_none(candidate):
name, repository = custom_name, candidate
else:
# Anything with ``://`` was meant to be a URL; failing it fast
@@ -905,7 +914,7 @@ def _node_key(
if repository:
is_git_prefixed = repository.startswith("git+")
split_result = urlsplit(repository.removeprefix("git+"))
if split_result.scheme == "file" and not is_git_prefixed:
if (scheme := split_result.scheme) in _LOCAL_SCHEMES and not is_git_prefixed:
# A plain file:// URL points at a local library directory. A local
# file URL is written file:///absolute/path (empty host) or, less
# commonly, file://localhost/path. Anything else -- a real host, or
@@ -913,8 +922,8 @@ def _node_key(
# rejected rather than silently resolved to the wrong directory.
if split_result.netloc not in ("", "localhost"):
raise RuntimeError(
f"Unsupported host in file:// library URL '{repository}'; "
"use an absolute path, e.g. file:///path/to/lib"
f"Unsupported host in {scheme}:// library URL '{repository}'; "
f"use an absolute path, e.g. {scheme}:///path/to/lib"
)
# Validate the URL path itself (always POSIX-style, leading slash),
# not the OS path: on Windows a "/foo" path is not is_absolute()
@@ -924,8 +933,8 @@ def _node_key(
url_path = split_result.path
if not url_path.startswith("/") or not PurePosixPath(url_path).name:
raise RuntimeError(
f"file:// library URL '{repository}' must be an absolute "
"directory path, e.g. file:///path/to/lib"
f"{scheme}:// library URL '{repository}' must be an absolute "
f"directory path, e.g. {scheme}:///path/to/lib"
)
path = url2pathname(url_path)
return (name or PurePosixPath(url_path).name), "local", (path, None)
@@ -1244,9 +1253,16 @@ def convert_libraries(
library_properties_path = source_dir / "library.properties"
has_json = library_json_path.is_file()
has_properties = library_properties_path.is_file()
if not has_json and not has_properties and not node.is_local:
if (
not has_json
and not has_properties
and not node.is_local
and not backend.manifest_optional
):
# An interrupted clone/extraction self-heals with one forced
# re-download; a local source has nothing to re-download
# re-download; a local source has nothing to re-download.
# A backend accepting manifest-less libraries cannot tell
# one from a torn download and would re-fetch every build.
_LOGGER.warning(
"Library %s at %s is missing library.json and library.properties; "
"re-downloading",
@@ -1260,6 +1276,12 @@ def convert_libraries(
component.data = parse_library_json(library_json_path)
elif has_properties:
component.data = parse_library_properties(library_properties_path)
elif backend.manifest_optional:
_LOGGER.debug(
"Library %s has no manifest; using PlatformIO's default layout",
key,
)
component.data = {"name": component.name}
else:
# Local sources are user input (EsphomeError); a registry/git
# miss means a corrupt cache (RuntimeError)
@@ -1286,7 +1308,7 @@ def convert_libraries(
# cross-platform skip stays at debug, other causes warn
if key in top_level_keys:
reason = (
f"is not compatible with {backend.framework}"
f"is not compatible with {backend.framework or backend.platform}"
if isinstance(e, IncompatiblePlatform)
else "has a malformed manifest"
)
esphome/storage_json.py
+1 -1
View File
@@ -155,7 +155,7 @@ class StorageJSON:
self.framework = framework
# The core platform of this firmware. Like "esp32", "rp2", "host" etc.
self.core_platform = core_platform
# The toolchain used for the build ("platformio" / "esp-idf")
# The toolchain used for the build ("platformio" / "esp-idf" / "host")
self.toolchain = toolchain
# The area of the node
self.area = area
requirements.txt
+1 -1
View File
@@ -28,7 +28,7 @@ smpclient==7.2.0
requests==2.34.2
py7zr==1.1.3
platformdirs==4.11.12 # native esp-idf toolchain global cache dir
ninja==1.13.2 # native esp8266 arduino toolchain build driver
ninja==1.13.2 # native host and esp8266 arduino toolchain build driver
filelock==4.0.1 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg
# esp-idf >= 5.0 requires this
script/build_helpers.py
+10 -11
View File
@@ -22,8 +22,8 @@ from esphome.__main__ import command_compile, parse_args
from esphome.config import validate_config
from esphome.const import CONF_PLATFORM
from esphome.core import CORE
from esphome.host.toolchain import get_elf_path
from esphome.loader import get_component, get_platform
from esphome.platformio.toolchain import get_idedata
from tests.testing_helpers import ComponentManifestOverride, set_testing_manifest
# This must coincide with the version in /platformio.ini
@@ -220,7 +220,7 @@ def create_host_config(
friendly_name: str,
libraries: str | list[str],
includes: list[str],
platformio_options: dict,
build_flags: list[str],
) -> dict:
"""Create an ESPHome host configuration for C++ builds.
@@ -229,7 +229,7 @@ def create_host_config(
friendly_name: Human-readable name
libraries: PlatformIO library specification(s)
includes: List of include folders for the build
platformio_options: Dict of platformio_options to set
build_flags: Compiler/linker flags for the build
Returns:
Configuration dict for ESPHome
@@ -239,7 +239,7 @@ def create_host_config(
"name": config_name,
"friendly_name": friendly_name,
"libraries": libraries,
"platformio_options": platformio_options,
"build_flags": build_flags,
"includes": includes,
},
HOST_KEY: {},
@@ -403,13 +403,12 @@ def compile_and_get_binary(
return EXIT_COMPILE_ERROR, None
# After a successful compilation, locate the executable:
idedata = get_idedata(config)
if idedata is None:
program_path = get_elf_path()
if not program_path.is_file():
print("Cannot find executable")
return EXIT_NO_EXECUTABLE, None
program_path: str = idedata.raw["prog_path"]
return EXIT_OK, program_path
return EXIT_OK, str(program_path)
def build_and_run(
@@ -419,7 +418,7 @@ def build_and_run(
config_prefix: str,
friendly_name: str,
libraries: str | list[str],
platformio_options: dict,
build_flags: list[str],
main_entry: str,
label: str = "build",
build_only: bool = False,
@@ -438,7 +437,7 @@ def build_and_run(
config_prefix: Prefix for the config name (e.g. "cpptests", "cppbench")
friendly_name: Human-readable name for the config
libraries: PlatformIO library specification(s)
platformio_options: PlatformIO options dict
build_flags: Compiler/linker flags for the build
main_entry: Name of the main entry file (e.g. "main.cpp")
label: Label for log messages
build_only: If True, print binary path and return without running
@@ -488,7 +487,7 @@ def build_and_run(
config_name: str = f"{config_prefix}-" + hash_components(components)
config = create_host_config(
config_name, friendly_name, libraries, includes, platformio_options
config_name, friendly_name, libraries, includes, build_flags
)
exit_code, program_path = compile_and_get_binary(
script/cpp_benchmark.py
+11 -19
View File
@@ -25,19 +25,14 @@ CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core"
# allow benchmarks to compile on the host platform.
STUBS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "stubs"
PLATFORMIO_OPTIONS = {
"build_flags": [
"-Os", # match firmware optimization level (detects inlining regressions)
"-g", # debug symbols for profiling
"-ffunction-sections", # required for dead-code stripping with -Os
"-fdata-sections", # required for dead-code stripping with -Os
"-DUSE_BENCHMARK", # disable WarnIfComponentBlockingGuard in finish()
f"-I{STUBS_DIR}", # stub headers for ESP32-only components
],
# Use deep+ LDF mode to ensure PlatformIO detects the benchmark
# library dependency from nested includes.
"lib_ldf_mode": "deep+",
}
BUILD_FLAGS = [
"-Os", # match firmware optimization level (detects inlining regressions)
"-g", # debug symbols for profiling
"-ffunction-sections", # required for dead-code stripping with -Os
"-fdata-sections", # required for dead-code stripping with -Os
"-DUSE_BENCHMARK", # disable WarnIfComponentBlockingGuard in finish()
f"-I{STUBS_DIR}", # stub headers for ESP32-only components
]
def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int:
@@ -46,7 +41,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) ->
# containing {"lib_path": "/path/to/google_benchmark"}.
lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG")
pio_options = PLATFORMIO_OPTIONS
build_flags = BUILD_FLAGS
if lib_config_json:
lib_config = json.loads(lib_config_json)
benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}"
@@ -61,10 +56,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) ->
"-DCODSPEED_ANALYSIS",
f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"',
]
pio_options = {
**PLATFORMIO_OPTIONS,
"build_flags": PLATFORMIO_OPTIONS["build_flags"] + codspeed_flags,
}
build_flags = BUILD_FLAGS + codspeed_flags
else:
benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB
@@ -77,7 +69,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) ->
config_prefix="cppbench",
friendly_name="CPP Benchmarks",
libraries=benchmark_lib,
platformio_options=pio_options,
build_flags=build_flags,
main_entry="main.cpp",
label="benchmarks",
build_only=build_only,
script/cpp_unit_test.py
+11 -19
View File
@@ -15,24 +15,16 @@ from helpers import get_all_components, root_path
# Path to /tests/components
COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components"
PLATFORMIO_OPTIONS = {
"build_type": "debug",
"build_unflags": [
"-Os", # remove size-opt flag
],
"build_flags": [
"-Og", # optimize for debug
"-DESPHOME_DEBUG", # enable debug assertions
# Enable the address and undefined behavior sanitizers
"-fsanitize=address",
"-fsanitize=undefined",
"-fno-omit-frame-pointer",
],
"debug_build_flags": [ # only for debug builds
"-g3", # max debug info
"-ggdb3",
],
}
BUILD_FLAGS = [
"-Og", # optimize for debug
"-g3", # max debug info
"-ggdb3",
"-DESPHOME_DEBUG", # enable debug assertions
# Enable the address and undefined behavior sanitizers
"-fsanitize=address",
"-fsanitize=undefined",
"-fno-omit-frame-pointer",
]
def run_tests(selected_components: list[str]) -> int:
@@ -47,7 +39,7 @@ def run_tests(selected_components: list[str]) -> int:
config_prefix="cpptests",
friendly_name="CPP Unit Tests",
libraries=PLATFORMIO_GOOGLE_TEST_LIB,
platformio_options=PLATFORMIO_OPTIONS,
build_flags=BUILD_FLAGS,
main_entry="main.cpp",
label="unit tests",
)
script/determine-jobs.py
+16 -5
View File
@@ -111,14 +111,25 @@ INTEGRATION_TESTS_SPLIT_THRESHOLD = 10
INTEGRATION_TESTS_SPLIT_BUCKETS = 5
INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0
# platformio and aioesphomeapi (requirements.txt), the pytest stack
# (requirements_test.txt) and the fixture every session compiles; a change
# to any runs the full matrix
# aioesphomeapi (requirements.txt), the pytest stack (requirements_test.txt)
# and the native host build backend every test compiles with; a change to
# any runs the full matrix
INTEGRATION_TESTS_TRIGGER_FILES = frozenset(
{
"requirements.txt",
"requirements_test.txt",
"tests/integration/fixtures/cache_init.yaml",
"esphome/arduino/library.py",
"esphome/build_gen/build_tool.py",
"esphome/build_gen/host.py",
"esphome/build_helpers/ccache.py",
"esphome/build_helpers/idedata.py",
"esphome/build_helpers/native.py",
"esphome/build_helpers/ninja.py",
"esphome/build_helpers/ninja_gen.py",
"esphome/build_helpers/tools_cache.py",
"esphome/framework_helpers.py",
"esphome/host/toolchain.py",
"esphome/platformio/library.py",
}
)
@@ -240,7 +251,7 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s
- conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc.
4. A file in INTEGRATION_TESTS_TRIGGER_FILES changed
- The dependency pins and the session init fixture affect every test
- The dependency pins and the host build backend affect every test
Returns (run_all=False, [test_files...]) when:
tests/integration/conftest.py
+55 -71
View File
@@ -16,7 +16,6 @@ import re
import shutil
import signal
import socket
import subprocess
import sys
import tempfile
import time
@@ -35,7 +34,7 @@ from esphome.helpers import (
write_file,
write_file_if_changed,
)
from esphome.platformio.toolchain import get_idedata
from esphome.host.toolchain import get_elf_path
from .const import (
API_CONNECTION_TIMEOUT,
@@ -82,16 +81,9 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent
INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests"
def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
"""Get environment variables for PlatformIO with shared cache."""
def _get_build_env() -> dict[str, str]:
"""Environment for an ``esphome compile`` subprocess."""
env = os.environ.copy()
env["PLATFORMIO_CORE_DIR"] = str(cache_dir)
env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache")
# libdeps is keyed only by env name (the device name), and fixtures share
# names; two xdist workers first-compiling the same name race pio pkg
# install in the same directory. Keep libdeps per worker.
worker = os.environ.get("PYTEST_XDIST_WORKER", "master")
env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker)
# Prevent cache cleaning during integration tests
env["ESPHOME_SKIP_CLEAN_BUILD"] = "1"
# Cap each compile's -j so several xdist workers do not each spawn a
@@ -110,53 +102,41 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
return env
# Registry libraries (noise-c, lvgl, ...) download into <data_dir>/pio_components,
# and every integration test gets its own data dir. Share one download cache
# per xdist worker instead: the converter has no cross-process lock, so
# workers must never write the same dir, while a worker's later tests (and
# later runs) reuse what it fetched.
LIBRARY_CACHE_ROOT = INTEGRATION_TESTS_ROOT / "pio_components"
@pytest.fixture(scope="session")
def shared_platformio_cache() -> Generator[Path]:
"""Initialize a shared PlatformIO cache for all integration tests."""
# Use a dedicated directory for integration tests to avoid conflicts.
test_cache_dir = INTEGRATION_TESTS_ROOT
cache_dir = test_cache_dir / "platformio"
def shared_library_cache() -> Path:
"""This worker's shared registry-library download cache."""
worker = os.environ.get("PYTEST_XDIST_WORKER", "master")
# Keyed by checkout too: two sessions on one machine share worker ids
cache_dir = LIBRARY_CACHE_ROOT / _REPO_KEY / worker
cache_dir.mkdir(parents=True, exist_ok=True)
return cache_dir
# Use a lock file in the home directory to ensure only one process initializes the cache
# This is needed when running with pytest-xdist
# The lock file must be in a directory that already exists to avoid race conditions
lock_file = Path.home() / ".esphome-integration-tests-init.lock"
# Always acquire the lock to ensure cache is ready before proceeding
with lock_file.open("w") as lock_fd:
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX)
# Check if the native platform is installed (the actual indicator of a populated cache)
native_platform = cache_dir / "platforms" / "native"
if not native_platform.exists():
# Create the test cache directory if it doesn't exist
test_cache_dir.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory() as tmpdir:
# Use the cache_init fixture for initialization
init_dir = Path(tmpdir)
fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml"
config_path = init_dir / "cache_init.yaml"
config_path.write_text(
fixture_path.read_text(encoding="utf-8"), encoding="utf-8"
)
# Run compilation to populate the cache
# We must succeed here to avoid race conditions where multiple
# tests try to populate the same cache directory simultaneously
env = _get_platformio_env(cache_dir)
subprocess.run(
[sys.executable, "-m", "esphome", "compile", str(config_path)],
check=True,
cwd=init_dir,
env=env,
close_fds=False,
)
# Lock is held until here, ensuring cache is fully populated before any test proceeds
yield cache_dir
def _link_library_cache(config_dir: Path, cache_dir: Path) -> None:
"""Point a config dir's library download cache at the shared one
(blocking, run in executor)."""
data_dir = config_dir / ".esphome"
data_dir.mkdir(exist_ok=True)
link = data_dir / "pio_components"
if link.is_symlink():
if link.resolve() == cache_dir.resolve():
return
# A shared build dir was linked by another worker; writing through
# it would put two workers in one cache
link.unlink()
elif link.exists():
# A real dir from a run predating the shared cache; nothing in it is
# worth more than a re-download
rmtree(link)
link.symlink_to(cache_dir, target_is_directory=True)
@pytest.fixture(scope="module", autouse=True)
@@ -238,18 +218,17 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s
content = content.replace("api:", f"api:\n port: {unused_tcp_port}")
# Add debug build flags for integration tests to enable assertions
if "esphome:" in content and "platformio_options:" not in content:
# Add platformio_options with debug flags after esphome:
if "esphome:" in content and "build_flags:" not in content:
# Add build_flags with debug flags after esphome:
content = content.replace(
"esphome:",
"esphome:\n"
" # Enable assertions for integration tests\n"
" platformio_options:\n"
" build_flags:\n"
' - "-DDEBUG" # Enable assert() statements\n'
' - "-DESPHOME_DEBUG" # Enable ESPHOME_DEBUG_ASSERT checks\n'
' - "-DESPHOME_DEBUG_API" # Enable API protocol asserts\n'
' - "-g" # Add debug symbols',
" build_flags:\n"
' - "-DDEBUG" # Enable assert() statements\n'
' - "-DESPHOME_DEBUG" # Enable ESPHOME_DEBUG_ASSERT checks\n'
' - "-DESPHOME_DEBUG_API" # Enable API protocol asserts\n'
' - "-g" # Add debug symbols',
)
# Replace external component path placeholder if present
@@ -283,8 +262,8 @@ async def write_yaml_config(
yield _write_config
# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale
# dirs for a fixture are pruned when its content hash changes.
# Deliberately not CI-cached; stale dirs for a fixture are pruned when its
# content hash changes.
SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds"
# In the dir name (not just the hash) so pruning stays inside this checkout
@@ -470,8 +449,7 @@ def _resolve_compiled_binary(config_path: Path) -> Path:
)
if config is None:
raise RuntimeError(f"Failed to read config from {config_path}")
idedata = get_idedata(config)
binary_path = Path(idedata.firmware_elf_path)
binary_path = get_elf_path()
if not binary_path.exists():
raise RuntimeError(f"Compiled binary not found at {binary_path}")
return binary_path
@@ -480,19 +458,22 @@ def _resolve_compiled_binary(config_path: Path) -> Path:
@pytest_asyncio.fixture
async def compile_esphome(
integration_test_dir: Path,
shared_platformio_cache: Path,
shared_library_cache: Path,
request: pytest.FixtureRequest,
) -> AsyncGenerator[CompileFunction]:
"""Compile an ESPHome configuration and return the binary path."""
async def _compile(config_path: Path) -> Path:
# Use the shared PlatformIO cache for faster compilation
# This avoids re-downloading dependencies for each test
env = _get_platformio_env(shared_platformio_cache)
env = _get_build_env()
loop = asyncio.get_running_loop()
name = _shared_yaml_name(request)
if name is None:
# Share the library download cache so a test never re-fetches
# what an earlier one already pulled from the registry
await loop.run_in_executor(
None, _link_library_cache, integration_test_dir, shared_library_cache
)
await _run_esphome_compile(config_path, integration_test_dir, env)
return await loop.run_in_executor(
None, _resolve_compiled_binary, config_path
@@ -568,6 +549,9 @@ async def compile_esphome(
await loop.run_in_executor(
None, write_file_if_changed, shared_config, content
)
await loop.run_in_executor(
None, _link_library_cache, shared_dir, shared_library_cache
)
await _run_esphome_compile(shared_config, shared_dir, env)
if built is None or not built.exists():
built = await loop.run_in_executor(
tests/integration/fixtures/api_action_timeout.yaml
+2 -3
View File
@@ -1,9 +1,8 @@
esphome:
name: api-action-timeout-test
# Use a short timeout for testing (500ms instead of 30s)
platformio_options:
build_flags:
- "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500"
build_flags:
- "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500"
host:
tests/integration/fixtures/cache_init.yaml
-10
View File
@@ -1,10 +0,0 @@
esphome:
name: cache-init
host:
api:
encryption:
key: "IIevImVI42I0FGos5nLqFK91jrJehrgidI0ArwMLr8w="
logger:
tests/integration/fixtures/fnv1_hash_object_id.yaml
+2 -3
View File
@@ -1,8 +1,7 @@
esphome:
name: fnv1-hash-object-id-test
platformio_options:
build_flags:
- "-DDEBUG"
build_flags:
- "-DDEBUG"
on_boot:
- lambda: |-
using esphome::fnv1_hash_object_id;
tests/integration/fixtures/fnv1a_hash.yaml
+2 -3
View File
@@ -1,8 +1,7 @@
esphome:
name: fnv1a-hash-test
platformio_options:
build_flags:
- "-DDEBUG"
build_flags:
- "-DDEBUG"
on_boot:
- lambda: |-
using esphome::fnv1a_hash;
tests/integration/fixtures/micros_to_millis.yaml
+2 -3
View File
@@ -1,8 +1,7 @@
esphome:
name: micros-to-millis-test
platformio_options:
build_flags:
- "-DDEBUG"
build_flags:
- "-DDEBUG"
on_boot:
- lambda: |-
using esphome::micros_to_millis;
tests/script/test_helpers.py
+1 -4
View File
@@ -2143,10 +2143,7 @@ def test_no_orphan_integration_fixtures() -> None:
fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve()
fixtures = list(fixtures_dir.glob("*.yaml"))
assert fixtures, f"no fixtures found under {fixtures_dir}"
# cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead
orphans = [
f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping
]
orphans = [f.stem for f in fixtures if f.stem not in mapping]
assert not orphans, f"fixtures invisible to CI test selection: {orphans}"
tests/unit_tests/build_gen/test_arduino8266.py
+4 -1
View File
@@ -604,7 +604,7 @@ def test_write_project_libraries_and_variant(
)
_set_flags("-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS")
with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.arduino8266"):
with caplog.at_level(logging.DEBUG, logger="esphome.build_helpers.ninja_gen"):
content = _write_ninja(
paths, libraries=[library, headers_only], ccache="/cc/ccache"
)
@@ -616,6 +616,9 @@ def test_write_project_libraries_and_variant(
assert "libHeadersOnly.a" not in content
assert "Library HeadersOnly has no source files" in caplog.text
assert " flags = -DMYLIB=1" in content
# A library's own include dirs lead its compile lines
assert " own_includes = -I" in content
assert "$own_includes $cxxflags $flags" in content
assert "-lalgobsec" in content
# Library link flags reach the firmware link line; .cc compiles as C++
assert "-Wl,--wrap=malloc" in content
tests/unit_tests/build_gen/test_host.py
+368
View File
@@ -0,0 +1,368 @@
"""Tests for esphome.build_gen.host (the host ninja generator)."""
from __future__ import annotations
import logging
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.arduino.library import ArduinoLibrary
from esphome.build_gen import host as build_gen
from esphome.build_helpers.ninja import escape as _e, quote_path as _q
from esphome.build_helpers.ninja_gen import Flag
from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_HOST
from esphome.core import CORE, EsphomeError, Library
from esphome.host.toolchain import PROGRAM_NAME, HostCompilers
COMPILERS = HostCompilers(cc=("/usr/bin/gcc", "-m32"), cxx=("/usr/bin/g++",))
@pytest.fixture(autouse=True)
def _core(tmp_path: Path) -> None:
CORE.build_path = tmp_path
CORE.name = "dev"
CORE.cpp_standard = "gnu++20"
CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_HOST}
def _make_src(tmp_path: Path, *names: str) -> Path:
src = tmp_path / "src"
for name in names:
path = src / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("")
return src
def _render(ccache: str | None = None) -> tuple[bool, str]:
changed = build_gen.write_project(COMPILERS, ccache)
ninja = CORE.build_path / ".pioenvs" / "dev" / "build.ninja"
return changed, ninja.read_text()
@pytest.mark.parametrize(
("entries", "compile_flags", "link_flags"),
[
(
["-DUSE_HOST", "-Iinc", "-Ufoo", "-Wall"],
[("-DUSE_HOST",), ("-Iinc",), ("-Ufoo",), ("-Wall",)],
[],
),
(["-std=gnu++20"], [("-std=gnu++20",)], []),
(
["-lssl", "-L/opt/lib", "-Wl,--gc-sections"],
[],
[("-lssl",), ("-L/opt/lib",), ("-Wl,--gc-sections",)],
),
# A link-only flag travels with its argument
(
["-Xlinker --wrap=malloc", "-z noexecstack", "-framework Cocoa"],
[],
[
("-Xlinker", "--wrap=malloc"),
("-z", "noexecstack"),
("-framework", "Cocoa"),
],
),
# Both lines, as SCons routes unclassified flags
(
["-g -O2", "-fsanitize=address", "-pthread", "--coverage"],
[
("-g",),
("-O2",),
("-fsanitize=address",),
("-pthread",),
("--coverage",),
],
[
("-g",),
("-O2",),
("-fsanitize=address",),
("-pthread",),
("--coverage",),
],
),
# A flag with a path argument travels with it, glued on or not
(
["-include pre.h", "-isystem /x", "-isystem/y", "-iquotez"],
[("-include", "pre.h"), ("-isystem", "/x"), ("-isystem/y",), ("-iquotez",)],
[],
),
],
)
def test_split_flags(
entries: list[str], compile_flags: list[Flag], link_flags: list[Flag]
) -> None:
flags = build_gen.parse_flags(entries, "esphome")
assert build_gen.split_flags(flags) == (compile_flags, link_flags)
@pytest.mark.parametrize(
"entries",
[
# The flag is the last token of its entry
["-g -include"],
# Entries are a set: the next entry is not this flag's argument
["-include", "pre.h"],
["-framework", "-pthread"],
# A path never starts with "-"
["-include -pthread"],
],
)
def test_parse_flags_refuses_a_flag_without_its_argument(entries: list[str]) -> None:
with pytest.raises(EsphomeError, match="with no argument; write the flag"):
build_gen.parse_flags(entries, "esphome")
def test_parse_flags_accepts_a_linker_argument_that_is_a_flag() -> None:
assert build_gen.parse_flags(["-Xlinker --wrap=malloc"], "esphome") == [
("-Xlinker", "--wrap=malloc")
]
def test_flag_lists_route_the_standard() -> None:
"""cpp_standard wins for C++; C never sees a C++ standard."""
CORE.build_flags = {
"-DUSE_HOST",
"-Iinc",
"-Wl,-x",
"-g",
"-lssl",
"-std=c++17",
"-std=gnu17",
}
CORE.cxx_build_flags = {"-Wno-volatile"}
cflags, cxxflags, link_flags = build_gen._flag_lists()
# The relative include is anchored at the build path
inc = f"-I{CORE.build_path / 'inc'}"
assert cflags == ["-DUSE_HOST", inc, "-g", "-std=gnu17"]
assert cxxflags == ["-std=gnu++20", "-DUSE_HOST", inc, "-g", "-Wno-volatile"]
assert link_flags == ["-Wl,-x", "-g", "-lssl"]
def test_flag_lists_without_cpp_standard_keeps_user_std() -> None:
CORE.cpp_standard = None
CORE.build_flags = {"-std=c++17"}
cflags, cxxflags, _link = build_gen._flag_lists()
assert cflags == []
assert cxxflags == ["-std=c++17"]
def test_flag_lists_apply_unflags(caplog: pytest.LogCaptureFixture) -> None:
CORE.build_flags = {"-g", "-DUSE_HOST"}
CORE.build_unflags = {"-g", "-Onope"}
with caplog.at_level(logging.WARNING):
cflags, cxxflags, link_flags = build_gen._flag_lists()
assert "-g" not in cflags + cxxflags + link_flags
assert "-DUSE_HOST" in cflags
assert "build_unflags entries matched no build flag: -Onope" in caplog.text
def test_resolve_host_libraries_without_libraries() -> None:
with patch("esphome.arduino.library.resolve_libraries") as resolve:
assert build_gen._resolve_host_libraries() == []
resolve.assert_not_called()
def test_resolve_host_libraries_is_framework_less() -> None:
CORE.add_library(Library(name="lvgl/lvgl", version="9.5.0"))
lib = ArduinoLibrary(name="lvgl")
with patch(
"esphome.arduino.library.resolve_libraries", return_value=[lib]
) as resolve:
assert build_gen._resolve_host_libraries() == [lib]
resolve.assert_called_once_with(
None,
pio_platform="native",
board_mcu="host",
cache_key="host",
framework=None,
manifest_optional=True,
)
def test_write_project_requires_generated_sources(tmp_path: Path) -> None:
with pytest.raises(EsphomeError, match="Generated source directory .* is missing"):
build_gen.write_project(COMPILERS, None)
_make_src(tmp_path, "esphome.h")
with pytest.raises(EsphomeError, match="No source files found"):
build_gen.write_project(COMPILERS, None)
def test_write_project_emits_every_source_kind(tmp_path: Path) -> None:
src = _make_src(tmp_path, "main.cpp", "esphome/core/a.c", "x.S", "y.s", "h.h")
CORE.build_flags = {"-DUSE_HOST", "-g"}
changed, ninja = _render(ccache="/usr/bin/ccache")
assert changed is True
# A compiler override's arguments follow the program
assert f"cc = {_q('/usr/bin/gcc')} -m32\n" in ninja
assert f"cxx = {_q('/usr/bin/g++')}" in ninja
assert f"ccache = {_q('/usr/bin/ccache')}" in ninja
assert f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')}" in ninja
assert (
f"build obj/src/esphome/core/a.c.o: c {_e(src / 'esphome/core/a.c')}" in ninja
)
assert "build obj/src/x.S.o: aspp " in ninja
assert "build obj/src/y.s.o: asm " in ninja
assert "h.h" not in ninja
# __FILE__ stays relative to the build path, as it was under PlatformIO,
# whether the source arrives absolute or rewritten by ccache
sep = os.sep
maps = (
f"{_q(f'-fmacro-prefix-map={tmp_path}{sep}=')} "
f"{_q(f'-fmacro-prefix-map=..{sep}..{sep}=')}"
)
assert f"cflags = -DUSE_HOST -g -I{_q(src)} {maps}\n" in ninja
assert f"cxxflags = -std=gnu++20 -DUSE_HOST -g -I{_q(src)} {maps}\n" in ninja
# Assembly gets the defines and includes only
assert f"asflags = -DUSE_HOST -I{_q(src)} {maps}\n" in ninja
assert "linkflags = -g\n" in ninja
assert "libdirflags = \n" in ninja
assert "libflags = \n" in ninja
assert "rule ar" not in ninja
assert (
f"build {PROGRAM_NAME}: link obj/src/esphome/core/a.c.o obj/src/main.cpp.o "
f"obj/src/x.S.o obj/src/y.s.o | \n archives = \ndefault {PROGRAM_NAME}\n"
) in ninja
# Unchanged content reports no change so the compile DB can be reused
changed, _ = _render(ccache="/usr/bin/ccache")
assert changed is False
def test_write_project_without_ccache(tmp_path: Path) -> None:
_make_src(tmp_path, "main.cpp")
_changed, ninja = _render()
assert "ccache = \n" in ninja
def test_write_project_routes_user_link_flags(tmp_path: Path) -> None:
_make_src(tmp_path, "main.cpp")
lib_dir = tmp_path / "opt" / "lib"
# Forward slashes: build flags are lexed like a POSIX shell line, which
# reads a backslash as an escape
CORE.build_flags = {
f"-L{lib_dir.as_posix()}",
"-lcrypto",
"-Wl,-framework,Security",
}
_changed, ninja = _render()
assert "linkflags = -Wl,-framework,Security\n" in ninja
assert f"libdirflags = -L{_q(lib_dir)}\n" in ninja
assert "libflags = -lcrypto\n" in ninja
def _libraries(tmp_path: Path) -> list[ArduinoLibrary]:
lib_dir = tmp_path / "libs"
for name in ("foo/src/a.cpp", "foo/src/sub/b.c", "bare/x.cpp"):
path = lib_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("")
archived = ArduinoLibrary(
name="foo",
sources=[lib_dir / "foo/src/a.cpp", lib_dir / "foo/src/sub/b.c"],
include_dirs=[lib_dir / "foo/src"],
flags=["-DFOO=1"],
link_dirs=[lib_dir / "foo/lib"],
link_libs=["bar"],
link_flags=["-Wl,--gc-sections"],
)
direct = ArduinoLibrary(
name="bare", sources=[lib_dir / "bare/x.cpp"], lib_archive=False
)
header_only = ArduinoLibrary(name="hdr", include_dirs=[lib_dir / "hdr"])
return [archived, direct, header_only]
def test_write_project_with_libraries(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
src = _make_src(tmp_path, "main.cpp")
libs = _libraries(tmp_path)
with (
patch.object(build_gen, "_resolve_host_libraries", return_value=libs),
patch.object(build_gen, "find_tool", return_value="/usr/bin/ar") as find_tool,
patch.object(build_gen.sys, "platform", "linux"),
caplog.at_level(logging.DEBUG),
):
_changed, ninja = _render()
find_tool.assert_called_once_with("AR", ("ar",))
assert (
f"rule ar\n command = $python $buildtool ar {_q('/usr/bin/ar')} $out $out.rsp"
in ninja
)
lib_dir = tmp_path / "libs"
# Every library's include dir joins the global include path
assert f"-I{_q(src)} -I{_q(lib_dir / 'foo/src')} -I{_q(lib_dir / 'hdr')}" in ninja
assert "linkflags = -Wl,--gc-sections\n" in ninja
assert f"libdirflags = -L{_q(lib_dir / 'foo/lib')}\n" in ninja
assert "libflags = -lbar\n" in ninja
# Library sources compile with the library's own flags, rooted at their
# common parent; its own include dirs lead the line so another library's
# header of the same name cannot shadow them
own = f"-I{_q(lib_dir / 'foo/src')}"
assert "$own_includes $cxxflags $flags" in ninja
assert (
f"build obj/lib/foo/a.cpp.o: cxx {_e(lib_dir / 'foo/src/a.cpp')}\n"
f" own_includes = {own}\n flags = -DFOO=1\n" in ninja
)
assert (
f"build obj/lib/foo/sub/b.c.o: c {_e(lib_dir / 'foo/src/sub/b.c')}\n"
f" own_includes = {own}\n flags = -DFOO=1\n" in ninja
)
assert "build libfoo.a: ar obj/lib/foo/a.cpp.o obj/lib/foo/sub/b.c.o\n" in ninja
# libArchive: false objects link directly; the archive is an order-only
# input wrapped in a group for GNU ld
assert (
f"build {PROGRAM_NAME}: link obj/src/main.cpp.o obj/lib/bare/x.cpp.o "
"| libfoo.a\n"
" archives = -Wl,--start-group libfoo.a -Wl,--end-group\n"
) in ninja
assert "Library hdr has no source files" in caplog.text
def test_write_project_darwin_links_archives_bare(tmp_path: Path) -> None:
_make_src(tmp_path, "main.cpp")
libs = _libraries(tmp_path)
with (
patch.object(build_gen, "_resolve_host_libraries", return_value=libs),
patch.object(build_gen, "find_tool", return_value="/usr/bin/ar"),
patch.object(build_gen.sys, "platform", "darwin"),
):
_changed, ninja = _render()
assert " archives = libfoo.a\n" in ninja
assert "--start-group" not in ninja
def test_build_unflags_remove_a_flag_with_its_argument(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Only the named pair goes; the same path under another flag stays."""
CORE.build_flags = {"-iquote inc", "-isystem inc", "-Xlinker -dead_strip", "-g"}
CORE.build_unflags = {"-isystem inc", "-Xlinker -dead_strip"}
with caplog.at_level(logging.WARNING):
cflags, _cxxflags, link_flags = build_gen._flag_lists()
assert cflags == ["-g", "-iquote", str(CORE.build_path / "inc")]
assert link_flags == ["-g"]
assert "matched no build flag" not in caplog.text
def test_build_unflags_warn_when_nothing_matches(
caplog: pytest.LogCaptureFixture,
) -> None:
CORE.build_flags = {"-g"}
CORE.build_unflags = {"-isystem inc", "-DNOPE"}
with caplog.at_level(logging.WARNING):
build_gen._flag_lists()
assert "matched no build flag: -DNOPE, -isystem " in caplog.text
def test_build_unflags_match_anchored_paths(caplog: pytest.LogCaptureFixture) -> None:
CORE.build_flags = {"-Iinc", "-g"}
CORE.build_unflags = {"-Iinc"}
with caplog.at_level(logging.WARNING):
cflags, _cxxflags, _link = build_gen._flag_lists()
assert cflags == ["-g"]
assert "matched no build flag" not in caplog.text
tests/unit_tests/build_helpers/test_ccache.py
+45
View File
@@ -120,3 +120,48 @@ def test_parse_enable_env_spelling_tables(
"""cv.boolean's spelling tables plus the 1/0 env convention."""
monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", raw)
assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected
def test_resolve_absolute_ccache_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Anchor the path: ninja runs from the build directory."""
monkeypatch.chdir(tmp_path)
with patch.object(ccache, "resolve_ccache_path", return_value="bin/ccache"):
resolved = ccache.resolve_absolute_ccache_path()
assert Path(resolved) == tmp_path / "bin" / "ccache"
with patch.object(ccache, "resolve_ccache_path", return_value=None):
assert ccache.resolve_absolute_ccache_path() is None
def test_ccache_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from esphome.core import CORE
CORE.build_path = tmp_path / "build"
spec = ("ESPHOME_TEST_PREFIX", "test")
monkeypatch.setenv("ESPHOME_TEST_PREFIX", str(tmp_path / "cache"))
monkeypatch.setenv("CCACHE_NOHASHDIR", "false")
for key in ("CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEPEND"):
monkeypatch.delenv(key, raising=False)
# None means resolved and disabled
assert ccache.ccache_env(None, spec) == {}
env = ccache.ccache_env("/usr/bin/ccache", spec)
# User-set values are respected; the rest get defaults
assert "CCACHE_NOHASHDIR" not in env
assert env["CCACHE_DEPEND"] == "1"
assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache")
assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve())
def test_ccache_defaults_env_escapes_a_dollar(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A literal $ is doubled, since ccache expands $VAR in its settings."""
from esphome.core import CORE
CORE.build_path = tmp_path / "a$b"
for key in ("CCACHE_DIR", "CCACHE_BASEDIR"):
monkeypatch.delenv(key, raising=False)
env = ccache.ccache_defaults_env(tmp_path / "c$d")
assert env["CCACHE_DIR"].endswith("c$$d")
assert env["CCACHE_BASEDIR"].endswith("a$$b")
tests/unit_tests/build_helpers/test_idedata.py
+33
View File
@@ -676,3 +676,36 @@ def test_load_or_build_idedata_cache_hit_skips_rebuild(tmp_path: Path) -> None:
)
mock_build.assert_not_called()
assert data["cached"] is True
def test_warn_if_idedata_missing_is_silent_on_success(
caplog: pytest.LogCaptureFixture,
) -> None:
with caplog.at_level(logging.WARNING):
idedata.warn_if_idedata_missing(lambda: {"cc_path": "gcc"})
assert "idedata" not in caplog.text
def test_warn_if_idedata_missing_warns_on_none(
caplog: pytest.LogCaptureFixture,
) -> None:
with caplog.at_level(logging.WARNING):
idedata.warn_if_idedata_missing(lambda: None)
assert "No idedata was generated for this build" in caplog.text
@pytest.mark.parametrize(
"error", [EsphomeError("db unusable"), ValueError("bad"), OSError("gone")]
)
def test_warn_if_idedata_missing_downgrades_failures(
caplog: pytest.LogCaptureFixture, error: Exception
) -> None:
"""The firmware already built; a broken idedata must not fail the build."""
def broken() -> dict | None:
raise error
with caplog.at_level(logging.DEBUG):
idedata.warn_if_idedata_missing(broken)
assert f"Could not generate idedata: {error}" in caplog.text
assert "Idedata failure detail" in caplog.text
tests/unit_tests/build_helpers/test_native.py
+34
View File
@@ -0,0 +1,34 @@
"""Tests for esphome.build_helpers.native."""
from __future__ import annotations
import logging
import pytest
from esphome.build_helpers import native
from esphome.const import Toolchain
from esphome.core import CORE
def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None:
"""Options a native build drops are warned by name; consumed ones stay quiet."""
CORE.toolchain = Toolchain.ARDUINO
CORE.platformio_options = {
"lib_ignore": ["x"],
"board_build.filesystem": "littlefs",
}
with caplog.at_level(logging.WARNING):
native.warn_ignored_platformio_options({"lib_ignore"})
assert "platformio_options->board_build.filesystem is ignored" in caplog.text
assert "native 'arduino' toolchain" in caplog.text
assert "lib_ignore" not in caplog.text
def test_warn_ignored_platformio_options_without_options(
caplog: pytest.LogCaptureFixture,
) -> None:
CORE.platformio_options = None
with caplog.at_level(logging.WARNING):
native.warn_ignored_platformio_options(())
assert caplog.text == ""
tests/unit_tests/build_helpers/test_ninja.py
+123
View File
@@ -141,3 +141,126 @@ def test_shell_token_windows_branch_uses_argv_rule() -> None:
with patch.object(os, "name", "nt"):
assert ninja_helper.shell_token("a b") == '"a b"'
assert ninja_helper.shell_token("", force=True) == '""'
@pytest.mark.parametrize("char", ["|", "\n", "\r"])
def test_escape_rejects_what_ninja_cannot_express(char: str) -> None:
with pytest.raises(EsphomeError, match="a ninja build file cannot express"):
ninja_helper.escape(f"src/a{char}b.cpp")
def test_escape_keeps_a_hash() -> None:
"""A # is only a comment at the start of a ninja line."""
assert ninja_helper.escape("src/a#b c.cpp") == "src/a#b$ c.cpp"
def test_absolute_tool_anchors_a_relative_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Anchor the path: ninja runs from the build directory."""
monkeypatch.chdir(tmp_path)
assert Path(ninja_helper.absolute_tool("bin/ninja")) == tmp_path / "bin" / "ninja"
def test_find_ninja_returns_an_absolute_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path)
with (
patch("shutil.which", return_value="bin/ninja"),
patch.object(ninja_helper, "_ninja_runs", return_value=True),
):
assert ninja_helper.find_ninja() == tmp_path / "bin" / "ninja"
@pytest.fixture
def compdb_dir(tmp_path: Path) -> Path:
build_dir = tmp_path / "build"
build_dir.mkdir()
(build_dir / "build.ninja").write_text("rule x\n")
return build_dir
def test_refresh_compile_commands_regenerates_when_stale(compdb_dir: Path) -> None:
ninja = Path("ninja")
compdb = compdb_dir / "compile_commands.json"
stamp = compdb_dir / ".compile_commands.stamp"
with patch.object(ninja_helper, "write_compile_commands") as write:
# A rewritten manifest always regenerates
ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, True)
assert write.call_count == 1
assert stamp.is_file()
# No compile DB yet
ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False)
assert write.call_count == 2
compdb.write_text("[]")
# Fresh stamp: nothing to do
ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False)
assert write.call_count == 2
# A manifest newer than the stamp (interrupted previous run)
os.utime(stamp, (1, 1))
ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False)
assert write.call_count == 3
# A missing stamp regenerates too
stamp.unlink()
ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False)
assert write.call_count == 4
def _completed(rc: int = 0, stdout: str = "", stderr: str = "") -> MagicMock:
return MagicMock(returncode=rc, stdout=stdout, stderr=stderr)
def test_write_compile_commands_success(compdb_dir: Path) -> None:
entries = '[{"file": "a.cpp", "command": "g++ -c a.cpp"}]'
with patch("subprocess.run", return_value=_completed(stdout=entries)) as run:
ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {"A": "1"})
assert (compdb_dir / "compile_commands.json").read_text() == entries
# Every compile rule, so preprocessed assembly reaches the database too
assert run.call_args.args[0] == [
"ninja",
"-C",
str(compdb_dir),
"-t",
"compdb",
"c",
"cxx",
"aspp",
"asm",
]
assert run.call_args.kwargs["env"] == {"A": "1"}
@pytest.mark.parametrize(
("result", "message"),
[
(_completed(rc=1, stderr="boom"), "Could not generate compile_commands.json"),
# A parse failure names its cause, not the rule-name story
(_completed(stdout="not json"), "unparsable compile database.*not json"),
(_completed(stdout="[]"), "empty compile database"),
],
)
def test_write_compile_commands_failures_drop_stale_db(
compdb_dir: Path, result: MagicMock, message: str
) -> None:
compdb = compdb_dir / "compile_commands.json"
compdb.write_text("[stale]")
with (
patch("subprocess.run", return_value=result),
pytest.raises(EsphomeError, match=message),
):
ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {})
assert not compdb.exists()
def test_write_compile_commands_keeps_the_mtime_of_an_unchanged_db(
compdb_dir: Path,
) -> None:
"""The idedata cache is keyed on the DB's mtime."""
entries = '[{"file": "a.cpp", "command": "g++ -c a.cpp"}]'
compdb = compdb_dir / "compile_commands.json"
compdb.write_text(entries)
os.utime(compdb, (1000, 1000))
with patch("subprocess.run", return_value=_completed(stdout=entries)):
ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {})
assert compdb.stat().st_mtime == 1000
tests/unit_tests/build_helpers/test_ninja_gen.py
+57
View File
@@ -0,0 +1,57 @@
"""Tests for esphome.build_helpers.ninja_gen."""
from __future__ import annotations
import os
from pathlib import Path
import pytest
from esphome.build_helpers import ninja_gen
from esphome.build_helpers.ninja_gen import Flag
@pytest.mark.parametrize(
("flag", "anchored"),
[
(("-Iinc",), ("-I{base}/inc",)),
(("-Llib",), ("-L{base}/lib",)),
(("-include", "pre.h"), ("-include", "{base}/pre.h")),
(("-isystem", "sys"), ("-isystem", "{base}/sys")),
# The glued spelling of a flag that takes a path
(("-isystemsys",), ("-isystem{base}/sys",)),
(("-includepre.h",), ("-include{base}/pre.h",)),
# An absolute operand is never changed
(("-I{base}/abs",), ("-I{base}/abs",)),
(("-include", "{base}/abs.h"), ("-include", "{base}/abs.h")),
# Not a path
(("-DUSE_HOST",), ("-DUSE_HOST",)),
(("-lssl",), ("-lssl",)),
(("-framework", "Cocoa"), ("-framework", "Cocoa")),
(("-I",), ("-I",)),
],
)
def test_anchor_path_flag(tmp_path: Path, flag: Flag, anchored: Flag) -> None:
"""Relative operands resolve from the build path, as under PlatformIO."""
def fill(tokens: Flag) -> Flag:
return tuple(
str(Path(tok.replace("{base}", str(tmp_path))))
if "{base}" in tok and not tok.startswith("-")
else tok.replace("{base}/", f"{tmp_path}{os.sep}")
for tok in tokens
)
assert ninja_gen.anchor_path_flag(fill(flag), tmp_path) == fill(anchored)
def test_collect_sources_skips_excluded_and_other_files(tmp_path: Path) -> None:
for name in ("b.cpp", "a.c", "skip.cpp", "notes.txt", "sub/c.S"):
path = tmp_path / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("")
assert ninja_gen.collect_sources(tmp_path, {"skip.cpp"}) == [
tmp_path / "a.c",
tmp_path / "b.cpp",
tmp_path / "sub" / "c.S",
]
tests/unit_tests/components/host/__init__.py
View File
tests/unit_tests/components/host/test_init.py
+86
View File
@@ -0,0 +1,86 @@
"""Tests for the host platform's native-toolchain wiring."""
from __future__ import annotations
import asyncio
from unittest.mock import patch
import pytest
from esphome.components import host
import esphome.config_validation as cv
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
PLATFORM_HOST,
Toolchain,
)
from esphome.core import CORE, EsphomeError
@pytest.fixture(autouse=True)
def _core_data() -> None:
CORE.data[KEY_CORE] = {}
def test_schema_resolves_the_host_toolchain() -> None:
"""The host serves exactly one toolchain; it is picked without config."""
config = host.CONFIG_SCHEMA({})
assert CORE.toolchain is Toolchain.HOST
assert CORE.using_native_toolchain
assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == PLATFORM_HOST
assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "host"
assert "mac_address" in config
def test_schema_rejects_the_platformio_toolchain() -> None:
"""There is no PlatformIO backend left to honor --toolchain platformio."""
CORE.toolchain = Toolchain.PLATFORMIO
with pytest.raises(cv.Invalid, match="Unsupported toolchain 'platformio' for host"):
host.CONFIG_SCHEMA({})
def test_to_code_configures_the_native_build() -> None:
config = host.CONFIG_SCHEMA({"mac_address": "AA:BB:CC:DD:EE:FF"})
asyncio.run(host.to_code(config))
assert "-DUSE_HOST" in CORE.build_flags
# The standard travels through cpp_standard, not a -std= build flag,
# so the generator can keep it off the C compile line
assert CORE.cpp_standard == "gnu++20"
assert not any(flag.startswith("-std=") for flag in CORE.build_flags)
assert not CORE.platformio_options
define_names = {define.name for define in CORE.defines}
assert {"ESPHOME_BOARD", "ESPHOME_VARIANT", "USE_ESPHOME_HOST_MAC_ADDRESS"} <= (
define_names
)
def test_run_compile_hook_claims_the_build() -> None:
config = {"esphome": {}}
with (
patch.object(host.sys, "platform", "linux"),
patch("esphome.host.toolchain.run_compile", return_value=0) as run,
):
assert host.run_compile(object(), config) is True
run.assert_called_once_with(config, CORE.verbose)
def test_run_compile_hook_raises_on_failure() -> None:
with (
patch.object(host.sys, "platform", "linux"),
patch("esphome.host.toolchain.run_compile", return_value=2),
pytest.raises(EsphomeError, match="Host build failed"),
):
host.run_compile(object(), {"esphome": {}})
def test_run_compile_hook_refuses_windows() -> None:
"""The host sources use POSIX interfaces; fail before building, by name."""
with (
patch.object(host.sys, "platform", "win32"),
patch("esphome.host.toolchain.run_compile") as run,
pytest.raises(EsphomeError, match="does not build on Windows"),
):
host.run_compile(object(), {"esphome": {}})
run.assert_not_called()
tests/unit_tests/test_arduino8266_framework.py
+11 -20
View File
@@ -112,17 +112,6 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None:
assert env["CCACHE_DIR"] == "x"
def test_ccache_env(tmp_path: Path) -> None:
assert framework.ccache_env(None) == {}
with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True):
env = framework.ccache_env("/usr/bin/ccache")
# User-set values are respected; the rest get defaults
assert "CCACHE_NOHASHDIR" not in env
assert env["CCACHE_DEPEND"] == "1"
assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve())
assert env["CCACHE_DIR"].endswith("ccache")
def test_check_and_install_rejects_old_core(tmp_path: Path) -> None:
"""Calling the installer below the floor fails before any download."""
with pytest.raises(EsphomeError, match=">= 3.1.1"):
@@ -148,18 +137,20 @@ def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None:
assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"]
def test_ccache_env_accepts_a_preresolved_path() -> None:
"""The caller resolves ccache once and threads it through; None means
resolved-and-disabled."""
with patch.dict(os.environ, {}, clear=True):
assert framework.ccache_env(None) == {}
env = framework.ccache_env("/usr/bin/ccache")
assert env["CCACHE_DIR"].endswith("ccache")
def test_toolchain_tool_layout(tmp_path: Path) -> None:
"""One owner for the bin/xtensa-lx106-elf-<name> layout."""
tool = framework.toolchain_tool(tmp_path, "addr2line")
assert tool.parent == tmp_path / "bin"
assert tool.name.startswith("xtensa-lx106-elf-addr2line")
assert (tool.suffix == ".exe") is (os.name == "nt")
def test_get_build_env_uses_the_arduino8266_ccache_dir(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("ESPHOME_ARDUINO8266_PREFIX", str(tmp_path / "cache"))
monkeypatch.delenv("CCACHE_DIR", raising=False)
env = framework.get_build_env(tmp_path / "toolchain", "/usr/bin/ccache")
assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache")
# None means resolved and disabled
assert "CCACHE_DIR" not in framework.get_build_env(tmp_path / "toolchain", None)
tests/unit_tests/test_arduino8266_toolchain.py
+56 -152
View File
@@ -16,6 +16,7 @@ from esphome.const import (
CONF_ESPHOME,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
Toolchain,
)
from esphome.core import CORE, EsphomeError
@@ -80,7 +81,7 @@ def test_run_compile_build_failure(tmp_path: Path) -> None:
patch.object(
toolchain.subprocess, "run", return_value=MagicMock(returncode=2)
) as mock_run,
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "refresh_compile_commands") as mock_compdb,
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=True) == 2
cmd = mock_run.call_args[0][0]
@@ -100,7 +101,7 @@ def test_run_compile_success(tmp_path: Path) -> None:
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
) as mock_run,
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "refresh_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary") as mock_size,
patch.object(toolchain, "get_idedata") as mock_idedata,
):
@@ -116,39 +117,12 @@ def test_run_compile_success(tmp_path: Path) -> None:
# cwd, not -C, so ninja prints no "Entering directory" banner
assert "-C" not in cmd
assert call[1]["cwd"] is not None
mock_compdb.assert_called_once()
# An unchanged manifest is passed on, so the shared refresh can skip
assert mock_compdb.call_args.args[3] is False
mock_size.assert_called_once()
mock_idedata.assert_called_once()
def test_run_compile_regenerates_stale_compdb(tmp_path: Path) -> None:
"""An interrupted run can leave build.ninja newer than the compile DB;
mere existence must not skip regeneration."""
build_dir = toolchain.get_build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
(build_dir / "build.ninja").write_text("")
compdb = build_dir / "compile_commands.json"
compdb.write_text("[]")
os.utime(compdb, ((build_dir / "build.ninja").stat().st_mtime - 5,) * 2)
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(
returncode=0, stdout="ninja: no work to do.\n", stderr=""
),
),
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
mock_compdb.assert_called_once()
def test_run_compile_missing_artifact_fails(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
@@ -164,7 +138,7 @@ def test_run_compile_missing_artifact_fails(
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "refresh_compile_commands"),
patch.object(toolchain, "_print_size_summary") as mock_size,
patch.object(toolchain, "get_idedata"),
):
@@ -188,7 +162,7 @@ def test_run_compile_warns_when_idedata_fails(
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "refresh_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata", return_value=None),
):
@@ -196,63 +170,6 @@ def test_run_compile_warns_when_idedata_fails(
assert "No idedata was generated for this build" in caplog.text
def test_write_compile_commands(tmp_path: Path) -> None:
build_dir = tmp_path / "build"
build_dir.mkdir()
entries = '[{"file": "a.cpp", "command": "cc"}]\n'
with patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=entries),
):
toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {})
assert (build_dir / "compile_commands.json").read_text() == entries
@pytest.mark.parametrize(
("stdout", "match"),
[
("[]\n", "empty compile database"),
# A parse failure names its cause, not the rule-name story
("not json", "unparsable compile database.*not json"),
],
)
def test_write_compile_commands_bad_db_raises(
tmp_path: Path, stdout: str, match: str
) -> None:
"""An empty or unparsable compile database fails the build with its
actual cause and drops any stale database."""
build_dir = tmp_path / "build"
build_dir.mkdir()
(build_dir / "compile_commands.json").write_text("[stale]")
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=stdout),
),
pytest.raises(EsphomeError, match=match),
):
toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {})
assert not (build_dir / "compile_commands.json").exists()
def test_write_compile_commands_failure_removes_stale_db(tmp_path: Path) -> None:
"""A failed compdb run must not leave a stale database behind."""
stale = tmp_path / "compile_commands.json"
stale.write_text("[]")
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=1, stderr="boom"),
),
pytest.raises(EsphomeError, match="compile_commands"),
):
toolchain._write_compile_commands(tmp_path / "ninja", tmp_path, {})
assert not stale.exists()
def test_parse_app_size(tmp_path: Path) -> None:
ld = tmp_path / "eagle.flash.4m.ld"
ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0xfeff0\n}\n")
@@ -342,7 +259,9 @@ def test_get_idedata_delegates(tmp_path: Path) -> None:
"esphome.build_helpers.idedata.load_or_build_idedata",
return_value={"cc_path": "x"},
) as mock_load,
patch.object(toolchain, "resolve_ccache_path", return_value="/cc/ccache"),
patch.object(
toolchain, "resolve_absolute_ccache_path", return_value="/cc/ccache"
),
):
assert toolchain.get_idedata() == {"cc_path": "x"}
compile_commands, elf, cache = mock_load.call_args[0]
@@ -359,43 +278,12 @@ def test_get_idedata_no_ccache(tmp_path: Path) -> None:
patch(
"esphome.build_helpers.idedata.load_or_build_idedata", return_value={}
) as mock_load,
patch.object(toolchain, "resolve_ccache_path", return_value=None),
patch.object(toolchain, "resolve_absolute_ccache_path", return_value=None),
):
toolchain.get_idedata()
assert mock_load.call_args.kwargs["launcher"] is None
def test_run_compile_skips_compdb_when_ninja_unchanged(tmp_path: Path) -> None:
"""An unchanged build.ninja means the compile DB is already current."""
build_dir = toolchain.get_build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
# write_project (stubbed below) always leaves a build.ninja behind
(build_dir / "build.ninja").write_text("# manifest")
def run(regenerate_expected: bool) -> None:
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert mock_compdb.called == regenerate_expected
# Missing compile DB: regenerated even though build.ninja is unchanged
run(regenerate_expected=True)
# Present compile DB + unchanged build.ninja: skipped
(build_dir / "compile_commands.json").write_text("[]")
run(regenerate_expected=False)
def test_print_size_summary_unparsable_section(
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
@@ -447,32 +335,6 @@ def test_print_size_summary_missing_section_skips_summary(
assert "missing section(s) .bss" in caplog.text
def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None:
"""Component-added options the native build drops are warned by name;
the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags,
flash_mode) stay quiet."""
CORE.platformio_options = {
"board_build.ldscript": "eagle.flash.4m2m.ld",
"board_build.f_cpu": "160000000L",
"board_build.filesystem": "littlefs",
"board_build.flash_mode": "dio",
"build_src_flags": "-include throw_stubs.h",
"lib_ignore": ["Updater"],
"upload_speed": "460800",
}
toolchain._warn_ignored_platformio_options()
assert "platformio_options->board_build.filesystem is ignored" in caplog.text
assert "native 'arduino' toolchain" in caplog.text
assert "board_build.ldscript is ignored" not in caplog.text
assert "board_build.f_cpu is ignored" not in caplog.text
assert "lib_ignore" not in caplog.text
assert "build_src_flags" not in caplog.text
assert "flash_mode" not in caplog.text
# Component-added upload_speed never gets read under the native
# toolchain, so it must warn
assert "upload_speed" in caplog.text
def test_run_compile_idedata_error_does_not_fail_build(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
@@ -486,7 +348,7 @@ def test_run_compile_idedata_error_does_not_fail_build(
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "refresh_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(
toolchain,
@@ -512,7 +374,7 @@ def test_run_compile_skipped_size_summary_names_consequence(
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "refresh_compile_commands"),
patch.object(toolchain, "_print_size_summary", return_value=False),
patch.object(toolchain, "get_idedata", return_value=None),
):
@@ -541,8 +403,50 @@ def test_get_idedata_accepts_preresolved_ccache() -> None:
"esphome.build_helpers.idedata.load_or_build_idedata",
return_value={"ok": True},
) as mock_build,
patch.object(toolchain, "resolve_ccache_path") as mock_resolve,
patch.object(toolchain, "resolve_absolute_ccache_path") as mock_resolve,
):
assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True}
mock_resolve.assert_not_called()
assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache"
def test_run_compile_warns_about_dropped_platformio_options(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""Component-added options the native build drops are warned by name;
the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags,
flash_mode) stay quiet."""
CORE.toolchain = Toolchain.ARDUINO
CORE.platformio_options = {
"board_build.ldscript": "eagle.flash.4m2m.ld",
"board_build.f_cpu": "160000000L",
"board_build.filesystem": "littlefs",
"board_build.flash_mode": "dio",
"build_src_flags": "-include throw_stubs.h",
"lib_ignore": ["Updater"],
"upload_speed": "460800",
}
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=True),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "refresh_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert "platformio_options->board_build.filesystem is ignored" in caplog.text
assert "platformio_options->upload_speed is ignored" in caplog.text
assert "native 'arduino' toolchain" in caplog.text
assert "board_build.ldscript is ignored" not in caplog.text
assert "board_build.f_cpu is ignored" not in caplog.text
assert "lib_ignore" not in caplog.text
assert "build_src_flags" not in caplog.text
assert "flash_mode" not in caplog.text
# A rewritten manifest is passed on, so the compile DB is regenerated
assert mock_compdb.call_args.args[3] is True
tests/unit_tests/test_arduino_library.py
+26
View File
@@ -1160,3 +1160,29 @@ def test_versionless_dependency_with_provider_stays_quiet(
libs = _resolve(framework)
assert "Wire" in [lib.name for lib in libs]
assert "has no version to resolve" not in caplog.text
def test_resolve_libraries_without_framework_is_all_external() -> None:
"""The host build has no core tree: nothing is bundled, the framework
check is skipped, and every name reaches the converter."""
_add_library("Wire", None)
_add_library("lvgl/lvgl", "9.5.0")
def fake_convert(libraries: list, backend: LibraryBackend) -> list:
assert backend.platform == "native"
assert backend.framework is None
assert backend.cache_key == "host"
assert backend.provides("Wire") is False
return []
with patch.object(component, "convert_libraries", side_effect=fake_convert) as mock:
libs = component.resolve_libraries(
None,
pio_platform="native",
board_mcu="host",
cache_key="host",
framework=None,
)
assert libs == []
(libraries, _backend), _ = mock.call_args
assert [lib.name for lib in libraries] == ["Wire", "lvgl/lvgl"]
tests/unit_tests/test_core.py
+8
View File
@@ -976,6 +976,14 @@ class TestEsphomeCore:
target.toolchain = const.Toolchain.SDK_NRF
assert target.using_native_toolchain is False
def test_using_toolchain_host(self, target):
"""The host's only toolchain: the system compiler driven by ninja."""
target.toolchain = const.Toolchain.HOST
assert target.using_toolchain_host is True
assert target.using_native_toolchain is True
target.toolchain = const.Toolchain.PLATFORMIO
assert target.using_toolchain_host is False
def test_add_library__extracts_short_name_from_path(self, target):
"""Test add_library extracts short name from library paths like owner/lib."""
target.data[const.KEY_CORE] = {
tests/unit_tests/test_host_toolchain.py
+329
View File
@@ -0,0 +1,329 @@
"""Tests for esphome.host.toolchain (the native host build driver)."""
from __future__ import annotations
from collections.abc import Generator
import logging
import os
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
CONF_ESPHOME,
KEY_CORE,
KEY_TARGET_PLATFORM,
PLATFORM_HOST,
Toolchain,
)
from esphome.core import CORE, EsphomeError
from esphome.host import toolchain
from esphome.host.toolchain import PROGRAM_NAME
@pytest.fixture(autouse=True)
def _core(tmp_path: Path) -> None:
CORE.config_path = tmp_path / "dev.yaml"
CORE.build_path = tmp_path
CORE.name = "dev"
CORE.toolchain = Toolchain.HOST
CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_HOST}
def _abs(path: str) -> str:
"""What the toolchain makes of a tool path (a drive is added on Windows)."""
return str(Path(path).absolute())
def test_find_command_prefers_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("CXX", "/opt/clang++")
with patch("shutil.which", side_effect={"/opt/clang++": "/opt/clang++"}.get):
assert toolchain.find_command("CXX", ("g++",)) == (_abs("/opt/clang++"),)
def test_find_command_keeps_the_override_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``CC="gcc -m32"`` is a command, as make and CMake read it."""
monkeypatch.setenv("CC", "gcc -m32 '--sysroot=/opt/my sdk'")
with patch("shutil.which", side_effect={"gcc": "/usr/bin/gcc"}.get):
assert toolchain.find_command("CC", ("cc",)) == (
_abs("/usr/bin/gcc"),
"-m32",
"--sysroot=/opt/my sdk",
)
@pytest.mark.parametrize(
("override", "expected_args"),
[("ccache gcc", ()), ("/opt/bin/ccache gcc -m32", ("-m32",))],
)
def test_find_command_drops_a_ccache_prefix(
monkeypatch: pytest.MonkeyPatch, override: str, expected_args: tuple[str, ...]
) -> None:
"""The build adds ccache itself; the compiler is the word after it."""
monkeypatch.setenv("CC", override)
with patch("shutil.which", side_effect={"gcc": "/usr/bin/gcc"}.get):
assert toolchain.find_command("CC", ("cc",)) == (
_abs("/usr/bin/gcc"),
*expected_args,
)
def test_find_command_accepts_a_compiler_named_ccache(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Alone, the word is the program: ccache's compiler links work that way."""
monkeypatch.setenv("CC", "ccache")
with patch("shutil.which", side_effect={"ccache": "/usr/bin/ccache"}.get):
assert toolchain.find_command("CC", ("cc",)) == (_abs("/usr/bin/ccache"),)
def test_find_command_env_override_must_run(monkeypatch: pytest.MonkeyPatch) -> None:
"""A broken override fails by name instead of silently using another compiler."""
monkeypatch.setenv("CXX", "nope++")
with (
patch("shutil.which", return_value=None),
pytest.raises(EsphomeError, match="CXX='nope\\+\\+' does not name"),
):
toolchain.find_command("CXX", ("g++",))
def test_find_command_blank_override_is_unset(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("CC", " ")
with patch("shutil.which", side_effect={"clang": "/usr/bin/clang"}.get):
assert toolchain.find_command("CC", ("gcc", "clang")) == (
_abs("/usr/bin/clang"),
)
def test_find_command_first_candidate_wins(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CC", raising=False)
table = {"gcc": "/usr/bin/gcc", "clang": "/usr/bin/clang"}
with patch("shutil.which", side_effect=table.get):
assert toolchain.find_command("CC", ("gcc", "clang")) == (_abs("/usr/bin/gcc"),)
def test_find_command_none_found(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CC", raising=False)
with (
patch("shutil.which", return_value=None),
pytest.raises(
EsphomeError,
match=r"gcc not found on PATH \(tried gcc, clang\); install it or set CC",
),
):
toolchain.find_command("CC", ("gcc", "clang"))
def test_find_command_makes_a_relative_hit_absolute(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Anchor the path: ninja runs from the build directory."""
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("CC", "./toolchain/gcc")
with patch("shutil.which", return_value="./toolchain/gcc"):
(found,) = toolchain.find_command("CC", ("gcc",))
assert Path(found) == Path("toolchain/gcc").absolute()
def test_find_tool_refuses_arguments(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AR", "ar --plugin x")
with (
patch("shutil.which", return_value="/usr/bin/ar"),
pytest.raises(EsphomeError, match="AR must name a program without arguments"),
):
toolchain.find_tool("AR", ("ar",))
def test_find_compilers(monkeypatch: pytest.MonkeyPatch) -> None:
for var in ("CC", "CXX"):
monkeypatch.delenv(var, raising=False)
table = {"gcc": "/usr/bin/gcc", "g++": "/usr/bin/g++"}
with patch("shutil.which", side_effect=table.get):
assert toolchain.find_compilers() == toolchain.HostCompilers(
cc=(_abs("/usr/bin/gcc"),), cxx=(_abs("/usr/bin/g++"),)
)
def test_build_paths(tmp_path: Path) -> None:
"""The PlatformIO layout is kept: CORE.firmware_bin resolves the same file."""
assert toolchain.get_build_dir() == tmp_path / ".pioenvs" / "dev"
assert toolchain.get_elf_path() == CORE.firmware_bin
def test_binutils_paths(monkeypatch: pytest.MonkeyPatch) -> None:
for var in ("OBJDUMP", "READELF"):
monkeypatch.delenv(var, raising=False)
table = {"objdump": "/usr/bin/objdump", "readelf": "/usr/bin/readelf"}
with patch("shutil.which", side_effect=table.get):
assert toolchain.get_objdump_path() == Path(_abs("/usr/bin/objdump"))
assert toolchain.get_readelf_path() == Path(_abs("/usr/bin/readelf"))
def test_get_build_env_merges_without_leaking(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("ESPHOME_HOST_PREFIX", str(tmp_path / "cache"))
monkeypatch.delenv("CCACHE_DIR", raising=False)
monkeypatch.setenv("KEEP_ME", "1")
env = toolchain.get_build_env("/usr/bin/ccache")
assert env["KEEP_ME"] == "1"
assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache")
assert "CCACHE_DIR" not in os.environ
def test_get_build_env_without_ccache(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CCACHE_DIR", raising=False)
assert "CCACHE_DIR" not in toolchain.get_build_env(None)
@pytest.fixture
def compile_env(tmp_path: Path) -> Generator[SimpleNamespace]:
"""Stub everything run_compile resolves; the build dir holds a manifest."""
build_dir = tmp_path / ".pioenvs" / "dev"
build_dir.mkdir(parents=True)
(build_dir / "build.ninja").write_text("rule x\n")
compilers = toolchain.HostCompilers(("gcc",), ("g++",))
with (
patch.object(toolchain, "find_ninja", return_value=Path("/usr/bin/ninja")),
patch.object(toolchain, "find_compilers", return_value=compilers),
patch.object(toolchain, "resolve_absolute_ccache_path", return_value=None),
patch("esphome.build_gen.host.write_project", return_value=True) as project,
patch.object(toolchain, "refresh_compile_commands") as refresh,
patch.object(
toolchain, "_load_idedata", return_value={"cc_path": "gcc"}
) as ide,
patch("subprocess.run") as run,
):
yield SimpleNamespace(
build_dir=build_dir,
compilers=compilers,
write_project=project,
refresh=refresh,
idedata=ide,
run=run,
)
def _completed(rc: int = 0) -> SimpleNamespace:
return SimpleNamespace(returncode=rc, stdout="", stderr="")
def test_run_compile_builds_and_reports_success(compile_env: SimpleNamespace) -> None:
elf = compile_env.build_dir / PROGRAM_NAME
def build(cmd: list[str], **kwargs: object) -> SimpleNamespace:
elf.write_text("")
return _completed()
compile_env.run.side_effect = build
config = {CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 4}}
assert toolchain.run_compile(config, verbose=True) == 0
compile_env.write_project.assert_called_once_with(compile_env.compilers, None)
compile_env.refresh.assert_called_once()
assert compile_env.refresh.call_args.args[3] is True
compile_env.run.assert_called_once()
assert compile_env.run.call_args.args[0] == [
str(Path("/usr/bin/ninja")),
"-v",
"-j",
"4",
PROGRAM_NAME,
]
assert compile_env.run.call_args.kwargs["cwd"] == compile_env.build_dir
compile_env.idedata.assert_called_once_with(None)
def test_run_compile_defaults(compile_env: SimpleNamespace) -> None:
"""No verbosity and no process limit: just the ninja target."""
(compile_env.build_dir / PROGRAM_NAME).write_text("")
compile_env.run.return_value = _completed()
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert compile_env.run.call_args.args[0] == [
str(Path("/usr/bin/ninja")),
PROGRAM_NAME,
]
# The build inherits stdio and never captures: progress must stream
kwargs = compile_env.run.call_args.kwargs
assert kwargs["check"] is False
assert kwargs["close_fds"] is False
assert "capture_output" not in kwargs
def test_run_compile_warns_about_dropped_platformio_options(
compile_env: SimpleNamespace, caplog: pytest.LogCaptureFixture
) -> None:
(compile_env.build_dir / PROGRAM_NAME).write_text("")
compile_env.run.return_value = _completed()
CORE.platformio_options = {"lib_ignore": ["x"], "board_build.f_cpu": "1"}
with caplog.at_level(logging.WARNING):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert "platformio_options->board_build.f_cpu is ignored" in caplog.text
assert "native 'host' toolchain" in caplog.text
assert "lib_ignore" not in caplog.text
def test_run_compile_passes_the_resolved_ccache(compile_env: SimpleNamespace) -> None:
(compile_env.build_dir / PROGRAM_NAME).write_text("")
compile_env.run.return_value = _completed()
with patch.object(
toolchain, "resolve_absolute_ccache_path", return_value="/usr/bin/ccache"
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
compile_env.write_project.assert_called_once_with(
compile_env.compilers, "/usr/bin/ccache"
)
compile_env.idedata.assert_called_once_with("/usr/bin/ccache")
def test_run_compile_build_failure_returns_code(compile_env: SimpleNamespace) -> None:
compile_env.run.return_value = _completed(rc=3)
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 3
compile_env.idedata.assert_not_called()
def test_run_compile_missing_program_fails(
compile_env: SimpleNamespace, caplog: pytest.LogCaptureFixture
) -> None:
"""A green ninja run that produced no program fails by name."""
compile_env.run.return_value = _completed()
with caplog.at_level(logging.ERROR):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 1
assert "Build produced no" in caplog.text
compile_env.idedata.assert_not_called()
@pytest.mark.parametrize("ccache", ["/usr/bin/ccache", None])
def test_get_idedata_resolves_ccache(tmp_path: Path, ccache: str | None) -> None:
with (
patch.object(toolchain, "resolve_absolute_ccache_path", return_value=ccache),
patch(
"esphome.build_helpers.idedata.load_or_build_idedata", return_value={"x": 1}
) as load,
):
assert toolchain.get_idedata() == {"x": 1}
load.assert_called_once_with(
tmp_path / ".pioenvs" / "dev" / "compile_commands.json",
tmp_path / ".pioenvs" / "dev" / PROGRAM_NAME,
CORE.relative_internal_path("idedata", "dev.json"),
launcher=ccache,
)
@pytest.mark.parametrize("platform", ["darwin", "win32"])
def test_check_analysis_supported_refuses_non_elf(platform: str) -> None:
with (
patch.object(toolchain.sys, "platform", platform),
pytest.raises(EsphomeError, match=f"the host build on {platform}"),
):
toolchain.check_analysis_supported()
def test_check_analysis_supported_accepts_linux() -> None:
with patch.object(toolchain.sys, "platform", "linux"):
toolchain.check_analysis_supported()
tests/unit_tests/test_main.py
+183 -29
View File
@@ -7805,52 +7805,206 @@ async def test_wrap_to_code_comment_is_insertion_order_independent() -> None:
assert second.index("a: 2") < second.index("z: 1")
def test_host_program_path_platformio_toolchain() -> None:
"""Host + PlatformIO toolchain reads the memoized idedata path."""
setup_core(platform=PLATFORM_HOST)
idedata = SimpleNamespace(firmware_elf_path="/build/x/.pioenvs/x/program")
with patch(
"esphome.platformio.toolchain.get_idedata", return_value=idedata
) as mock_get:
assert main._host_program_path({}) == "/build/x/.pioenvs/x/program"
mock_get.assert_called_once_with({})
def test_host_program_path_esp_idf_toolchain() -> None:
"""Host + native ESP-IDF toolchain asks the espidf toolchain for the ELF."""
setup_core(platform=PLATFORM_HOST)
CORE.toolchain = Toolchain.ESP_IDF
with patch(
"esphome.espidf.toolchain.get_elf_path", return_value=Path("/b/app.elf")
):
assert main._host_program_path({}) == str(Path("/b/app.elf"))
def test_command_compile_host_logs_program_path(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
"""command_compile on host logs the compiled program path."""
setup_core(platform=PLATFORM_HOST)
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
with (
patch.object(main, "write_cpp", return_value=0),
patch.object(main, "compile_program", return_value=0),
patch.object(main, "_host_program_path", return_value="/b/program"),
caplog.at_level(logging.INFO),
):
assert main.command_compile(SimpleNamespace(only_generate=False), {}) == 0
assert "Successfully compiled program to path '/b/program'" in caplog.text
assert f"Successfully compiled program to path '{CORE.firmware_bin}'" in caplog.text
def test_command_run_host_executes_program(caplog: pytest.LogCaptureFixture) -> None:
def test_command_run_host_executes_program(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""command_run on host logs and executes the compiled program directly."""
setup_core(platform=PLATFORM_HOST)
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
with (
patch.object(main, "write_cpp", return_value=0),
patch.object(main, "compile_program", return_value=0),
patch.object(main, "_host_program_path", return_value="/b/program"),
patch.object(main, "run_external_process", return_value=0) as mock_run,
caplog.at_level(logging.INFO),
):
assert main.command_run(SimpleNamespace(), {}) == 0
mock_run.assert_called_with("/b/program")
assert "Running program from path '/b/program'" in caplog.text
program = str(CORE.firmware_bin)
mock_run.assert_called_with(program)
assert f"Running program from path '{program}'" in caplog.text
def test_write_cpp_file_project_generation_follows_host_toolchain() -> None:
"""Only PlatformIO gets a platformio.ini; ESP-IDF writes its CMake project
here, and the other native builds generate theirs at compile time."""
setup_core(platform=PLATFORM_HOST)
with (
patch("esphome.writer.write_cpp"),
patch("esphome.build_gen.platformio.write_project") as mock_pio,
patch("esphome.build_gen.espidf.write_project") as mock_idf,
):
CORE.toolchain = Toolchain.HOST
assert main.write_cpp_file() == 0
mock_pio.assert_not_called()
mock_idf.assert_not_called()
CORE.toolchain = Toolchain.PLATFORMIO
assert main.write_cpp_file() == 0
mock_pio.assert_called_once()
CORE.toolchain = Toolchain.ESP_IDF
assert main.write_cpp_file() == 0
mock_idf.assert_called_once()
def test_compile_program_host_uses_the_platform_hook() -> None:
"""The host component's run_compile claims the build."""
setup_core(platform=PLATFORM_HOST)
CORE.toolchain = Toolchain.HOST
with (
patch("esphome.components.host.run_compile", return_value=True) as hook,
patch("esphome.__main__._check_and_emit_build_info") as build_info,
):
assert compile_program(MagicMock(), {}) == 0
hook.assert_called_once()
build_info.assert_called_once()
def test_compile_program_native_toolchain_needs_a_backend() -> None:
"""A native toolchain no hook claims must not fall through to PlatformIO."""
setup_core(platform=PLATFORM_HOST)
CORE.toolchain = Toolchain.HOST
with (
patch("esphome.platformio.toolchain.run_compile") as mock_pio,
# A platform package without a run_compile hook
patch.dict(sys.modules, {"esphome.components.host": SimpleNamespace()}),
pytest.raises(EsphomeError, match="no platform backend claimed the build"),
):
compile_program(MagicMock(), {})
mock_pio.assert_not_called()
def test_native_backend_resolves_host() -> None:
from esphome.build_helpers.native import native_backend
from esphome.host import toolchain as host_toolchain
setup_core(platform=PLATFORM_HOST)
CORE.toolchain = Toolchain.PLATFORMIO
assert native_backend() is None
CORE.toolchain = Toolchain.HOST
assert native_backend() is host_toolchain
# A native toolchain without a backend for the platform must not degrade
CORE.toolchain = Toolchain.ARDUINO
with pytest.raises(
EsphomeError, match="has no native build backend module for platform host"
):
native_backend()
def test_command_idedata_host_prints_json(capsys: CaptureFixture) -> None:
setup_core(platform=PLATFORM_HOST)
CORE.toolchain = Toolchain.HOST
data = {"cxx_path": "g++", "prog_path": "/build/program"}
with patch("esphome.host.toolchain.get_idedata", return_value=data) as mock_get:
assert command_idedata(MagicMock(), CORE.config) == 0
mock_get.assert_called_once_with()
assert json.loads(capsys.readouterr().out) == data
def _native_host_tools(tmp_path: Path) -> tuple[Path, Path, Path]:
objdump, readelf, elf = (tmp_path / n for n in ("objdump", "readelf", "program"))
for tool in (objdump, readelf, elf):
tool.write_text("")
return objdump, readelf, elf
def test_command_analyze_memory_native_toolchain(
tmp_path: Path,
capfd: CaptureFixture[str],
mock_write_cpp: Mock,
mock_compile_program: Mock,
mock_get_esphome_components: Mock,
mock_memory_analyzer_cli: Mock,
mock_ram_strings_analyzer: Mock,
) -> None:
"""A native backend supplies its own binutils and ELF, with no idedata."""
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
CORE.toolchain = Toolchain.HOST
objdump, readelf, elf = _native_host_tools(tmp_path)
config = {CONF_ESPHOME: {CONF_NAME: "dev"}}
with (
patch("esphome.host.toolchain.get_objdump_path", return_value=objdump),
patch("esphome.host.toolchain.get_readelf_path", return_value=readelf),
patch("esphome.host.toolchain.get_elf_path", return_value=elf),
patch("esphome.host.toolchain.check_analysis_supported"),
):
assert command_analyze_memory(MockArgs(), config) == 0
mock_memory_analyzer_cli.assert_called_once_with(
str(elf), str(objdump), str(readelf), set(), idedata=None
)
mock_ram_strings_analyzer.assert_called_once_with(
str(elf), objdump_path=str(objdump), platform="host"
)
assert "Mock Memory Report" in capfd.readouterr().out
def test_command_analyze_memory_native_missing_tool_fails(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
mock_write_cpp: Mock,
mock_compile_program: Mock,
) -> None:
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
CORE.toolchain = Toolchain.HOST
objdump, readelf, elf = _native_host_tools(tmp_path)
readelf.unlink()
with (
patch("esphome.host.toolchain.get_objdump_path", return_value=objdump),
patch("esphome.host.toolchain.get_readelf_path", return_value=readelf),
patch("esphome.host.toolchain.get_elf_path", return_value=elf),
patch("esphome.host.toolchain.check_analysis_supported"),
caplog.at_level(logging.ERROR),
):
assert command_analyze_memory(MockArgs(), {}) == 1
assert f"{readelf} is missing; the toolchain install may be incomplete" in (
caplog.text
)
def test_command_analyze_memory_host_missing_elf_fails(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
mock_write_cpp: Mock,
mock_compile_program: Mock,
) -> None:
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
CORE.toolchain = Toolchain.HOST
objdump, readelf, elf = _native_host_tools(tmp_path)
elf.unlink()
with (
patch("esphome.host.toolchain.get_objdump_path", return_value=objdump),
patch("esphome.host.toolchain.get_readelf_path", return_value=readelf),
patch("esphome.host.toolchain.get_elf_path", return_value=elf),
patch("esphome.host.toolchain.check_analysis_supported"),
caplog.at_level(logging.ERROR),
):
assert command_analyze_memory(MockArgs(), {}) == 1
assert f"{elf} is missing; compile the configuration first" in caplog.text
def test_command_analyze_memory_host_refuses_before_compiling(
tmp_path: Path,
mock_write_cpp: Mock,
mock_compile_program: Mock,
) -> None:
"""A machine whose host program is not ELF fails before the compile."""
setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev")
CORE.toolchain = Toolchain.HOST
with (
patch("esphome.host.toolchain.sys.platform", "darwin"),
pytest.raises(EsphomeError, match="analyze-memory reads ELF files"),
):
command_analyze_memory(MockArgs(), {})
mock_write_cpp.assert_not_called()
mock_compile_program.assert_not_called()
tests/unit_tests/test_platformio_library.py
+69
View File
@@ -407,6 +407,28 @@ def test_convert_libraries_redownloads_when_manifest_missing(
assert top[0].data["name"] == "A"
def test_convert_libraries_manifest_optional_uses_default_layout(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A backend accepting manifest-less libraries (the host build, like
PlatformIO's native platform) gets a default manifest instead of a
re-download that would repeat every build."""
calls = _patch_download_without_manifest(
monkeypatch, tmp_path, manifest_on_force=True
)
emitted: list[ConvertedLibrary] = []
backend = _backend(emit=emitted.append)
backend.manifest_optional = True
with caplog.at_level(logging.DEBUG, logger="esphome.platformio.library"):
top = convert_libraries([Library("esphome/A", "1.0.0", None)], backend)
assert calls == [False]
assert top[0].data == {"name": "esphome/A"}
assert emitted == top
assert "has no manifest; using PlatformIO's default layout" in caplog.text
def test_convert_libraries_raises_when_manifest_missing_after_retry(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -1251,3 +1273,50 @@ def test_versionless_dependency_matching_resolved_manifest_name_stays_quiet(
_backend(),
)
assert "has no version to resolve" not in caplog.text
def test_convert_libraries_symlink_url_resolves_as_local(setup_core: Path) -> None:
"""symlink:// is PlatformIO's other spelling for a local library folder."""
src = setup_core / "lib_dev"
(src / "src").mkdir(parents=True)
(src / "library.json").write_text(json.dumps({"name": "benchmark"}))
url = src.as_uri().replace("file://", "symlink://", 1)
# Both the name=URL form and an explicit repository take the scheme
for library in (
Library(f"benchmark={url}", None, None),
Library("benchmark", None, url),
):
top = convert_libraries([library], _backend())
assert isinstance(top[0].source, LocalSource)
assert top[0].source_path == src
def test_convert_libraries_incompatible_names_the_platform_without_a_framework(
setup_core: Path,
) -> None:
"""The host has no framework; the error must not read 'compatible with None'."""
src = setup_core / "lib_dev"
src.mkdir()
(src / "library.json").write_text(
json.dumps({"name": "Only32", "platforms": "espressif32"})
)
backend = _backend()
backend.platform = "native"
backend.framework = None
with pytest.raises(RuntimeError, match="Only32 is not compatible with native"):
convert_libraries([Library("Only32", None, src.as_uri())], backend)
@pytest.mark.parametrize(
("url", "message"),
[
("symlink://host/lib", "Unsupported host in symlink:// library URL"),
("symlink:lib_dev", "symlink:// library URL .* must be an absolute"),
],
)
def test_convert_libraries_symlink_url_errors_name_the_scheme(
setup_core: Path, url: str, message: str
) -> None:
with pytest.raises(RuntimeError, match=message):
convert_libraries([Library("benchmark", None, url)], _backend())