diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c89712e05..4dac899a41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,14 +400,9 @@ jobs: matrix: bucket: ${{ fromJson(needs.determine-jobs.outputs.integration-test-buckets) }} env: - # What the cache steps persist; libdeps is excluded (keyed per xdist - # worker and env, it never crosses runs). - INTEGRATION_PIO_CACHE_PATH: | - ~/.esphome-integration-tests/platformio/platforms - ~/.esphome-integration-tests/platformio/packages - ~/.esphome-integration-tests/platformio/appstate.json - ~/.esphome-integration-tests/platformio/.cache - ~/.esphome-integration-tests/platformio/.esphome.pio.stamp.json + # Registry libraries (noise-c, libsodium, ArduinoJson, lvgl) the host builds + # download, shared per xdist worker by tests/integration/conftest.py + INTEGRATION_LIBRARY_CACHE_PATH: ~/.esphome-integration-tests/pio_components steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -427,14 +422,13 @@ jobs: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" - - name: Restore integration PlatformIO cache - # Native platform + toolchain installed by shared_platformio_cache in - # tests/integration/conftest.py; a miss self-heals, so no restore-keys. - id: pio-cache + - name: Restore integration library cache + # A miss or a changed pin self-heals with a download, so no restore-keys + id: library-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ env.INTEGRATION_PIO_CACHE_PATH }} - key: integration-pio-v1-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements.txt', 'tests/integration/fixtures/cache_init.yaml', 'esphome/components/host/__init__.py') }} + path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }} + key: integration-libraries-v1-${{ runner.os }}-${{ hashFiles('esphome/components/json/__init__.py', 'esphome/components/noise/__init__.py', 'esphome/components/lvgl/__init__.py', 'esphome/components/improv_base/__init__.py') }} - name: Restore Python virtual environment id: cache-venv uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -502,16 +496,16 @@ jobs: # A full cron period of margin for the weekly refresh retention-days: 14 - name: Print ccache statistics - # esphome stores the PlatformIO ccache under the machine-global cache - # dir (see _ccache_env() in esphome/platformio/toolchain.py). - run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s - - name: Save integration PlatformIO cache + # esphome stores the host build's ccache under the machine-global + # cache dir (see get_build_env() in esphome/host/toolchain.py). + run: CCACHE_DIR="$HOME/.cache/esphome/host/ccache" ccache -s + - name: Save integration library cache # Bucket 0 only; the others would race the same immutable key. - if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.pio-cache.outputs.cache-hit != 'true' + if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.library-cache.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ env.INTEGRATION_PIO_CACHE_PATH }} - key: ${{ steps.pio-cache.outputs.cache-primary-key }} + path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }} + key: ${{ steps.library-cache.outputs.cache-primary-key }} import-time: name: Check import esphome.__main__ time diff --git a/esphome/__main__.py b/esphome/__main__.py index c3502db4ba..8ed97ccae8 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1714,26 +1714,12 @@ def command_compile(args: ArgsProtocol, config: ConfigType) -> int | None: if exit_code != 0: return exit_code if CORE.is_host: - _LOGGER.info( - "Successfully compiled program to path '%s'", _host_program_path(config) - ) + _LOGGER.info("Successfully compiled program to path '%s'", CORE.firmware_bin) else: _LOGGER.info("Successfully compiled program.") return 0 -def _host_program_path(config: ConfigType) -> str: - """Return the compiled host ELF path.""" - if CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain - - return str(toolchain.get_elf_path()) - from esphome.platformio.toolchain import get_idedata - - # Memoized by compile_program's own call; this is a dict lookup - return str(get_idedata(config).firmware_elf_path) - - def command_upload(args: ArgsProtocol, config: ConfigType) -> int | None: # Get devices, resolving special identifiers like OTA devices = choose_upload_log_host( @@ -1778,7 +1764,7 @@ def command_run(args: ArgsProtocol, config: ConfigType) -> int | None: return exit_code _LOGGER.info("Successfully compiled program.") if CORE.is_host: - program_path = _host_program_path(config) + program_path = str(CORE.firmware_bin) _LOGGER.info("Running program from path '%s'", program_path) return run_external_process(program_path) @@ -2024,6 +2010,11 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int: CORE.target_platform, ) return 1 + if ( + check_supported := getattr(analysis_toolchain, "check_analysis_supported", None) + ) is not None: + # Raises with the reason; before the compile, not after it + check_supported() # Always compile to ensure fresh data (fast if no changes - just relinks) exit_code = write_cpp(config) diff --git a/esphome/arduino/library.py b/esphome/arduino/library.py index e224e62589..143b6d23a0 100644 --- a/esphome/arduino/library.py +++ b/esphome/arduino/library.py @@ -2,7 +2,9 @@ Bundled names build straight from the framework tree; everything else goes through ``esphome.platformio.library``. Mirrors ``lib_ldf_mode=off``: each -library builds its own archive; all include dirs join one global path. +library builds its own archive; all include dirs join one global path. The +host build reuses it without a framework tree: nothing is bundled there and +every name resolves from the registry. Deviations from PlatformIO: flat-layout libraries get the recursive default source filter; ``dot_a_linkage`` is honored; bundled libraries never run a @@ -342,13 +344,24 @@ def _check_unfulfilled_provides( def resolve_libraries( - framework_path: Path, *, pio_platform: str, board_mcu: str, cache_key: str + framework_path: Path | None, + *, + pio_platform: str, + board_mcu: str, + cache_key: str, + framework: str | None = "arduino", + manifest_optional: bool = False, ) -> list[ArduinoLibrary]: """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`. ``pio_platform``/``board_mcu`` filter manifests the way PlatformIO would for that core (e.g. ``espressif8266``/``esp8266``); ``cache_key`` keys the - shared converter's download cache. + shared converter's download cache. ``framework`` is the manifest + framework token the compatibility check warns about; None skips it. + A None ``framework_path`` means no core-bundled libraries exist (the + host build): every name resolves from the registry. + ``manifest_optional`` accepts libraries without a manifest, built with + PlatformIO's default layout. The returned list is not topologically sorted, so the caller must link the archives inside one ``--start-group``/``--end-group`` pair (the @@ -359,18 +372,22 @@ def resolve_libraries( # PlatformIO's lib_ignore covers framework-bundled libraries too; the # shared converter only filters the registry/git ones. lib_ignore = lib_ignore_set() - # Exact directory names keep membership case-sensitive everywhere - # (an is_dir() probe would match "wire" on macOS/Windows and build - # the bundled Wire twice) - libraries_dir = framework_path / "libraries" - if not libraries_dir.is_dir(): - # A registry fallback would fail later with a misleading - # package-not-found error per bundled name - raise EsphomeError( - f"{libraries_dir} is missing; the framework install may be " - "incomplete (run 'esphome clean-all')" + bundled_dir_names: frozenset[str] = frozenset() + if framework_path is not None: + # Exact directory names keep membership case-sensitive everywhere + # (an is_dir() probe would match "wire" on macOS/Windows and build + # the bundled Wire twice) + libraries_dir = framework_path / "libraries" + if not libraries_dir.is_dir(): + # A registry fallback would fail later with a misleading + # package-not-found error per bundled name + raise EsphomeError( + f"{libraries_dir} is missing; the framework install may be " + "incomplete (run 'esphome clean-all')" + ) + bundled_dir_names = frozenset( + p.name for p in libraries_dir.iterdir() if p.is_dir() ) - bundled_dir_names = frozenset(p.name for p in libraries_dir.iterdir() if p.is_dir()) def _provided(name: object) -> bool: return _is_safe_library_name(name) and name in bundled_dir_names @@ -497,12 +514,13 @@ def resolve_libraries( backend = LibraryBackend( platform=pio_platform, - framework="arduino", + framework=framework, emit=_emit, cache_key=cache_key, # The walk must not resolve bundled names from the registry; # _add_bundled_dependencies adds them after emit provides=_provided, + manifest_optional=manifest_optional, ) if external: convert_libraries(external, backend) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 663002b3b1..d32d61375b 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -17,7 +17,7 @@ import os from pathlib import Path from typing import NamedTuple -from esphome.build_helpers.ccache import ccache_defaults_env +from esphome.build_helpers.ccache import ccache_env from esphome.build_helpers.ninja import find_ninja from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path from esphome.core import EsphomeError, Version @@ -143,17 +143,5 @@ def get_build_env(toolchain_path: Path, ccache: str | None) -> dict[str, str]: *filter(None, env.get("PATH", "").split(os.pathsep)), ] env["PATH"] = os.pathsep.join(parts) - env.update(ccache_env(ccache)) + env.update(ccache_env(ccache, ARDUINO8266_TOOLS_CACHE)) return env - - -def ccache_env(ccache: str | None) -> dict[str, str]: - """Return ccache settings for the build subprocess (not os.environ). - - ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None - when disabled. Values the user already set in the environment are - respected. - """ - if ccache is None: - return {} - return ccache_defaults_env(get_arduino8266_tools_path() / "ccache") diff --git a/esphome/arduino8266/toolchain.py b/esphome/arduino8266/toolchain.py index 339fb44337..5bc1889a21 100644 --- a/esphome/arduino8266/toolchain.py +++ b/esphome/arduino8266/toolchain.py @@ -2,21 +2,21 @@ from __future__ import annotations -import json import logging from pathlib import Path import subprocess from typing import TYPE_CHECKING -from esphome.build_helpers.ccache import resolve_ccache_path +from esphome.build_helpers.ccache import resolve_absolute_ccache_path +from esphome.build_helpers.native import warn_ignored_platformio_options +from esphome.build_helpers.ninja import refresh_compile_commands from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE, KEY_FRAMEWORK_VERSION, ) -from esphome.core import CORE, EsphomeError -from esphome.helpers import write_file +from esphome.core import CORE from esphome.types import ConfigType if TYPE_CHECKING: @@ -28,20 +28,6 @@ _LOGGER = logging.getLogger(__name__) _MAX_RAM_SIZE = 81920 -def _warn_ignored_platformio_options() -> None: - """Warn for component-added platformio options the native build drops.""" - from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS - - consumed = NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS - for key in sorted(CORE.platformio_options or {}): - if key not in consumed: - _LOGGER.warning( - "platformio_options->%s is ignored when building with the " - "native 'arduino' toolchain", - key, - ) - - _RAM_SECTIONS = (".data", ".rodata", ".bss") _FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata") @@ -84,21 +70,17 @@ def get_readelf_path() -> Path: def run_compile(config: ConfigType, verbose: bool) -> int: from esphome.arduino8266 import framework from esphome.build_gen import arduino8266 as build_gen + from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS - _warn_ignored_platformio_options() + warn_ignored_platformio_options(NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS) paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]) # Resolved once: the probe is not free and three consumers need it - ccache = resolve_ccache_path() - build_gen.write_project(paths, ccache) + ccache = resolve_absolute_ccache_path() + ninja_changed = build_gen.write_project(paths, ccache) build_dir = get_build_dir() env = framework.get_build_env(paths.toolchain, ccache) - - # The DB is a pure function of build.ninja; regenerate when it is older - compdb = build_dir / "compile_commands.json" - ninja_file = build_dir / "build.ninja" - if not compdb.is_file() or compdb.stat().st_mtime < ninja_file.stat().st_mtime: - _write_compile_commands(paths.ninja, build_dir, env) + refresh_compile_commands(paths.ninja, build_dir, env, ninja_changed) cmd = [str(paths.ninja)] if verbose: @@ -140,42 +122,6 @@ def run_compile(config: ConfigType, verbose: bool) -> int: return 0 -def _write_compile_commands( - ninja_path: Path, build_dir: Path, env: dict[str, str] -) -> None: - compdb = build_dir / "compile_commands.json" - result = subprocess.run( - [str(ninja_path), "-C", str(build_dir), "-t", "compdb", "c", "cxx", "asm"], - env=env, - capture_output=True, - text=True, - check=False, - close_fds=False, - ) - if result.returncode != 0: - # Drop any stale database so consumers (IDE integration, clang-tidy, - # the memory analyzer) can't silently read outdated data. - compdb.unlink(missing_ok=True) - raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}") - try: - entries = json.loads(result.stdout) - except ValueError as err: - compdb.unlink(missing_ok=True) - raise EsphomeError( - f"ninja produced an unparsable compile database: {err} " - f"(output starts {result.stdout[:120]!r})" - ) from err - if not entries: - # compdb exits 0 with [] for unknown rule names; a renamed compile - # rule must fail the build, not silently strand every consumer - compdb.unlink(missing_ok=True) - raise EsphomeError( - "ninja produced an empty compile database; the generator's rule " - "names no longer match" - ) - write_file(compdb, result.stdout) - - def _parse_app_size(build_dir: Path, paths: InstalledPaths) -> int | None: """Read the app flash budget (irom0_0_seg length) from the linker script.""" from esphome.build_gen.arduino8266 import get_flash_ld_path @@ -253,7 +199,7 @@ def get_idedata(ccache: str | None = None) -> dict | None: # A disabled ccache resolves to None without spawning anything, so # re-resolving here costs nothing when the caller has no answer - launcher = ccache or resolve_ccache_path() + launcher = ccache or resolve_absolute_ccache_path() return load_or_build_idedata( get_build_dir() / "compile_commands.json", get_elf_path(), diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index 285379ceea..4a0e29ab3c 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -14,7 +14,6 @@ from the build flags with the same precedence as the PlatformIO builder. from __future__ import annotations -from collections.abc import Collection from dataclasses import dataclass import hashlib import logging @@ -23,7 +22,6 @@ from pathlib import Path import re import shlex import subprocess -import sys from typing import TYPE_CHECKING, NamedTuple from esphome.arduino8266.framework import toolchain_tool @@ -32,6 +30,14 @@ from esphome.build_helpers.ninja import ( quote_path as _q, shell_token as _shell_token, ) +from esphome.build_helpers.ninja_gen import ( + ar_rule_lines, + collect_sources, + compile_edges, + compile_rule_lines, + library_edges, + tool_lines, +) from esphome.components.esp8266 import build_surgery from esphome.components.esp8266.boards import ( BOARDS, @@ -46,12 +52,9 @@ from esphome.components.esp8266.const import ( KEY_SCANF_FLOAT, ) from esphome.core import CORE, EsphomeError -from esphome.framework_helpers import ( - get_project_cxx_compile_flags, - strip_win_long_path_prefix, -) +from esphome.framework_helpers import get_project_cxx_compile_flags from esphome.helpers import mkdir_p, write_file_if_changed -from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX, lex_build_flags +from esphome.platformio.library import lex_build_flags if TYPE_CHECKING: from esphome.arduino8266.framework import InstalledPaths @@ -643,14 +646,6 @@ _PLAIN_LINKER_PREFIXES = ("-T", "-Xlinker") _PLAIN_DRIVER_LINK_PREFIXES = ("-fuse-ld=", "--specs=", "-specs=") -def _collect_sources(root: Path, exclude: Collection[str] = frozenset()) -> list[Path]: - return sorted( - p - for p in root.rglob("*") - if p.suffix in SOURCE_KIND_FOR_SUFFIX and p.name not in exclude - ) - - def _stat_sig(path: Path) -> str: """Size and mtime cache-stamp signature for one input file. @@ -868,34 +863,6 @@ def _generate_testing_flash_ld( ) -def _ninja_compile_edges( - lines: list[str], - sources: list[Path], - root: Path, - group: str, - flags: str = "", -) -> list[str]: - """Emit compile edges for ``sources``; return the object paths.""" - objects = [] - for src in sources: - rel = src.relative_to(root).as_posix() - obj = f"obj/{group}/{rel}.o" - escaped_obj = _e(obj) - lines.append( - f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}" - ) - if flags: - lines.append(f" flags = {flags}") - # Escaped once here: the returned paths only ever appear in build - # statements (archive/link inputs), which use ninja escaping. - objects.append(escaped_obj) - return objects - - -def _common_parent(paths: list[Path]) -> Path: - return Path(os.path.commonpath([str(p.parent) for p in paths])) - - class _FlagSets(NamedTuple): cflags: list[str] cxxflags: list[str] @@ -1082,48 +1049,14 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"] ) - build_tool = Path(__file__).parent / "build_tool.py" - - # $in/$out stay unquoted: ninja escapes its built-in path variables - # itself; only literal paths need _q(). lines = [ - "# Auto-generated by ESPHome", - "ninja_required_version = 1.5", - f"cc = {_q(toolchain_tool(paths.toolchain, 'gcc'))}", - f"cxx = {_q(toolchain_tool(paths.toolchain, 'g++'))}", - # The NSIS launcher starts Python with a \\?\ extended-length path - # that cmd.exe cannot spawn; same strip every other emitted binary - # path gets - f"python = {_q(strip_win_long_path_prefix(sys.executable))}", - f"buildtool = {_q(build_tool)}", - f"ccache = {_q(ccache) if ccache else ''}", - "", - # Rule names match SOURCE_KIND_FOR_SUFFIX values (c, cxx, asm, aspp) - "rule c", - " command = $ccache $cc -MMD -MF $out.d $cflags $flags -c $in -o $out", - " depfile = $out.d", - " deps = gcc", - " description = CC $out", - "rule cxx", - " command = $ccache $cxx -MMD -MF $out.d $cxxflags $flags -c $in -o $out", - " depfile = $out.d", - " deps = gcc", - " description = CXX $out", - "rule aspp", - " command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $asflags $flags -c $in -o $out", - " depfile = $out.d", - " deps = gcc", - " description = AS $out", - # Plain assembler, as SCons's ASCOM: no preprocessor, so no - # depfile and no $flags (defines/includes) either - "rule asm", - " command = $ccache $cc -x assembler $asflags -c $in -o $out", - " description = AS $out", - "rule ar", - f" command = $python $buildtool ar {_q(toolchain_tool(paths.toolchain, 'ar'))} $out $out.rsp", - " rspfile = $out.rsp", - " rspfile_content = $in_newline", - " description = AR $out", + *tool_lines( + (toolchain_tool(paths.toolchain, "gcc"),), + (toolchain_tool(paths.toolchain, "g++"),), + ccache, + ), + *compile_rule_lines(), + *ar_rule_lines(toolchain_tool(paths.toolchain, "ar")), "rule link", " command = $cxx -o $out $linkflags @$out.rsp $libdirflags -Wl,--start-group $archives $libflags -Wl,--end-group", " rspfile = $out.rsp", @@ -1153,16 +1086,15 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: core_exclude |= _CORE_EXCLUDE_WAVEFORM archives = [] - direct_objs: list[str] = [] # variant_dir existence was already enforced with the include dirs - variant_sources = _collect_sources(variant_dir) + variant_sources = collect_sources(variant_dir) if variant_sources: - objs = _ninja_compile_edges(lines, variant_sources, variant_dir, "variant") + objs = compile_edges(lines, variant_sources, variant_dir, "variant") lines.append(f"build libFrameworkArduinoVariant.a: ar {' '.join(objs)}") archives.append("libFrameworkArduinoVariant.a") - core_objs = _ninja_compile_edges( - lines, _collect_sources(core_dir, core_exclude), core_dir, "core" + core_objs = compile_edges( + lines, collect_sources(core_dir, core_exclude), core_dir, "core" ) if not core_objs: # An empty archive would link into a wall of undefined references @@ -1173,32 +1105,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: lines.append(f"build libFrameworkArduino.a: ar {' '.join(core_objs)}") archives.append("libFrameworkArduino.a") - for lib in libraries: - if not lib.sources: - # Header-only libraries are legitimate; the log makes an empty - # srcFilter or broken tree traceable before link errors do. - _LOGGER.debug( - "Library %s has no source files; contributing includes only", - lib.name, - ) - continue - lib_root = _common_parent(lib.sources) - objs = _ninja_compile_edges( - lines, - lib.sources, - lib_root, - f"lib/{lib.name}", - flags=" ".join(_shell_token(f) for f in lib.flags), - ) - if not lib.lib_archive: - # libArchive: false / dot_a_linkage=false: hand the objects to - # the linker directly so unreferenced-but-required symbols - # (exception handlers, weak overrides) survive - direct_objs.extend(objs) - continue - archive = f"lib{lib.name}.a" - lines.append(f"build {_e(archive)}: ar {' '.join(objs)}") - archives.append(archive) + lib_archives, direct_objs = library_edges(lines, libraries) + archives += lib_archives # One source of truth with the PlatformIO path: esp8266/__init__ pins # build_src_flags (the throw_stubs force-include); -include paths @@ -1221,8 +1129,8 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: # One shared variable instead of repeating the flags line on every src # edge (hundreds of edges in a real project) lines.append(f"srcflags = {src_extra}") - src_objs = _ninja_compile_edges( - lines, _collect_sources(src_dir), src_dir, "src", flags="$srcflags" + src_objs = compile_edges( + lines, collect_sources(src_dir), src_dir, "src", flags="$srcflags" ) ld_deps = [f"ld/{_COMMON_LD_NAME}"] diff --git a/esphome/build_gen/host.py b/esphome/build_gen/host.py new file mode 100644 index 0000000000..c883d912eb --- /dev/null +++ b/esphome/build_gen/host.py @@ -0,0 +1,272 @@ +"""Native ninja build generator for the host platform. + +Emits ``build.ninja`` under ``.pioenvs//``: every source in the +generated ``src/`` tree plus the resolved registry libraries compiles with +the machine's compiler and links into ``program``, the name PlatformIO's +native platform produced. Build flags route the way SCons's ``ParseFlags`` +did under PlatformIO: ``-D``/``-I``/``-std=``/``-W`` shapes reach the +compile lines only, ``-l``/``-L``/``-Wl,`` the link line only, everything +else both. +""" + +from __future__ import annotations + +from collections.abc import Iterable +import logging +import os +from pathlib import Path +import sys +from typing import TYPE_CHECKING + +from esphome.build_helpers.ninja import escape as _e, quote_path as _q, shell_token +from esphome.build_helpers.ninja_gen import ( + PATH_ARG_FLAGS, + Flag, + anchor_path_flag, + ar_rule_lines, + collect_sources, + compile_edges, + compile_rule_lines, + library_edges, + tool_lines, +) +from esphome.core import CORE, EsphomeError +from esphome.framework_helpers import get_project_cxx_compile_flags +from esphome.helpers import mkdir_p, write_file_if_changed +from esphome.host.toolchain import PROGRAM_NAME, HostCompilers, find_tool, get_build_dir +from esphome.platformio.library import lex_build_flags + +if TYPE_CHECKING: + from esphome.arduino.library import ArduinoLibrary + +_LOGGER = logging.getLogger(__name__) + +# The PlatformIO platform the host built under; registry manifests declare +# compatibility against it, as lib_compat_mode=strict checked before +PIO_PLATFORM = "native" +# Namespaces the shared library download cache (pio_components/host/) +LIBRARY_CACHE_KEY = "host" + +# Flag shapes that only the compiler understands; dropped from the link line +_COMPILE_ONLY_PREFIXES = ("-D", "-U", "-I", "-std=", "-W", *PATH_ARG_FLAGS) +# Flag shapes that only the linker consumes; inert on a -c compile line +_LINK_ONLY_PREFIXES = ("-l", "-L", "-Wl,") +# Link-only flags whose argument is the next token +_LINK_ONLY_ARG_FLAGS = ("-framework", "-Xlinker", "-z") + + +def parse_flags(entries: Iterable[str], owner: str) -> list[Flag]: + """Lex build flag entries into flags, each with its argument. + + Entries are a set, so their order is not the user's: a flag and the + argument it takes as the next token must share one entry. + """ + flags: list[Flag] = [] + for entry in entries: + it = iter(lex_build_flags(entry, owner)) + for tok in it: + if tok not in PATH_ARG_FLAGS and tok not in _LINK_ONLY_ARG_FLAGS: + flags.append((tok,)) + continue + arg = next(it, None) + # A path never starts with "-"; that is the next flag + if arg is None or (tok in PATH_ARG_FLAGS and arg.startswith("-")): + raise EsphomeError( + f"{owner} build flags have '{tok}' with no argument; write " + f"the flag and its argument as one entry" + ) + flags.append((tok, arg)) + return flags + + +def split_flags(flags: list[Flag]) -> tuple[list[Flag], list[Flag]]: + """Route build flags to the compile and link lines.""" + compile_flags: list[Flag] = [] + link_flags: list[Flag] = [] + for flag in flags: + name = flag[0] + if len(flag) > 1: + (compile_flags if name in PATH_ARG_FLAGS else link_flags).append(flag) + elif name.startswith(_LINK_ONLY_PREFIXES): + # Checked before the compile prefixes: -Wl, would match -W + link_flags.append(flag) + elif name.startswith(_COMPILE_ONLY_PREFIXES): + compile_flags.append(flag) + else: + # -g, -O, -f*, -m*, -pthread, --coverage: both lines, as SCons + compile_flags.append(flag) + link_flags.append(flag) + return compile_flags, link_flags + + +def _is_std(flag: Flag) -> bool: + return flag[0].startswith("-std=") + + +def _is_cxx_std(flag: Flag) -> bool: + return _is_std(flag) and "++" in flag[0] + + +def _anchored_flags(entries: Iterable[str], owner: str) -> list[Flag]: + build_path = Path(CORE.build_path) + return [ + anchor_path_flag(flag, build_path) + for flag in parse_flags(sorted(entries), owner) + ] + + +def _flag_lists() -> tuple[list[str], list[str], list[str]]: + """The C, C++, and link flag lists (raw tokens), build_unflags applied. + + ``cg.set_cpp_standard`` wins over any ``-std=`` in the build flags for + C++ compiles, as PlatformIO's unflag of every other standard did; C + compiles never see a C++ standard. + """ + # The funnel warns and drops empty glued arguments (-D "") itself + compile_flags, link_flags = split_flags( + _anchored_flags(CORE.build_flags, "esphome") + ) + cflags = [f for f in compile_flags if not _is_cxx_std(f)] + cxx_std = CORE.cpp_standard + cxxflags = [f for f in compile_flags if not (cxx_std and _is_std(f))] + if cxx_std: + cxxflags.insert(0, (f"-std={cxx_std}",)) + cxxflags += [(tok,) for tok in get_project_cxx_compile_flags()] + + # A flag is removed whole, with its argument, as PlatformIO did + unflags = set(_anchored_flags(CORE.build_unflags, "esphome build_unflags")) + # An unflag that hits nothing (a typo, or -DUSE_FOO against + # -DUSE_FOO=1) must be visible, since the user believes the flag is + # gone while it still drives the build + if unmatched := sorted(unflags - set(cflags) - set(cxxflags) - set(link_flags)): + _LOGGER.warning( + "build_unflags entries matched no build flag: %s", + ", ".join(" ".join(flag) for flag in unmatched), + ) + + def keep(flags: list[Flag]) -> list[str]: + return [tok for flag in flags if flag not in unflags for tok in flag] + + return keep(cflags), keep(cxxflags), keep(link_flags) + + +def _resolve_host_libraries() -> list[ArduinoLibrary]: + """Every ``cg.add_library()`` entry, fetched from the registry. + + The host has no framework, so nothing is bundled and no framework + compatibility check applies; the platform check keeps the strict + manifest gate PlatformIO's native platform enforced. Manifest-less + libraries (a bare git checkout) build with PlatformIO's default + layout, as they did under its native platform. + """ + if not CORE.platformio_libraries: + return [] + from esphome.arduino.library import resolve_libraries + + return resolve_libraries( + None, + pio_platform=PIO_PLATFORM, + board_mcu="host", + cache_key=LIBRARY_CACHE_KEY, + framework=None, + manifest_optional=True, + ) + + +def _file_macro_maps(build_dir: Path) -> list[str]: + """Flags that keep ``__FILE__`` relative to the build path. + + PlatformIO compiled ``src/x.cpp`` from the build path, and tools name + things after that spelling (CodSpeed's benchmark ids). Here a source + reaches the compiler by its absolute path, or relative to the build + directory when ccache rewrites it. + """ + build_path = Path(CORE.build_path) + prefixes = (build_path, Path(os.path.relpath(build_path, build_dir))) + return [ + shell_token(f"-fmacro-prefix-map={prefix}{os.sep}=", force=True) + for prefix in prefixes + ] + + +def write_project(compilers: HostCompilers, ccache: str | None) -> bool: + """Write the ninja build for the current configuration. + + ``ccache`` is the caller's already-resolved binary (None when disabled). + Returns True when ``build.ninja`` changed, so the caller can skip work + derived purely from it (the compile database) on unchanged builds. + """ + build_dir = get_build_dir() + mkdir_p(build_dir) + src_dir = CORE.relative_src_path() + if not src_dir.is_dir(): + # Generated project state, not install state: clean-all would not help + raise EsphomeError(f"Generated source directory {src_dir} is missing") + + cflags, cxxflags, link_flags = _flag_lists() + libraries = _resolve_host_libraries() + + include_dirs = [src_dir] + for lib in libraries: + include_dirs += lib.include_dirs + includes = [f"-I{_q(d)}" for d in include_dirs] + includes += _file_macro_maps(build_dir) + + # SCons's link line: $LINKFLAGS $SOURCES $_LIBDIRFLAGS $_LIBFLAGS, so + # -L and -l trail the objects while every other link token leads + lib_dirs = [Path(t[2:]) for t in link_flags if t.startswith("-L")] + libs = [t for t in link_flags if t.startswith("-l")] + linkflags = [shell_token(t) for t in link_flags if not t.startswith(("-L", "-l"))] + for lib in libraries: + lib_dirs += lib.link_dirs + libs += [f"-l{name}" for name in lib.link_libs] + linkflags += [shell_token(f) for f in lib.link_flags] + + # PlatformIO's ASPPCOM passes only -D/-I user flags to assembly + asflags = [t for t in cflags if t.startswith(("-D", "-I"))] + + lines = [ + *tool_lines(compilers.cc, compilers.cxx, ccache), + *compile_rule_lines(), + "rule link", + " command = $cxx -o $out $linkflags @$out.rsp $archives $libdirflags $libflags", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = LINK $out", + ] + if any(lib.sources and lib.lib_archive for lib in libraries): + # Resolved only when an archive is built, so a system without + # binutils still links a library-free configuration + lines += ar_rule_lines(find_tool("AR", ("ar",))) + lines += [ + "", + f"cflags = {' '.join([*map(shell_token, cflags), *includes])}", + f"cxxflags = {' '.join([*map(shell_token, cxxflags), *includes])}", + f"asflags = {' '.join([*map(shell_token, asflags), *includes])}", + f"linkflags = {' '.join(linkflags)}", + f"libdirflags = {' '.join(f'-L{_q(d)}' for d in lib_dirs)}", + f"libflags = {' '.join(shell_token(lib) for lib in libs)}", + "", + ] + + archives, direct_objs = library_edges(lines, libraries) + + src_objs = compile_edges(lines, collect_sources(src_dir), src_dir, "src") + if not src_objs: + raise EsphomeError(f"No source files found under {src_dir}") + + # Archives are not topologically sorted; GNU ld needs the group to + # resolve references between them. ld64 loads archives iteratively and + # rejects the option, so macOS lists them bare. + archive_tokens = [shell_token(a) for a in archives] + if archive_tokens and sys.platform != "darwin": + archive_tokens = ["-Wl,--start-group", *archive_tokens, "-Wl,--end-group"] + lines.append( + f"build {PROGRAM_NAME}: link {' '.join(src_objs + direct_objs)} | " + f"{' '.join(_e(a) for a in archives)}" + ) + lines.append(f" archives = {' '.join(archive_tokens)}") + lines.append(f"default {PROGRAM_NAME}") + lines.append("") + + return write_file_if_changed(build_dir / "build.ninja", "\n".join(lines)) diff --git a/esphome/build_helpers/ccache.py b/esphome/build_helpers/ccache.py index 5b5c7f247f..541dcbe3d3 100644 --- a/esphome/build_helpers/ccache.py +++ b/esphome/build_helpers/ccache.py @@ -68,6 +68,15 @@ def resolve_ccache_path() -> str | None: return ccache +def resolve_absolute_ccache_path() -> str | None: + """``resolve_ccache_path`` for the ninja backends, which run their + commands from the build directory, where a relative path is lost.""" + from esphome.build_helpers.ninja import absolute_tool + + ccache = resolve_ccache_path() + return absolute_tool(ccache) if ccache else None + + def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: """Default ``CCACHE_*`` values for a build subprocess (not os.environ). @@ -84,9 +93,24 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]: "CORE.build_path must be set before constructing the build environment" ) defaults = { - "CCACHE_DIR": str(cache_dir), + # ccache expands $VAR in its settings; $$ is a literal $ + "CCACHE_DIR": str(cache_dir).replace("$", "$$"), "CCACHE_NOHASHDIR": "true", "CCACHE_DEPEND": "1", - "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()), + "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()).replace("$", "$$"), } return {k: v for k, v in defaults.items() if k not in os.environ} + + +def ccache_env(ccache: str | None, tools_cache: tuple[str, str]) -> dict[str, str]: + """The ccache settings for a build subprocess (not os.environ). + + ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None when + disabled; ``tools_cache`` is the backend's tools cache spec, which holds + its ccache dir. + """ + if ccache is None: + return {} + from esphome.build_helpers.tools_cache import tools_cache_path + + return ccache_defaults_env(tools_cache_path(*tools_cache) / "ccache") diff --git a/esphome/build_helpers/native.py b/esphome/build_helpers/native.py index 24499c3ccd..1d2dc1b3ca 100644 --- a/esphome/build_helpers/native.py +++ b/esphome/build_helpers/native.py @@ -7,16 +7,21 @@ platform component package or the backend itself. from __future__ import annotations +from collections.abc import Collection import importlib +import logging from types import ModuleType from esphome.const import Toolchain from esphome.core import CORE, EsphomeError +_LOGGER = logging.getLogger(__name__) + # Native build backend per (target platform, toolchain) NATIVE_TOOLCHAIN_MODULES = { ("esp32", Toolchain.ESP_IDF): "esphome.espidf.toolchain", ("esp8266", Toolchain.ARDUINO): "esphome.arduino8266.toolchain", + ("host", Toolchain.HOST): "esphome.host.toolchain", } @@ -50,3 +55,20 @@ def analysis_backend() -> ModuleType | None: return native module_path = ANALYSIS_TOOLCHAIN_MODULES.get((CORE.target_platform, CORE.toolchain)) return importlib.import_module(module_path) if module_path else None + + +def warn_ignored_platformio_options(consumed: Collection[str]) -> None: + """Warn for component-added platformio options a native build drops. + + User-supplied keys were already routed or warned about by + ``core/config.py``; what survives into ``CORE.platformio_options`` came + from ``cg.add_platformio_option`` calls in components. + """ + for key in sorted(CORE.platformio_options or {}): + if key not in consumed: + _LOGGER.warning( + "platformio_options->%s is ignored when building with the " + "native '%s' toolchain", + key, + CORE.toolchain.value, + ) diff --git a/esphome/build_helpers/ninja.py b/esphome/build_helpers/ninja.py index 8c25bc9513..7634b0f2f3 100644 --- a/esphome/build_helpers/ninja.py +++ b/esphome/build_helpers/ninja.py @@ -2,14 +2,17 @@ from __future__ import annotations +import json import logging import os from pathlib import Path import re import shutil +import subprocess from esphome.core import EsphomeError from esphome.framework_helpers import strip_win_long_path_prefix, tool_version_runs +from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -23,11 +26,26 @@ def _ninja_runs(binary: str) -> bool: ) +# Compile rule names the generators emit; ninja's compdb tool is asked for +# exactly these, so a renamed rule fails the build instead of stranding idedata +COMPILE_RULES = ("c", "cxx", "aspp", "asm") + + +def absolute_tool(tool: str | Path) -> str: + """A tool path that still resolves from the build directory. + + ``shutil.which`` returns a relative path for a relative PATH entry, and + ninja runs the commands from ``.pioenvs/``. Symlinks are kept: + ccache's compiler links depend on the name they are called by. + """ + return strip_win_long_path_prefix(str(Path(tool).absolute())) + + def find_ninja() -> Path: """Locate the ninja binary: a runnable PATH hit first, else the ninja PyPI wheel.""" if binary := shutil.which("ninja"): - binary = strip_win_long_path_prefix(binary) + binary = absolute_tool(binary) if _ninja_runs(binary): return Path(binary) import_error: ImportError | None = None @@ -49,8 +67,18 @@ def find_ninja() -> Path: def escape(value: Path | str) -> str: - """Escape a path or token for a ninja file.""" - return str(value).replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + """Escape a path or token for a ninja file. + + ninja has no escape for ``|`` or a line break in a path, so those fail + here by name instead of producing a build file ninja misreads. + """ + text = str(value) + if bad := next((c for c in "|\r\n" if c in text), None): + raise EsphomeError( + f"Path {text!r} contains {bad!r}, which a ninja build file cannot " + "express; rename the file or directory" + ) + return text.replace("$", "$$").replace(":", "$:").replace(" ", "$ ") def quote_arg(tok: str) -> str: @@ -90,3 +118,65 @@ def shell_token(tok: str, force: bool = False) -> str: def quote_path(value: Path | str) -> str: """Force-quote a path for the ninja command line (shell/CreateProcess).""" return shell_token(str(value), force=True) + + +def refresh_compile_commands( + ninja_path: Path, build_dir: Path, env: dict[str, str], ninja_changed: bool +) -> None: + """Regenerate the compile DB (a pure function of build.ninja) when stale. + + Freshness rides a stamp: the DB itself is written through + write_file_if_changed (its mtime feeds the idedata cache), so a + regeneration with identical content would stay "stale" forever. An + interrupted previous run may have rewritten the manifest without + regenerating the DB, hence the mtime comparison. + """ + compdb = build_dir / "compile_commands.json" + compdb_stamp = build_dir / ".compile_commands.stamp" + ninja_file = build_dir / "build.ninja" + if ( + ninja_changed + or not compdb.is_file() + or not compdb_stamp.is_file() + or compdb_stamp.stat().st_mtime < ninja_file.stat().st_mtime + ): + write_compile_commands(ninja_path, build_dir, env) + compdb_stamp.touch() + + +def write_compile_commands( + ninja_path: Path, build_dir: Path, env: dict[str, str] +) -> None: + compdb = build_dir / "compile_commands.json" + result = subprocess.run( + [str(ninja_path), "-C", str(build_dir), "-t", "compdb", *COMPILE_RULES], + env=env, + capture_output=True, + text=True, + check=False, + close_fds=False, + ) + if result.returncode != 0: + # Drop any stale database so consumers (IDE integration, clang-tidy, + # the memory analyzer) can't silently read outdated data + compdb.unlink(missing_ok=True) + raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}") + try: + entries = json.loads(result.stdout) + except ValueError as err: + compdb.unlink(missing_ok=True) + raise EsphomeError( + f"ninja produced an unparsable compile database: {err} " + f"(output starts {result.stdout[:120]!r})" + ) from err + if not entries: + # compdb exits 0 with [] for unknown rule names; a renamed compile + # rule must fail the build, not silently strand every consumer + compdb.unlink(missing_ok=True) + raise EsphomeError( + "ninja produced an empty compile database; the generator's rule " + "names no longer match" + ) + # write_file_if_changed keeps the mtime stable on no-op builds so the + # idedata cache stays valid + write_file_if_changed(compdb, result.stdout) diff --git a/esphome/build_helpers/ninja_gen.py b/esphome/build_helpers/ninja_gen.py new file mode 100644 index 0000000000..d4cfc301ac --- /dev/null +++ b/esphome/build_helpers/ninja_gen.py @@ -0,0 +1,204 @@ +"""The parts of a ``build.ninja`` every native backend emits the same way. + +Rule names match ``SOURCE_KIND_FOR_SUFFIX`` values (c, cxx, asm, aspp) and +``ninja.COMPILE_RULES``, which the compile database is asked for. +""" + +from __future__ import annotations + +from collections.abc import Collection, Sequence +import logging +import os +from pathlib import Path +import sys +from typing import TYPE_CHECKING + +from esphome.build_helpers.ninja import ( + escape as _e, + quote_path as _q, + shell_token as _shell_token, +) +from esphome.framework_helpers import strip_win_long_path_prefix +from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX + +if TYPE_CHECKING: + from esphome.arduino.library import ArduinoLibrary + +_LOGGER = logging.getLogger(__name__) + +_BUILD_TOOL = Path(__file__).parent.parent / "build_gen" / "build_tool.py" + + +def collect_sources(root: Path, exclude: Collection[str] = frozenset()) -> list[Path]: + return sorted( + p + for p in root.rglob("*") + if p.suffix in SOURCE_KIND_FOR_SUFFIX and p.name not in exclude + ) + + +def common_parent(paths: list[Path]) -> Path: + return Path(os.path.commonpath([str(p.parent) for p in paths])) + + +def _command(words: Sequence[Path | str]) -> str: + program, *args = words + return " ".join([_q(program), *(_shell_token(str(a)) for a in args)]) + + +def tool_lines( + cc: Sequence[Path | str], cxx: Sequence[Path | str], ccache: str | None +) -> list[str]: + """The file header: the compilers and the helper tools as variables. + + A compiler is its program followed by any arguments it always takes. + """ + return [ + "# Auto-generated by ESPHome", + "ninja_required_version = 1.5", + f"cc = {_command(cc)}", + f"cxx = {_command(cxx)}", + # The NSIS launcher starts Python with a \\?\ extended-length path + # that cmd.exe cannot spawn; same strip every other emitted binary + # path gets + f"python = {_q(strip_win_long_path_prefix(sys.executable))}", + f"buildtool = {_q(_BUILD_TOOL)}", + f"ccache = {_q(ccache) if ccache else ''}", + "", + ] + + +def compile_rule_lines() -> list[str]: + """The compile rules; ``$own_includes`` is empty unless an edge sets it.""" + return [ + "rule c", + " command = $ccache $cc -MMD -MF $out.d $own_includes $cflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CC $out", + "rule cxx", + " command = $ccache $cxx -MMD -MF $out.d $own_includes $cxxflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = CXX $out", + "rule aspp", + " command = $ccache $cc -MMD -MF $out.d -x assembler-with-cpp $own_includes $asflags $flags -c $in -o $out", + " depfile = $out.d", + " deps = gcc", + " description = AS $out", + # Plain assembler, as SCons's ASCOM: no preprocessor, so no + # depfile and no $flags (defines/includes) either + "rule asm", + " command = $ccache $cc -x assembler $asflags -c $in -o $out", + " description = AS $out", + ] + + +def ar_rule_lines(ar: Path | str) -> list[str]: + return [ + "rule ar", + f" command = $python $buildtool ar {_q(ar)} $out $out.rsp", + " rspfile = $out.rsp", + " rspfile_content = $in_newline", + " description = AR $out", + ] + + +def compile_edges( + lines: list[str], + sources: list[Path], + root: Path, + group: str, + flags: str = "", + own_includes: str = "", +) -> list[str]: + """Emit compile edges for ``sources``; return the object paths.""" + objects = [] + for src in sources: + rel = src.relative_to(root).as_posix() + obj = f"obj/{group}/{rel}.o" + escaped_obj = _e(obj) + lines.append( + f"build {escaped_obj}: {SOURCE_KIND_FOR_SUFFIX[src.suffix]} {_e(src)}" + ) + if own_includes: + lines.append(f" own_includes = {own_includes}") + if flags: + lines.append(f" flags = {flags}") + # Escaped once here: the returned paths only ever appear in build + # statements (archive/link inputs), which use ninja escaping + objects.append(escaped_obj) + return objects + + +def library_edges( + lines: list[str], libraries: list[ArduinoLibrary] +) -> tuple[list[str], list[str]]: + """Emit every library's compile and archive edges. + + Returns the archive names and the objects that link directly. A + library's own include dirs lead its compile lines, as PlatformIO searched + them first: the include path is one global list, so another library's + header of the same name would shadow them. + """ + archives: list[str] = [] + direct_objs: list[str] = [] + for lib in libraries: + if not lib.sources: + # Header-only libraries are legitimate; the log makes an empty + # srcFilter or broken tree traceable before link errors do + _LOGGER.debug( + "Library %s has no source files; contributing includes only", + lib.name, + ) + continue + objs = compile_edges( + lines, + lib.sources, + common_parent(lib.sources), + f"lib/{lib.name}", + flags=" ".join(_shell_token(f) for f in lib.flags), + own_includes=" ".join(f"-I{_q(d)}" for d in lib.include_dirs), + ) + if not lib.lib_archive: + # libArchive: false / dot_a_linkage=false: hand the objects to + # the linker directly so unreferenced-but-required symbols + # (exception handlers, weak overrides) survive + direct_objs.extend(objs) + continue + archive = f"lib{lib.name}.a" + lines.append(f"build {_e(archive)}: ar {' '.join(objs)}") + archives.append(archive) + return archives, direct_objs + + +# One build flag: the flag, plus its argument when that is a separate token +Flag = tuple[str, ...] + +# Flags whose path operand is the next token; gcc also takes it glued on +PATH_ARG_FLAGS = ("-include", "-imacros", "-isystem", "-iquote", "-idirafter") +# Flags whose path operand is glued on +PATH_PREFIXES = ("-I", "-L", *PATH_ARG_FLAGS) + + +def _anchor(path: str, base: Path) -> str: + if not path or Path(path).is_absolute(): + return path + return str(base / path) + + +def anchor_path_flag(flag: Flag, base: Path) -> Flag: + """Anchor a flag's relative path operand at ``base``. + + PlatformIO ran the compiler from the build path; ninja runs it from + ``.pioenvs/``, where a relative operand would point elsewhere. + """ + name, *args = flag + if args: + if name in PATH_ARG_FLAGS: + return (name, _anchor(args[0], base)) + return flag + for prefix in PATH_PREFIXES: + if name.startswith(prefix): + return (prefix + _anchor(name[len(prefix) :], base),) + return flag diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py index e7193a8e2a..2dbbb2ffc2 100644 --- a/esphome/build_helpers/tools_cache.py +++ b/esphome/build_helpers/tools_cache.py @@ -33,4 +33,11 @@ def tools_cache_path(env_var: str, subdir: str) -> Path: IDF_TOOLS_CACHE = ("ESPHOME_ESP_IDF_PREFIX", "idf") SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf") ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266") -TOOLS_CACHE_SPECS = (IDF_TOOLS_CACHE, SDK_NRF_TOOLS_CACHE, ARDUINO8266_TOOLS_CACHE) +# The host backend installs nothing; the entry only holds its ccache dir +HOST_TOOLS_CACHE = ("ESPHOME_HOST_PREFIX", "host") +TOOLS_CACHE_SPECS = ( + IDF_TOOLS_CACHE, + SDK_NRF_TOOLS_CACHE, + ARDUINO8266_TOOLS_CACHE, + HOST_TOOLS_CACHE, +) diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index bd074ab6b5..16e4c81d34 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -1,3 +1,5 @@ +import sys + import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import ( @@ -8,9 +10,9 @@ from esphome.const import ( KEY_TARGET_PLATFORM, PLATFORM_HOST, ThreadModel, + Toolchain, ) -from esphome.core import CORE -from esphome.platformio.toolchain import copy_ccache_script +from esphome.core import CORE, EsphomeError from esphome.types import ConfigType from .const import KEY_HOST @@ -37,7 +39,9 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_MAC_ADDRESS, default="98:35:69:ab:f6:79"): cv.mac_address, } ), - cv.require_platformio_toolchain("host"), + # The host builds with the machine's own compiler through ninja; there + # is no PlatformIO backend, so a CLI --toolchain must name this one + cv.resolve_toolchain("host", (Toolchain.HOST,), Toolchain.HOST), set_core_data, ) @@ -48,16 +52,23 @@ async def to_code(config: ConfigType) -> None: # The prefs file finds stored preferences by key, so key migration is possible cg.add_define("USE_PREFERENCE_KEY_LOOKUP") cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts) - cg.add_build_flag("-std=gnu++20") + cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", "host") cg.add_define("ESPHOME_VARIANT", "HOST") cg.add_define(ThreadModel.MULTI_ATOMICS) - cg.add_platformio_option("platform", "platformio/native") - cg.add_platformio_option("lib_ldf_mode", "off") - cg.add_platformio_option("lib_compat_mode", "strict") - cg.add_platformio_option("extra_scripts", ["pre:ccache.py"]) -# Called by writer.py -def copy_files() -> None: - copy_ccache_script() +# Called by __main__.compile_program; True means this platform built the +# program itself instead of falling through to the PlatformIO toolchain. +def run_compile(args: object, config: ConfigType) -> bool: + from esphome.host import toolchain + + if sys.platform == "win32": + # Here, not in validation: `esphome config` must keep working + raise EsphomeError( + "The host platform uses POSIX interfaces and does not build on " + "Windows; use WSL or a Linux or macOS machine" + ) + if toolchain.run_compile(config, CORE.verbose) != 0: + raise EsphomeError("Host build failed") + return True diff --git a/esphome/const.py b/esphome/const.py index 5ffbf8c49a..fb02d70281 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -23,12 +23,14 @@ class Toolchain(StrEnum): SDK_NRF = "sdk-nrf" # ESP8266: the Arduino core built directly (no PlatformIO) ARDUINO = "arduino" + # host: the system C/C++ compiler driven by ninja (no PlatformIO) + HOST = "host" # Toolchains that drive their build natively and never read platformio.ini. # SDK_NRF is absent on purpose: the zephyr backend keeps consuming # platformio_options. -NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO}) +NATIVE_TOOLCHAINS = frozenset({Toolchain.ESP_IDF, Toolchain.ARDUINO, Toolchain.HOST}) class Platform(StrEnum): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index fb75285081..fa3d6bb64e 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -996,6 +996,12 @@ class EsphomeCore: ``using_arduino``, which is the target framework).""" return self.toolchain == Toolchain.ARDUINO + @property + def using_toolchain_host(self): + """The native host build toolchain: the system compiler driven by + ninja (the only toolchain the host platform serves).""" + return self.toolchain == Toolchain.HOST + @property def using_native_toolchain(self): """Whether the selected toolchain builds natively, without reading diff --git a/esphome/host/__init__.py b/esphome/host/__init__.py new file mode 100644 index 0000000000..003b47eb99 --- /dev/null +++ b/esphome/host/__init__.py @@ -0,0 +1,10 @@ +"""Native (PlatformIO-free) build support for the host platform. + +Builds the generated sources with the machine's own C/C++ compiler through a +ninja file and drives the build directly -- the host equivalent of +``esphome.espidf``. Nothing is downloaded: the compiler comes from PATH and +registry libraries go through the shared library converter. + +Deliberately importable without the host component to avoid circular +imports; the component wires these modules in via lazy imports. +""" diff --git a/esphome/host/toolchain.py b/esphome/host/toolchain.py new file mode 100644 index 0000000000..47b5bda48c --- /dev/null +++ b/esphome/host/toolchain.py @@ -0,0 +1,196 @@ +"""Native host build driver (the PlatformIO ``run`` equivalent). + +The compiler and binutils come from PATH (``CC``/``CXX``/``AR``/``OBJDUMP``/ +``READELF`` override the lookup, like make and CMake), ninja from PATH or +the ninja PyPI wheel, and ccache is used when found. The build lives under +``.pioenvs//`` so ``CORE.firmware_bin`` and the clean paths stay the +ones the PlatformIO build used. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +from typing import NamedTuple + +from esphome.build_helpers.ccache import ccache_env, resolve_absolute_ccache_path +from esphome.build_helpers.native import warn_ignored_platformio_options +from esphome.build_helpers.ninja import ( + absolute_tool, + find_ninja, + refresh_compile_commands, +) +from esphome.build_helpers.tools_cache import HOST_TOOLS_CACHE +from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME +from esphome.core import CORE, EsphomeError +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + +# The output name PlatformIO's native platform produced; CORE.firmware_bin +# and the integration-test harness resolve it by this name +PROGRAM_NAME = "program" + +# platformio_options keys the host build reads (lib_ignore feeds the library +# converter); anything else has no native equivalent and is warned about +CONSUMED_PIO_OPTIONS = frozenset({"lib_ignore"}) + + +class HostCompilers(NamedTuple): + """The resolved C and C++ compiler commands: the program, then any + arguments its override carried (``CC="gcc -m32"``).""" + + cc: tuple[str, ...] + cxx: tuple[str, ...] + + +def find_command(env_var: str, candidates: tuple[str, ...]) -> tuple[str, ...]: + """Resolve a build tool: ``env_var`` when set, else the first candidate + found on PATH. Returns the program and the override's arguments. + + An override that does not resolve fails by name rather than falling back + silently to a different compiler than the user asked for. + """ + if override := os.environ.get(env_var, "").strip(): + program, *args = shlex.split(override) + if Path(program).stem == "ccache" and args: + # The build adds ccache itself; twice would hide the compiler + program, *args = args + # which() accepts an absolute path as well as a bare program name + if (resolved := shutil.which(program)) is None: + raise EsphomeError( + f"{env_var}={override!r} does not name a runnable program" + ) + return (absolute_tool(resolved), *args) + for name in candidates: + if (found := shutil.which(name)) is not None: + return (absolute_tool(found),) + raise EsphomeError( + f"{candidates[0]} not found on PATH (tried {', '.join(candidates)}); " + f"install it or set {env_var}" + ) + + +def find_tool(env_var: str, candidates: tuple[str, ...]) -> str: + """Resolve a build tool that is called as a bare program.""" + program, *args = find_command(env_var, candidates) + if args: + raise EsphomeError(f"{env_var} must name a program without arguments") + return program + + +def find_compilers() -> HostCompilers: + """The C and C++ compilers the build uses (gcc first, as PlatformIO did).""" + return HostCompilers( + cc=find_command("CC", ("gcc", "clang", "cc")), + cxx=find_command("CXX", ("g++", "clang++", "c++")), + ) + + +def get_build_dir() -> Path: + return CORE.relative_pioenvs_path(CORE.name) + + +def get_elf_path() -> Path: + return get_build_dir() / PROGRAM_NAME + + +def get_objdump_path() -> Path: + return Path(find_tool("OBJDUMP", ("objdump",))) + + +def get_readelf_path() -> Path: + return Path(find_tool("READELF", ("readelf",))) + + +def check_analysis_supported() -> None: + """Refuse analyze-memory where the program is not an ELF file. + + Called before the compile, so an unsupported machine fails at once. + """ + if sys.platform != "linux": + raise EsphomeError( + "analyze-memory reads ELF files; the host build on " + f"{sys.platform} produces a different format" + ) + + +def get_build_env(ccache: str | None) -> dict[str, str]: + return {**os.environ, **ccache_env(ccache, HOST_TOOLS_CACHE)} + + +def run_compile(config: ConfigType, verbose: bool) -> int: + from esphome.build_gen import host as build_gen + + warn_ignored_platformio_options(CONSUMED_PIO_OPTIONS) + # Probe the cheap local dependencies before resolving libraries + ninja_path = find_ninja() + compilers = find_compilers() + # Resolved once per build: the resolution probes PATH and spawns the + # runnability check, and three consumers need the same answer + ccache = resolve_absolute_ccache_path() + ninja_changed = build_gen.write_project(compilers, ccache) + + build_dir = get_build_dir() + env = get_build_env(ccache) + refresh_compile_commands(ninja_path, build_dir, env, ninja_changed) + + cmd = [str(ninja_path)] + if verbose: + cmd.append("-v") + if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT): + cmd += ["-j", str(jobs)] + # The explicit target, not the default statement: a generator defect + # that drops it fails loudly with "unknown target" instead of a green + # no-op run that leaves a stale program in place + cmd.append(PROGRAM_NAME) + + _LOGGER.debug("Running: %s", " ".join(cmd)) + # cwd instead of -C also drops the "Entering directory" banner + rc = subprocess.run( + cmd, cwd=build_dir, env=env, check=False, close_fds=False + ).returncode + if rc != 0: + return rc + + elf = get_elf_path() + if not elf.is_file(): + # ninja refused a manifest missing the target above; this covers a + # rule that ran but wrote elsewhere + _LOGGER.error("Build produced no %s", elf) + return 1 + + from esphome.build_helpers.idedata import warn_if_idedata_missing + + warn_if_idedata_missing(lambda: _load_idedata(ccache)) + return 0 + + +def get_idedata() -> dict | None: + """Derive idedata from the build's compile_commands.json. + + Same contract as ``espidf.toolchain.get_idedata``: the fields IDE + integrations, clang-tidy, and the memory analyzer expect. Returns None + when nothing has been built yet. + """ + # Deliberately uncached: env/PATH can change between builds in a + # long-lived host process + return _load_idedata(resolve_absolute_ccache_path()) + + +def _load_idedata(ccache: str | None) -> dict | None: + from esphome.build_helpers.idedata import load_or_build_idedata + + return load_or_build_idedata( + get_build_dir() / "compile_commands.json", + get_elf_path(), + CORE.relative_internal_path("idedata", f"{CORE.name}.json"), + # The compile DB's commands carry the same ccache prefix the ninja + # rules were generated with + launcher=ccache, + ) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index fb6779b807..af2010711f 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -360,7 +360,7 @@ class LibraryBackend: """ platform: str | None - framework: str + framework: str | None emit: Callable[["ConvertedLibrary"], None] cache_key: str # Owner-less names this returns True for are skipped by the walk; @@ -368,6 +368,10 @@ class LibraryBackend: # reconciles provided_requests after resolving provides: Callable[[str], bool] | None = None provided_requests: set[str] = field(default_factory=set) + # Accept a library without library.json/library.properties, as + # PlatformIO does (its defaults: src/ or the root, plus include/). Off + # for backends whose emitted build files need the manifest. + manifest_optional: bool = False def ensure_list[T](obj: T | list[T]) -> list[T]: @@ -859,6 +863,11 @@ def _url_or_none(value: Any) -> str | None: return value if parsed.scheme and parsed.netloc else None +# URL schemes that name a local library folder; symlink:// is PlatformIO's +# spelling for one it links instead of copying, which is the same to us +_LOCAL_SCHEMES = ("file", "symlink") + + def _node_key( name: str | None, version: str | None, repository: str | None ) -> tuple[str, str, tuple[str | None, str | None]]: @@ -896,7 +905,7 @@ def _node_key( scheme = urlsplit(candidate).scheme except ValueError: scheme = "" - if scheme == "file" or _url_or_none(candidate): + if scheme in _LOCAL_SCHEMES or _url_or_none(candidate): name, repository = custom_name, candidate else: # Anything with ``://`` was meant to be a URL; failing it fast @@ -905,7 +914,7 @@ def _node_key( if repository: is_git_prefixed = repository.startswith("git+") split_result = urlsplit(repository.removeprefix("git+")) - if split_result.scheme == "file" and not is_git_prefixed: + if (scheme := split_result.scheme) in _LOCAL_SCHEMES and not is_git_prefixed: # A plain file:// URL points at a local library directory. A local # file URL is written file:///absolute/path (empty host) or, less # commonly, file://localhost/path. Anything else -- a real host, or @@ -913,8 +922,8 @@ def _node_key( # rejected rather than silently resolved to the wrong directory. if split_result.netloc not in ("", "localhost"): raise RuntimeError( - f"Unsupported host in file:// library URL '{repository}'; " - "use an absolute path, e.g. file:///path/to/lib" + f"Unsupported host in {scheme}:// library URL '{repository}'; " + f"use an absolute path, e.g. {scheme}:///path/to/lib" ) # Validate the URL path itself (always POSIX-style, leading slash), # not the OS path: on Windows a "/foo" path is not is_absolute() @@ -924,8 +933,8 @@ def _node_key( url_path = split_result.path if not url_path.startswith("/") or not PurePosixPath(url_path).name: raise RuntimeError( - f"file:// library URL '{repository}' must be an absolute " - "directory path, e.g. file:///path/to/lib" + f"{scheme}:// library URL '{repository}' must be an absolute " + f"directory path, e.g. {scheme}:///path/to/lib" ) path = url2pathname(url_path) return (name or PurePosixPath(url_path).name), "local", (path, None) @@ -1244,9 +1253,16 @@ def convert_libraries( library_properties_path = source_dir / "library.properties" has_json = library_json_path.is_file() has_properties = library_properties_path.is_file() - if not has_json and not has_properties and not node.is_local: + if ( + not has_json + and not has_properties + and not node.is_local + and not backend.manifest_optional + ): # An interrupted clone/extraction self-heals with one forced - # re-download; a local source has nothing to re-download + # re-download; a local source has nothing to re-download. + # A backend accepting manifest-less libraries cannot tell + # one from a torn download and would re-fetch every build. _LOGGER.warning( "Library %s at %s is missing library.json and library.properties; " "re-downloading", @@ -1260,6 +1276,12 @@ def convert_libraries( component.data = parse_library_json(library_json_path) elif has_properties: component.data = parse_library_properties(library_properties_path) + elif backend.manifest_optional: + _LOGGER.debug( + "Library %s has no manifest; using PlatformIO's default layout", + key, + ) + component.data = {"name": component.name} else: # Local sources are user input (EsphomeError); a registry/git # miss means a corrupt cache (RuntimeError) @@ -1286,7 +1308,7 @@ def convert_libraries( # cross-platform skip stays at debug, other causes warn if key in top_level_keys: reason = ( - f"is not compatible with {backend.framework}" + f"is not compatible with {backend.framework or backend.platform}" if isinstance(e, IncompatiblePlatform) else "has a malformed manifest" ) diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 9219914529..adaa809d6e 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -155,7 +155,7 @@ class StorageJSON: self.framework = framework # The core platform of this firmware. Like "esp32", "rp2", "host" etc. self.core_platform = core_platform - # The toolchain used for the build ("platformio" / "esp-idf") + # The toolchain used for the build ("platformio" / "esp-idf" / "host") self.toolchain = toolchain # The area of the node self.area = area diff --git a/requirements.txt b/requirements.txt index ff0ad9c944..81f3cbcdd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.12 # native esp-idf toolchain global cache dir -ninja==1.13.2 # native esp8266 arduino toolchain build driver +ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.1 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this diff --git a/script/build_helpers.py b/script/build_helpers.py index b4b25924c3..f0b354a8a7 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -22,8 +22,8 @@ from esphome.__main__ import command_compile, parse_args from esphome.config import validate_config from esphome.const import CONF_PLATFORM from esphome.core import CORE +from esphome.host.toolchain import get_elf_path from esphome.loader import get_component, get_platform -from esphome.platformio.toolchain import get_idedata from tests.testing_helpers import ComponentManifestOverride, set_testing_manifest # This must coincide with the version in /platformio.ini @@ -220,7 +220,7 @@ def create_host_config( friendly_name: str, libraries: str | list[str], includes: list[str], - platformio_options: dict, + build_flags: list[str], ) -> dict: """Create an ESPHome host configuration for C++ builds. @@ -229,7 +229,7 @@ def create_host_config( friendly_name: Human-readable name libraries: PlatformIO library specification(s) includes: List of include folders for the build - platformio_options: Dict of platformio_options to set + build_flags: Compiler/linker flags for the build Returns: Configuration dict for ESPHome @@ -239,7 +239,7 @@ def create_host_config( "name": config_name, "friendly_name": friendly_name, "libraries": libraries, - "platformio_options": platformio_options, + "build_flags": build_flags, "includes": includes, }, HOST_KEY: {}, @@ -403,13 +403,12 @@ def compile_and_get_binary( return EXIT_COMPILE_ERROR, None # After a successful compilation, locate the executable: - idedata = get_idedata(config) - if idedata is None: + program_path = get_elf_path() + if not program_path.is_file(): print("Cannot find executable") return EXIT_NO_EXECUTABLE, None - program_path: str = idedata.raw["prog_path"] - return EXIT_OK, program_path + return EXIT_OK, str(program_path) def build_and_run( @@ -419,7 +418,7 @@ def build_and_run( config_prefix: str, friendly_name: str, libraries: str | list[str], - platformio_options: dict, + build_flags: list[str], main_entry: str, label: str = "build", build_only: bool = False, @@ -438,7 +437,7 @@ def build_and_run( config_prefix: Prefix for the config name (e.g. "cpptests", "cppbench") friendly_name: Human-readable name for the config libraries: PlatformIO library specification(s) - platformio_options: PlatformIO options dict + build_flags: Compiler/linker flags for the build main_entry: Name of the main entry file (e.g. "main.cpp") label: Label for log messages build_only: If True, print binary path and return without running @@ -488,7 +487,7 @@ def build_and_run( config_name: str = f"{config_prefix}-" + hash_components(components) config = create_host_config( - config_name, friendly_name, libraries, includes, platformio_options + config_name, friendly_name, libraries, includes, build_flags ) exit_code, program_path = compile_and_get_binary( diff --git a/script/cpp_benchmark.py b/script/cpp_benchmark.py index 5080a9fec7..678356496e 100755 --- a/script/cpp_benchmark.py +++ b/script/cpp_benchmark.py @@ -25,19 +25,14 @@ CORE_BENCHMARKS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "core" # allow benchmarks to compile on the host platform. STUBS_DIR: Path = Path(root_path) / "tests" / "benchmarks" / "stubs" -PLATFORMIO_OPTIONS = { - "build_flags": [ - "-Os", # match firmware optimization level (detects inlining regressions) - "-g", # debug symbols for profiling - "-ffunction-sections", # required for dead-code stripping with -Os - "-fdata-sections", # required for dead-code stripping with -Os - "-DUSE_BENCHMARK", # disable WarnIfComponentBlockingGuard in finish() - f"-I{STUBS_DIR}", # stub headers for ESP32-only components - ], - # Use deep+ LDF mode to ensure PlatformIO detects the benchmark - # library dependency from nested includes. - "lib_ldf_mode": "deep+", -} +BUILD_FLAGS = [ + "-Os", # match firmware optimization level (detects inlining regressions) + "-g", # debug symbols for profiling + "-ffunction-sections", # required for dead-code stripping with -Os + "-fdata-sections", # required for dead-code stripping with -Os + "-DUSE_BENCHMARK", # disable WarnIfComponentBlockingGuard in finish() + f"-I{STUBS_DIR}", # stub headers for ESP32-only components +] def run_benchmarks(selected_components: list[str], build_only: bool = False) -> int: @@ -46,7 +41,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> # containing {"lib_path": "/path/to/google_benchmark"}. lib_config_json = os.environ.get("BENCHMARK_LIB_CONFIG") - pio_options = PLATFORMIO_OPTIONS + build_flags = BUILD_FLAGS if lib_config_json: lib_config = json.loads(lib_config_json) benchmark_lib = f"benchmark=symlink://{lib_config['lib_path']}" @@ -61,10 +56,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> "-DCODSPEED_ANALYSIS", f'-DCODSPEED_ROOT_DIR=\\"{project_root}\\"', ] - pio_options = { - **PLATFORMIO_OPTIONS, - "build_flags": PLATFORMIO_OPTIONS["build_flags"] + codspeed_flags, - } + build_flags = BUILD_FLAGS + codspeed_flags else: benchmark_lib = PLATFORMIO_GOOGLE_BENCHMARK_LIB @@ -77,7 +69,7 @@ def run_benchmarks(selected_components: list[str], build_only: bool = False) -> config_prefix="cppbench", friendly_name="CPP Benchmarks", libraries=benchmark_lib, - platformio_options=pio_options, + build_flags=build_flags, main_entry="main.cpp", label="benchmarks", build_only=build_only, diff --git a/script/cpp_unit_test.py b/script/cpp_unit_test.py index 8cb18d0875..c5179a1b1e 100755 --- a/script/cpp_unit_test.py +++ b/script/cpp_unit_test.py @@ -15,24 +15,16 @@ from helpers import get_all_components, root_path # Path to /tests/components COMPONENTS_TESTS_DIR: Path = Path(root_path) / "tests" / "components" -PLATFORMIO_OPTIONS = { - "build_type": "debug", - "build_unflags": [ - "-Os", # remove size-opt flag - ], - "build_flags": [ - "-Og", # optimize for debug - "-DESPHOME_DEBUG", # enable debug assertions - # Enable the address and undefined behavior sanitizers - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer", - ], - "debug_build_flags": [ # only for debug builds - "-g3", # max debug info - "-ggdb3", - ], -} +BUILD_FLAGS = [ + "-Og", # optimize for debug + "-g3", # max debug info + "-ggdb3", + "-DESPHOME_DEBUG", # enable debug assertions + # Enable the address and undefined behavior sanitizers + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer", +] def run_tests(selected_components: list[str]) -> int: @@ -47,7 +39,7 @@ def run_tests(selected_components: list[str]) -> int: config_prefix="cpptests", friendly_name="CPP Unit Tests", libraries=PLATFORMIO_GOOGLE_TEST_LIB, - platformio_options=PLATFORMIO_OPTIONS, + build_flags=BUILD_FLAGS, main_entry="main.cpp", label="unit tests", ) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index a45951a52a..3f0143076b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -111,14 +111,25 @@ INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 INTEGRATION_TESTS_SPLIT_BUCKETS = 5 INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0 -# platformio and aioesphomeapi (requirements.txt), the pytest stack -# (requirements_test.txt) and the fixture every session compiles; a change -# to any runs the full matrix +# aioesphomeapi (requirements.txt), the pytest stack (requirements_test.txt) +# and the native host build backend every test compiles with; a change to +# any runs the full matrix INTEGRATION_TESTS_TRIGGER_FILES = frozenset( { "requirements.txt", "requirements_test.txt", - "tests/integration/fixtures/cache_init.yaml", + "esphome/arduino/library.py", + "esphome/build_gen/build_tool.py", + "esphome/build_gen/host.py", + "esphome/build_helpers/ccache.py", + "esphome/build_helpers/idedata.py", + "esphome/build_helpers/native.py", + "esphome/build_helpers/ninja.py", + "esphome/build_helpers/ninja_gen.py", + "esphome/build_helpers/tools_cache.py", + "esphome/framework_helpers.py", + "esphome/host/toolchain.py", + "esphome/platformio/library.py", } ) @@ -240,7 +251,7 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s - conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc. 4. A file in INTEGRATION_TESTS_TRIGGER_FILES changed - - The dependency pins and the session init fixture affect every test + - The dependency pins and the host build backend affect every test Returns (run_all=False, [test_files...]) when: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 78e0b1a36c..ce2e7f5bbc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,7 +16,6 @@ import re import shutil import signal import socket -import subprocess import sys import tempfile import time @@ -35,7 +34,7 @@ from esphome.helpers import ( write_file, write_file_if_changed, ) -from esphome.platformio.toolchain import get_idedata +from esphome.host.toolchain import get_elf_path from .const import ( API_CONNECTION_TIMEOUT, @@ -82,16 +81,9 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests" -def _get_platformio_env(cache_dir: Path) -> dict[str, str]: - """Get environment variables for PlatformIO with shared cache.""" +def _get_build_env() -> dict[str, str]: + """Environment for an ``esphome compile`` subprocess.""" env = os.environ.copy() - env["PLATFORMIO_CORE_DIR"] = str(cache_dir) - env["PLATFORMIO_CACHE_DIR"] = str(cache_dir / ".cache") - # libdeps is keyed only by env name (the device name), and fixtures share - # names; two xdist workers first-compiling the same name race pio pkg - # install in the same directory. Keep libdeps per worker. - worker = os.environ.get("PYTEST_XDIST_WORKER", "master") - env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker) # Prevent cache cleaning during integration tests env["ESPHOME_SKIP_CLEAN_BUILD"] = "1" # Cap each compile's -j so several xdist workers do not each spawn a @@ -110,53 +102,41 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: return env +# Registry libraries (noise-c, lvgl, ...) download into /pio_components, +# and every integration test gets its own data dir. Share one download cache +# per xdist worker instead: the converter has no cross-process lock, so +# workers must never write the same dir, while a worker's later tests (and +# later runs) reuse what it fetched. +LIBRARY_CACHE_ROOT = INTEGRATION_TESTS_ROOT / "pio_components" + + @pytest.fixture(scope="session") -def shared_platformio_cache() -> Generator[Path]: - """Initialize a shared PlatformIO cache for all integration tests.""" - # Use a dedicated directory for integration tests to avoid conflicts. - test_cache_dir = INTEGRATION_TESTS_ROOT - cache_dir = test_cache_dir / "platformio" +def shared_library_cache() -> Path: + """This worker's shared registry-library download cache.""" + worker = os.environ.get("PYTEST_XDIST_WORKER", "master") + # Keyed by checkout too: two sessions on one machine share worker ids + cache_dir = LIBRARY_CACHE_ROOT / _REPO_KEY / worker + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir - # Use a lock file in the home directory to ensure only one process initializes the cache - # This is needed when running with pytest-xdist - # The lock file must be in a directory that already exists to avoid race conditions - lock_file = Path.home() / ".esphome-integration-tests-init.lock" - # Always acquire the lock to ensure cache is ready before proceeding - with lock_file.open("w") as lock_fd: - fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) - - # Check if the native platform is installed (the actual indicator of a populated cache) - native_platform = cache_dir / "platforms" / "native" - if not native_platform.exists(): - # Create the test cache directory if it doesn't exist - test_cache_dir.mkdir(exist_ok=True) - - with tempfile.TemporaryDirectory() as tmpdir: - # Use the cache_init fixture for initialization - init_dir = Path(tmpdir) - fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml" - config_path = init_dir / "cache_init.yaml" - config_path.write_text( - fixture_path.read_text(encoding="utf-8"), encoding="utf-8" - ) - - # Run compilation to populate the cache - # We must succeed here to avoid race conditions where multiple - # tests try to populate the same cache directory simultaneously - env = _get_platformio_env(cache_dir) - - subprocess.run( - [sys.executable, "-m", "esphome", "compile", str(config_path)], - check=True, - cwd=init_dir, - env=env, - close_fds=False, - ) - - # Lock is held until here, ensuring cache is fully populated before any test proceeds - - yield cache_dir +def _link_library_cache(config_dir: Path, cache_dir: Path) -> None: + """Point a config dir's library download cache at the shared one + (blocking, run in executor).""" + data_dir = config_dir / ".esphome" + data_dir.mkdir(exist_ok=True) + link = data_dir / "pio_components" + if link.is_symlink(): + if link.resolve() == cache_dir.resolve(): + return + # A shared build dir was linked by another worker; writing through + # it would put two workers in one cache + link.unlink() + elif link.exists(): + # A real dir from a run predating the shared cache; nothing in it is + # worth more than a re-download + rmtree(link) + link.symlink_to(cache_dir, target_is_directory=True) @pytest.fixture(scope="module", autouse=True) @@ -238,18 +218,17 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s content = content.replace("api:", f"api:\n port: {unused_tcp_port}") # Add debug build flags for integration tests to enable assertions - if "esphome:" in content and "platformio_options:" not in content: - # Add platformio_options with debug flags after esphome: + if "esphome:" in content and "build_flags:" not in content: + # Add build_flags with debug flags after esphome: content = content.replace( "esphome:", "esphome:\n" " # Enable assertions for integration tests\n" - " platformio_options:\n" - " build_flags:\n" - ' - "-DDEBUG" # Enable assert() statements\n' - ' - "-DESPHOME_DEBUG" # Enable ESPHOME_DEBUG_ASSERT checks\n' - ' - "-DESPHOME_DEBUG_API" # Enable API protocol asserts\n' - ' - "-g" # Add debug symbols', + " build_flags:\n" + ' - "-DDEBUG" # Enable assert() statements\n' + ' - "-DESPHOME_DEBUG" # Enable ESPHOME_DEBUG_ASSERT checks\n' + ' - "-DESPHOME_DEBUG_API" # Enable API protocol asserts\n' + ' - "-g" # Add debug symbols', ) # Replace external component path placeholder if present @@ -283,8 +262,8 @@ async def write_yaml_config( yield _write_config -# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale -# dirs for a fixture are pruned when its content hash changes. +# Deliberately not CI-cached; stale dirs for a fixture are pruned when its +# content hash changes. SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds" # In the dir name (not just the hash) so pruning stays inside this checkout @@ -470,8 +449,7 @@ def _resolve_compiled_binary(config_path: Path) -> Path: ) if config is None: raise RuntimeError(f"Failed to read config from {config_path}") - idedata = get_idedata(config) - binary_path = Path(idedata.firmware_elf_path) + binary_path = get_elf_path() if not binary_path.exists(): raise RuntimeError(f"Compiled binary not found at {binary_path}") return binary_path @@ -480,19 +458,22 @@ def _resolve_compiled_binary(config_path: Path) -> Path: @pytest_asyncio.fixture async def compile_esphome( integration_test_dir: Path, - shared_platformio_cache: Path, + shared_library_cache: Path, request: pytest.FixtureRequest, ) -> AsyncGenerator[CompileFunction]: """Compile an ESPHome configuration and return the binary path.""" async def _compile(config_path: Path) -> Path: - # Use the shared PlatformIO cache for faster compilation - # This avoids re-downloading dependencies for each test - env = _get_platformio_env(shared_platformio_cache) + env = _get_build_env() loop = asyncio.get_running_loop() name = _shared_yaml_name(request) if name is None: + # Share the library download cache so a test never re-fetches + # what an earlier one already pulled from the registry + await loop.run_in_executor( + None, _link_library_cache, integration_test_dir, shared_library_cache + ) await _run_esphome_compile(config_path, integration_test_dir, env) return await loop.run_in_executor( None, _resolve_compiled_binary, config_path @@ -568,6 +549,9 @@ async def compile_esphome( await loop.run_in_executor( None, write_file_if_changed, shared_config, content ) + await loop.run_in_executor( + None, _link_library_cache, shared_dir, shared_library_cache + ) await _run_esphome_compile(shared_config, shared_dir, env) if built is None or not built.exists(): built = await loop.run_in_executor( diff --git a/tests/integration/fixtures/api_action_timeout.yaml b/tests/integration/fixtures/api_action_timeout.yaml index 405d9d0e2b..5dc6e1db05 100644 --- a/tests/integration/fixtures/api_action_timeout.yaml +++ b/tests/integration/fixtures/api_action_timeout.yaml @@ -1,9 +1,8 @@ esphome: name: api-action-timeout-test # Use a short timeout for testing (500ms instead of 30s) - platformio_options: - build_flags: - - "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500" + build_flags: + - "-DUSE_API_ACTION_CALL_TIMEOUT_MS=500" host: diff --git a/tests/integration/fixtures/cache_init.yaml b/tests/integration/fixtures/cache_init.yaml deleted file mode 100644 index de208196cd..0000000000 --- a/tests/integration/fixtures/cache_init.yaml +++ /dev/null @@ -1,10 +0,0 @@ -esphome: - name: cache-init - -host: - -api: - encryption: - key: "IIevImVI42I0FGos5nLqFK91jrJehrgidI0ArwMLr8w=" - -logger: diff --git a/tests/integration/fixtures/fnv1_hash_object_id.yaml b/tests/integration/fixtures/fnv1_hash_object_id.yaml index 2097b2fbf9..05449afe86 100644 --- a/tests/integration/fixtures/fnv1_hash_object_id.yaml +++ b/tests/integration/fixtures/fnv1_hash_object_id.yaml @@ -1,8 +1,7 @@ esphome: name: fnv1-hash-object-id-test - platformio_options: - build_flags: - - "-DDEBUG" + build_flags: + - "-DDEBUG" on_boot: - lambda: |- using esphome::fnv1_hash_object_id; diff --git a/tests/integration/fixtures/fnv1a_hash.yaml b/tests/integration/fixtures/fnv1a_hash.yaml index d9c80601b8..3da0042e50 100644 --- a/tests/integration/fixtures/fnv1a_hash.yaml +++ b/tests/integration/fixtures/fnv1a_hash.yaml @@ -1,8 +1,7 @@ esphome: name: fnv1a-hash-test - platformio_options: - build_flags: - - "-DDEBUG" + build_flags: + - "-DDEBUG" on_boot: - lambda: |- using esphome::fnv1a_hash; diff --git a/tests/integration/fixtures/micros_to_millis.yaml b/tests/integration/fixtures/micros_to_millis.yaml index d11808c43a..a7ba1c719a 100644 --- a/tests/integration/fixtures/micros_to_millis.yaml +++ b/tests/integration/fixtures/micros_to_millis.yaml @@ -1,8 +1,7 @@ esphome: name: micros-to-millis-test - platformio_options: - build_flags: - - "-DDEBUG" + build_flags: + - "-DDEBUG" on_boot: - lambda: |- using esphome::micros_to_millis; diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 8f82a121c6..ded7ef0a72 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2143,10 +2143,7 @@ def test_no_orphan_integration_fixtures() -> None: fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve() fixtures = list(fixtures_dir.glob("*.yaml")) assert fixtures, f"no fixtures found under {fixtures_dir}" - # cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead - orphans = [ - f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping - ] + orphans = [f.stem for f in fixtures if f.stem not in mapping] assert not orphans, f"fixtures invisible to CI test selection: {orphans}" diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 4ad88347e2..65a78c0253 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -604,7 +604,7 @@ def test_write_project_libraries_and_variant( ) _set_flags("-DPIO_FRAMEWORK_ARDUINO_ENABLE_EXCEPTIONS") - with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.arduino8266"): + with caplog.at_level(logging.DEBUG, logger="esphome.build_helpers.ninja_gen"): content = _write_ninja( paths, libraries=[library, headers_only], ccache="/cc/ccache" ) @@ -616,6 +616,9 @@ def test_write_project_libraries_and_variant( assert "libHeadersOnly.a" not in content assert "Library HeadersOnly has no source files" in caplog.text assert " flags = -DMYLIB=1" in content + # A library's own include dirs lead its compile lines + assert " own_includes = -I" in content + assert "$own_includes $cxxflags $flags" in content assert "-lalgobsec" in content # Library link flags reach the firmware link line; .cc compiles as C++ assert "-Wl,--wrap=malloc" in content diff --git a/tests/unit_tests/build_gen/test_host.py b/tests/unit_tests/build_gen/test_host.py new file mode 100644 index 0000000000..d7fa73e897 --- /dev/null +++ b/tests/unit_tests/build_gen/test_host.py @@ -0,0 +1,368 @@ +"""Tests for esphome.build_gen.host (the host ninja generator).""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino.library import ArduinoLibrary +from esphome.build_gen import host as build_gen +from esphome.build_helpers.ninja import escape as _e, quote_path as _q +from esphome.build_helpers.ninja_gen import Flag +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_HOST +from esphome.core import CORE, EsphomeError, Library +from esphome.host.toolchain import PROGRAM_NAME, HostCompilers + +COMPILERS = HostCompilers(cc=("/usr/bin/gcc", "-m32"), cxx=("/usr/bin/g++",)) + + +@pytest.fixture(autouse=True) +def _core(tmp_path: Path) -> None: + CORE.build_path = tmp_path + CORE.name = "dev" + CORE.cpp_standard = "gnu++20" + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_HOST} + + +def _make_src(tmp_path: Path, *names: str) -> Path: + src = tmp_path / "src" + for name in names: + path = src / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + return src + + +def _render(ccache: str | None = None) -> tuple[bool, str]: + changed = build_gen.write_project(COMPILERS, ccache) + ninja = CORE.build_path / ".pioenvs" / "dev" / "build.ninja" + return changed, ninja.read_text() + + +@pytest.mark.parametrize( + ("entries", "compile_flags", "link_flags"), + [ + ( + ["-DUSE_HOST", "-Iinc", "-Ufoo", "-Wall"], + [("-DUSE_HOST",), ("-Iinc",), ("-Ufoo",), ("-Wall",)], + [], + ), + (["-std=gnu++20"], [("-std=gnu++20",)], []), + ( + ["-lssl", "-L/opt/lib", "-Wl,--gc-sections"], + [], + [("-lssl",), ("-L/opt/lib",), ("-Wl,--gc-sections",)], + ), + # A link-only flag travels with its argument + ( + ["-Xlinker --wrap=malloc", "-z noexecstack", "-framework Cocoa"], + [], + [ + ("-Xlinker", "--wrap=malloc"), + ("-z", "noexecstack"), + ("-framework", "Cocoa"), + ], + ), + # Both lines, as SCons routes unclassified flags + ( + ["-g -O2", "-fsanitize=address", "-pthread", "--coverage"], + [ + ("-g",), + ("-O2",), + ("-fsanitize=address",), + ("-pthread",), + ("--coverage",), + ], + [ + ("-g",), + ("-O2",), + ("-fsanitize=address",), + ("-pthread",), + ("--coverage",), + ], + ), + # A flag with a path argument travels with it, glued on or not + ( + ["-include pre.h", "-isystem /x", "-isystem/y", "-iquotez"], + [("-include", "pre.h"), ("-isystem", "/x"), ("-isystem/y",), ("-iquotez",)], + [], + ), + ], +) +def test_split_flags( + entries: list[str], compile_flags: list[Flag], link_flags: list[Flag] +) -> None: + flags = build_gen.parse_flags(entries, "esphome") + assert build_gen.split_flags(flags) == (compile_flags, link_flags) + + +@pytest.mark.parametrize( + "entries", + [ + # The flag is the last token of its entry + ["-g -include"], + # Entries are a set: the next entry is not this flag's argument + ["-include", "pre.h"], + ["-framework", "-pthread"], + # A path never starts with "-" + ["-include -pthread"], + ], +) +def test_parse_flags_refuses_a_flag_without_its_argument(entries: list[str]) -> None: + with pytest.raises(EsphomeError, match="with no argument; write the flag"): + build_gen.parse_flags(entries, "esphome") + + +def test_parse_flags_accepts_a_linker_argument_that_is_a_flag() -> None: + assert build_gen.parse_flags(["-Xlinker --wrap=malloc"], "esphome") == [ + ("-Xlinker", "--wrap=malloc") + ] + + +def test_flag_lists_route_the_standard() -> None: + """cpp_standard wins for C++; C never sees a C++ standard.""" + CORE.build_flags = { + "-DUSE_HOST", + "-Iinc", + "-Wl,-x", + "-g", + "-lssl", + "-std=c++17", + "-std=gnu17", + } + CORE.cxx_build_flags = {"-Wno-volatile"} + cflags, cxxflags, link_flags = build_gen._flag_lists() + # The relative include is anchored at the build path + inc = f"-I{CORE.build_path / 'inc'}" + assert cflags == ["-DUSE_HOST", inc, "-g", "-std=gnu17"] + assert cxxflags == ["-std=gnu++20", "-DUSE_HOST", inc, "-g", "-Wno-volatile"] + assert link_flags == ["-Wl,-x", "-g", "-lssl"] + + +def test_flag_lists_without_cpp_standard_keeps_user_std() -> None: + CORE.cpp_standard = None + CORE.build_flags = {"-std=c++17"} + cflags, cxxflags, _link = build_gen._flag_lists() + assert cflags == [] + assert cxxflags == ["-std=c++17"] + + +def test_flag_lists_apply_unflags(caplog: pytest.LogCaptureFixture) -> None: + CORE.build_flags = {"-g", "-DUSE_HOST"} + CORE.build_unflags = {"-g", "-Onope"} + with caplog.at_level(logging.WARNING): + cflags, cxxflags, link_flags = build_gen._flag_lists() + assert "-g" not in cflags + cxxflags + link_flags + assert "-DUSE_HOST" in cflags + assert "build_unflags entries matched no build flag: -Onope" in caplog.text + + +def test_resolve_host_libraries_without_libraries() -> None: + with patch("esphome.arduino.library.resolve_libraries") as resolve: + assert build_gen._resolve_host_libraries() == [] + resolve.assert_not_called() + + +def test_resolve_host_libraries_is_framework_less() -> None: + CORE.add_library(Library(name="lvgl/lvgl", version="9.5.0")) + lib = ArduinoLibrary(name="lvgl") + with patch( + "esphome.arduino.library.resolve_libraries", return_value=[lib] + ) as resolve: + assert build_gen._resolve_host_libraries() == [lib] + resolve.assert_called_once_with( + None, + pio_platform="native", + board_mcu="host", + cache_key="host", + framework=None, + manifest_optional=True, + ) + + +def test_write_project_requires_generated_sources(tmp_path: Path) -> None: + with pytest.raises(EsphomeError, match="Generated source directory .* is missing"): + build_gen.write_project(COMPILERS, None) + _make_src(tmp_path, "esphome.h") + with pytest.raises(EsphomeError, match="No source files found"): + build_gen.write_project(COMPILERS, None) + + +def test_write_project_emits_every_source_kind(tmp_path: Path) -> None: + src = _make_src(tmp_path, "main.cpp", "esphome/core/a.c", "x.S", "y.s", "h.h") + CORE.build_flags = {"-DUSE_HOST", "-g"} + changed, ninja = _render(ccache="/usr/bin/ccache") + assert changed is True + # A compiler override's arguments follow the program + assert f"cc = {_q('/usr/bin/gcc')} -m32\n" in ninja + assert f"cxx = {_q('/usr/bin/g++')}" in ninja + assert f"ccache = {_q('/usr/bin/ccache')}" in ninja + assert f"build obj/src/main.cpp.o: cxx {_e(src / 'main.cpp')}" in ninja + assert ( + f"build obj/src/esphome/core/a.c.o: c {_e(src / 'esphome/core/a.c')}" in ninja + ) + assert "build obj/src/x.S.o: aspp " in ninja + assert "build obj/src/y.s.o: asm " in ninja + assert "h.h" not in ninja + # __FILE__ stays relative to the build path, as it was under PlatformIO, + # whether the source arrives absolute or rewritten by ccache + sep = os.sep + maps = ( + f"{_q(f'-fmacro-prefix-map={tmp_path}{sep}=')} " + f"{_q(f'-fmacro-prefix-map=..{sep}..{sep}=')}" + ) + assert f"cflags = -DUSE_HOST -g -I{_q(src)} {maps}\n" in ninja + assert f"cxxflags = -std=gnu++20 -DUSE_HOST -g -I{_q(src)} {maps}\n" in ninja + # Assembly gets the defines and includes only + assert f"asflags = -DUSE_HOST -I{_q(src)} {maps}\n" in ninja + assert "linkflags = -g\n" in ninja + assert "libdirflags = \n" in ninja + assert "libflags = \n" in ninja + assert "rule ar" not in ninja + assert ( + f"build {PROGRAM_NAME}: link obj/src/esphome/core/a.c.o obj/src/main.cpp.o " + f"obj/src/x.S.o obj/src/y.s.o | \n archives = \ndefault {PROGRAM_NAME}\n" + ) in ninja + # Unchanged content reports no change so the compile DB can be reused + changed, _ = _render(ccache="/usr/bin/ccache") + assert changed is False + + +def test_write_project_without_ccache(tmp_path: Path) -> None: + _make_src(tmp_path, "main.cpp") + _changed, ninja = _render() + assert "ccache = \n" in ninja + + +def test_write_project_routes_user_link_flags(tmp_path: Path) -> None: + _make_src(tmp_path, "main.cpp") + lib_dir = tmp_path / "opt" / "lib" + # Forward slashes: build flags are lexed like a POSIX shell line, which + # reads a backslash as an escape + CORE.build_flags = { + f"-L{lib_dir.as_posix()}", + "-lcrypto", + "-Wl,-framework,Security", + } + _changed, ninja = _render() + assert "linkflags = -Wl,-framework,Security\n" in ninja + assert f"libdirflags = -L{_q(lib_dir)}\n" in ninja + assert "libflags = -lcrypto\n" in ninja + + +def _libraries(tmp_path: Path) -> list[ArduinoLibrary]: + lib_dir = tmp_path / "libs" + for name in ("foo/src/a.cpp", "foo/src/sub/b.c", "bare/x.cpp"): + path = lib_dir / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + archived = ArduinoLibrary( + name="foo", + sources=[lib_dir / "foo/src/a.cpp", lib_dir / "foo/src/sub/b.c"], + include_dirs=[lib_dir / "foo/src"], + flags=["-DFOO=1"], + link_dirs=[lib_dir / "foo/lib"], + link_libs=["bar"], + link_flags=["-Wl,--gc-sections"], + ) + direct = ArduinoLibrary( + name="bare", sources=[lib_dir / "bare/x.cpp"], lib_archive=False + ) + header_only = ArduinoLibrary(name="hdr", include_dirs=[lib_dir / "hdr"]) + return [archived, direct, header_only] + + +def test_write_project_with_libraries( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + src = _make_src(tmp_path, "main.cpp") + libs = _libraries(tmp_path) + with ( + patch.object(build_gen, "_resolve_host_libraries", return_value=libs), + patch.object(build_gen, "find_tool", return_value="/usr/bin/ar") as find_tool, + patch.object(build_gen.sys, "platform", "linux"), + caplog.at_level(logging.DEBUG), + ): + _changed, ninja = _render() + find_tool.assert_called_once_with("AR", ("ar",)) + assert ( + f"rule ar\n command = $python $buildtool ar {_q('/usr/bin/ar')} $out $out.rsp" + in ninja + ) + lib_dir = tmp_path / "libs" + # Every library's include dir joins the global include path + assert f"-I{_q(src)} -I{_q(lib_dir / 'foo/src')} -I{_q(lib_dir / 'hdr')}" in ninja + assert "linkflags = -Wl,--gc-sections\n" in ninja + assert f"libdirflags = -L{_q(lib_dir / 'foo/lib')}\n" in ninja + assert "libflags = -lbar\n" in ninja + # Library sources compile with the library's own flags, rooted at their + # common parent; its own include dirs lead the line so another library's + # header of the same name cannot shadow them + own = f"-I{_q(lib_dir / 'foo/src')}" + assert "$own_includes $cxxflags $flags" in ninja + assert ( + f"build obj/lib/foo/a.cpp.o: cxx {_e(lib_dir / 'foo/src/a.cpp')}\n" + f" own_includes = {own}\n flags = -DFOO=1\n" in ninja + ) + assert ( + f"build obj/lib/foo/sub/b.c.o: c {_e(lib_dir / 'foo/src/sub/b.c')}\n" + f" own_includes = {own}\n flags = -DFOO=1\n" in ninja + ) + assert "build libfoo.a: ar obj/lib/foo/a.cpp.o obj/lib/foo/sub/b.c.o\n" in ninja + # libArchive: false objects link directly; the archive is an order-only + # input wrapped in a group for GNU ld + assert ( + f"build {PROGRAM_NAME}: link obj/src/main.cpp.o obj/lib/bare/x.cpp.o " + "| libfoo.a\n" + " archives = -Wl,--start-group libfoo.a -Wl,--end-group\n" + ) in ninja + assert "Library hdr has no source files" in caplog.text + + +def test_write_project_darwin_links_archives_bare(tmp_path: Path) -> None: + _make_src(tmp_path, "main.cpp") + libs = _libraries(tmp_path) + with ( + patch.object(build_gen, "_resolve_host_libraries", return_value=libs), + patch.object(build_gen, "find_tool", return_value="/usr/bin/ar"), + patch.object(build_gen.sys, "platform", "darwin"), + ): + _changed, ninja = _render() + assert " archives = libfoo.a\n" in ninja + assert "--start-group" not in ninja + + +def test_build_unflags_remove_a_flag_with_its_argument( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the named pair goes; the same path under another flag stays.""" + CORE.build_flags = {"-iquote inc", "-isystem inc", "-Xlinker -dead_strip", "-g"} + CORE.build_unflags = {"-isystem inc", "-Xlinker -dead_strip"} + with caplog.at_level(logging.WARNING): + cflags, _cxxflags, link_flags = build_gen._flag_lists() + assert cflags == ["-g", "-iquote", str(CORE.build_path / "inc")] + assert link_flags == ["-g"] + assert "matched no build flag" not in caplog.text + + +def test_build_unflags_warn_when_nothing_matches( + caplog: pytest.LogCaptureFixture, +) -> None: + CORE.build_flags = {"-g"} + CORE.build_unflags = {"-isystem inc", "-DNOPE"} + with caplog.at_level(logging.WARNING): + build_gen._flag_lists() + assert "matched no build flag: -DNOPE, -isystem " in caplog.text + + +def test_build_unflags_match_anchored_paths(caplog: pytest.LogCaptureFixture) -> None: + CORE.build_flags = {"-Iinc", "-g"} + CORE.build_unflags = {"-Iinc"} + with caplog.at_level(logging.WARNING): + cflags, _cxxflags, _link = build_gen._flag_lists() + assert cflags == ["-g"] + assert "matched no build flag" not in caplog.text diff --git a/tests/unit_tests/build_helpers/test_ccache.py b/tests/unit_tests/build_helpers/test_ccache.py index 0237db4081..468a97daab 100644 --- a/tests/unit_tests/build_helpers/test_ccache.py +++ b/tests/unit_tests/build_helpers/test_ccache.py @@ -120,3 +120,48 @@ def test_parse_enable_env_spelling_tables( """cv.boolean's spelling tables plus the 1/0 env convention.""" monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", raw) assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected + + +def test_resolve_absolute_ccache_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Anchor the path: ninja runs from the build directory.""" + monkeypatch.chdir(tmp_path) + with patch.object(ccache, "resolve_ccache_path", return_value="bin/ccache"): + resolved = ccache.resolve_absolute_ccache_path() + assert Path(resolved) == tmp_path / "bin" / "ccache" + with patch.object(ccache, "resolve_ccache_path", return_value=None): + assert ccache.resolve_absolute_ccache_path() is None + + +def test_ccache_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from esphome.core import CORE + + CORE.build_path = tmp_path / "build" + spec = ("ESPHOME_TEST_PREFIX", "test") + monkeypatch.setenv("ESPHOME_TEST_PREFIX", str(tmp_path / "cache")) + monkeypatch.setenv("CCACHE_NOHASHDIR", "false") + for key in ("CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEPEND"): + monkeypatch.delenv(key, raising=False) + # None means resolved and disabled + assert ccache.ccache_env(None, spec) == {} + env = ccache.ccache_env("/usr/bin/ccache", spec) + # User-set values are respected; the rest get defaults + assert "CCACHE_NOHASHDIR" not in env + assert env["CCACHE_DEPEND"] == "1" + assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache") + assert env["CCACHE_BASEDIR"] == str((tmp_path / "build").resolve()) + + +def test_ccache_defaults_env_escapes_a_dollar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A literal $ is doubled, since ccache expands $VAR in its settings.""" + from esphome.core import CORE + + CORE.build_path = tmp_path / "a$b" + for key in ("CCACHE_DIR", "CCACHE_BASEDIR"): + monkeypatch.delenv(key, raising=False) + env = ccache.ccache_defaults_env(tmp_path / "c$d") + assert env["CCACHE_DIR"].endswith("c$$d") + assert env["CCACHE_BASEDIR"].endswith("a$$b") diff --git a/tests/unit_tests/build_helpers/test_idedata.py b/tests/unit_tests/build_helpers/test_idedata.py index fcf9c67086..9279ca52b2 100644 --- a/tests/unit_tests/build_helpers/test_idedata.py +++ b/tests/unit_tests/build_helpers/test_idedata.py @@ -676,3 +676,36 @@ def test_load_or_build_idedata_cache_hit_skips_rebuild(tmp_path: Path) -> None: ) mock_build.assert_not_called() assert data["cached"] is True + + +def test_warn_if_idedata_missing_is_silent_on_success( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING): + idedata.warn_if_idedata_missing(lambda: {"cc_path": "gcc"}) + assert "idedata" not in caplog.text + + +def test_warn_if_idedata_missing_warns_on_none( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING): + idedata.warn_if_idedata_missing(lambda: None) + assert "No idedata was generated for this build" in caplog.text + + +@pytest.mark.parametrize( + "error", [EsphomeError("db unusable"), ValueError("bad"), OSError("gone")] +) +def test_warn_if_idedata_missing_downgrades_failures( + caplog: pytest.LogCaptureFixture, error: Exception +) -> None: + """The firmware already built; a broken idedata must not fail the build.""" + + def broken() -> dict | None: + raise error + + with caplog.at_level(logging.DEBUG): + idedata.warn_if_idedata_missing(broken) + assert f"Could not generate idedata: {error}" in caplog.text + assert "Idedata failure detail" in caplog.text diff --git a/tests/unit_tests/build_helpers/test_native.py b/tests/unit_tests/build_helpers/test_native.py new file mode 100644 index 0000000000..57be9db26a --- /dev/null +++ b/tests/unit_tests/build_helpers/test_native.py @@ -0,0 +1,34 @@ +"""Tests for esphome.build_helpers.native.""" + +from __future__ import annotations + +import logging + +import pytest + +from esphome.build_helpers import native +from esphome.const import Toolchain +from esphome.core import CORE + + +def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None: + """Options a native build drops are warned by name; consumed ones stay quiet.""" + CORE.toolchain = Toolchain.ARDUINO + CORE.platformio_options = { + "lib_ignore": ["x"], + "board_build.filesystem": "littlefs", + } + with caplog.at_level(logging.WARNING): + native.warn_ignored_platformio_options({"lib_ignore"}) + assert "platformio_options->board_build.filesystem is ignored" in caplog.text + assert "native 'arduino' toolchain" in caplog.text + assert "lib_ignore" not in caplog.text + + +def test_warn_ignored_platformio_options_without_options( + caplog: pytest.LogCaptureFixture, +) -> None: + CORE.platformio_options = None + with caplog.at_level(logging.WARNING): + native.warn_ignored_platformio_options(()) + assert caplog.text == "" diff --git a/tests/unit_tests/build_helpers/test_ninja.py b/tests/unit_tests/build_helpers/test_ninja.py index 6f0bbda0b9..42782a1f05 100644 --- a/tests/unit_tests/build_helpers/test_ninja.py +++ b/tests/unit_tests/build_helpers/test_ninja.py @@ -141,3 +141,126 @@ def test_shell_token_windows_branch_uses_argv_rule() -> None: with patch.object(os, "name", "nt"): assert ninja_helper.shell_token("a b") == '"a b"' assert ninja_helper.shell_token("", force=True) == '""' + + +@pytest.mark.parametrize("char", ["|", "\n", "\r"]) +def test_escape_rejects_what_ninja_cannot_express(char: str) -> None: + with pytest.raises(EsphomeError, match="a ninja build file cannot express"): + ninja_helper.escape(f"src/a{char}b.cpp") + + +def test_escape_keeps_a_hash() -> None: + """A # is only a comment at the start of a ninja line.""" + assert ninja_helper.escape("src/a#b c.cpp") == "src/a#b$ c.cpp" + + +def test_absolute_tool_anchors_a_relative_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Anchor the path: ninja runs from the build directory.""" + monkeypatch.chdir(tmp_path) + assert Path(ninja_helper.absolute_tool("bin/ninja")) == tmp_path / "bin" / "ninja" + + +def test_find_ninja_returns_an_absolute_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + with ( + patch("shutil.which", return_value="bin/ninja"), + patch.object(ninja_helper, "_ninja_runs", return_value=True), + ): + assert ninja_helper.find_ninja() == tmp_path / "bin" / "ninja" + + +@pytest.fixture +def compdb_dir(tmp_path: Path) -> Path: + build_dir = tmp_path / "build" + build_dir.mkdir() + (build_dir / "build.ninja").write_text("rule x\n") + return build_dir + + +def test_refresh_compile_commands_regenerates_when_stale(compdb_dir: Path) -> None: + ninja = Path("ninja") + compdb = compdb_dir / "compile_commands.json" + stamp = compdb_dir / ".compile_commands.stamp" + with patch.object(ninja_helper, "write_compile_commands") as write: + # A rewritten manifest always regenerates + ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, True) + assert write.call_count == 1 + assert stamp.is_file() + # No compile DB yet + ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False) + assert write.call_count == 2 + compdb.write_text("[]") + # Fresh stamp: nothing to do + ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False) + assert write.call_count == 2 + # A manifest newer than the stamp (interrupted previous run) + os.utime(stamp, (1, 1)) + ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False) + assert write.call_count == 3 + # A missing stamp regenerates too + stamp.unlink() + ninja_helper.refresh_compile_commands(ninja, compdb_dir, {}, False) + assert write.call_count == 4 + + +def _completed(rc: int = 0, stdout: str = "", stderr: str = "") -> MagicMock: + return MagicMock(returncode=rc, stdout=stdout, stderr=stderr) + + +def test_write_compile_commands_success(compdb_dir: Path) -> None: + entries = '[{"file": "a.cpp", "command": "g++ -c a.cpp"}]' + with patch("subprocess.run", return_value=_completed(stdout=entries)) as run: + ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {"A": "1"}) + assert (compdb_dir / "compile_commands.json").read_text() == entries + # Every compile rule, so preprocessed assembly reaches the database too + assert run.call_args.args[0] == [ + "ninja", + "-C", + str(compdb_dir), + "-t", + "compdb", + "c", + "cxx", + "aspp", + "asm", + ] + assert run.call_args.kwargs["env"] == {"A": "1"} + + +@pytest.mark.parametrize( + ("result", "message"), + [ + (_completed(rc=1, stderr="boom"), "Could not generate compile_commands.json"), + # A parse failure names its cause, not the rule-name story + (_completed(stdout="not json"), "unparsable compile database.*not json"), + (_completed(stdout="[]"), "empty compile database"), + ], +) +def test_write_compile_commands_failures_drop_stale_db( + compdb_dir: Path, result: MagicMock, message: str +) -> None: + compdb = compdb_dir / "compile_commands.json" + compdb.write_text("[stale]") + with ( + patch("subprocess.run", return_value=result), + pytest.raises(EsphomeError, match=message), + ): + ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {}) + assert not compdb.exists() + + +def test_write_compile_commands_keeps_the_mtime_of_an_unchanged_db( + compdb_dir: Path, +) -> None: + """The idedata cache is keyed on the DB's mtime.""" + entries = '[{"file": "a.cpp", "command": "g++ -c a.cpp"}]' + compdb = compdb_dir / "compile_commands.json" + compdb.write_text(entries) + os.utime(compdb, (1000, 1000)) + with patch("subprocess.run", return_value=_completed(stdout=entries)): + ninja_helper.write_compile_commands(Path("ninja"), compdb_dir, {}) + assert compdb.stat().st_mtime == 1000 diff --git a/tests/unit_tests/build_helpers/test_ninja_gen.py b/tests/unit_tests/build_helpers/test_ninja_gen.py new file mode 100644 index 0000000000..e7f9b96af2 --- /dev/null +++ b/tests/unit_tests/build_helpers/test_ninja_gen.py @@ -0,0 +1,57 @@ +"""Tests for esphome.build_helpers.ninja_gen.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from esphome.build_helpers import ninja_gen +from esphome.build_helpers.ninja_gen import Flag + + +@pytest.mark.parametrize( + ("flag", "anchored"), + [ + (("-Iinc",), ("-I{base}/inc",)), + (("-Llib",), ("-L{base}/lib",)), + (("-include", "pre.h"), ("-include", "{base}/pre.h")), + (("-isystem", "sys"), ("-isystem", "{base}/sys")), + # The glued spelling of a flag that takes a path + (("-isystemsys",), ("-isystem{base}/sys",)), + (("-includepre.h",), ("-include{base}/pre.h",)), + # An absolute operand is never changed + (("-I{base}/abs",), ("-I{base}/abs",)), + (("-include", "{base}/abs.h"), ("-include", "{base}/abs.h")), + # Not a path + (("-DUSE_HOST",), ("-DUSE_HOST",)), + (("-lssl",), ("-lssl",)), + (("-framework", "Cocoa"), ("-framework", "Cocoa")), + (("-I",), ("-I",)), + ], +) +def test_anchor_path_flag(tmp_path: Path, flag: Flag, anchored: Flag) -> None: + """Relative operands resolve from the build path, as under PlatformIO.""" + + def fill(tokens: Flag) -> Flag: + return tuple( + str(Path(tok.replace("{base}", str(tmp_path)))) + if "{base}" in tok and not tok.startswith("-") + else tok.replace("{base}/", f"{tmp_path}{os.sep}") + for tok in tokens + ) + + assert ninja_gen.anchor_path_flag(fill(flag), tmp_path) == fill(anchored) + + +def test_collect_sources_skips_excluded_and_other_files(tmp_path: Path) -> None: + for name in ("b.cpp", "a.c", "skip.cpp", "notes.txt", "sub/c.S"): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + assert ninja_gen.collect_sources(tmp_path, {"skip.cpp"}) == [ + tmp_path / "a.c", + tmp_path / "b.cpp", + tmp_path / "sub" / "c.S", + ] diff --git a/tests/unit_tests/components/host/__init__.py b/tests/unit_tests/components/host/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/host/test_init.py b/tests/unit_tests/components/host/test_init.py new file mode 100644 index 0000000000..d7662e0b0a --- /dev/null +++ b/tests/unit_tests/components/host/test_init.py @@ -0,0 +1,86 @@ +"""Tests for the host platform's native-toolchain wiring.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import patch + +import pytest + +from esphome.components import host +import esphome.config_validation as cv +from esphome.const import ( + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + PLATFORM_HOST, + Toolchain, +) +from esphome.core import CORE, EsphomeError + + +@pytest.fixture(autouse=True) +def _core_data() -> None: + CORE.data[KEY_CORE] = {} + + +def test_schema_resolves_the_host_toolchain() -> None: + """The host serves exactly one toolchain; it is picked without config.""" + config = host.CONFIG_SCHEMA({}) + assert CORE.toolchain is Toolchain.HOST + assert CORE.using_native_toolchain + assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == PLATFORM_HOST + assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "host" + assert "mac_address" in config + + +def test_schema_rejects_the_platformio_toolchain() -> None: + """There is no PlatformIO backend left to honor --toolchain platformio.""" + CORE.toolchain = Toolchain.PLATFORMIO + with pytest.raises(cv.Invalid, match="Unsupported toolchain 'platformio' for host"): + host.CONFIG_SCHEMA({}) + + +def test_to_code_configures_the_native_build() -> None: + config = host.CONFIG_SCHEMA({"mac_address": "AA:BB:CC:DD:EE:FF"}) + asyncio.run(host.to_code(config)) + assert "-DUSE_HOST" in CORE.build_flags + # The standard travels through cpp_standard, not a -std= build flag, + # so the generator can keep it off the C compile line + assert CORE.cpp_standard == "gnu++20" + assert not any(flag.startswith("-std=") for flag in CORE.build_flags) + assert not CORE.platformio_options + define_names = {define.name for define in CORE.defines} + assert {"ESPHOME_BOARD", "ESPHOME_VARIANT", "USE_ESPHOME_HOST_MAC_ADDRESS"} <= ( + define_names + ) + + +def test_run_compile_hook_claims_the_build() -> None: + config = {"esphome": {}} + with ( + patch.object(host.sys, "platform", "linux"), + patch("esphome.host.toolchain.run_compile", return_value=0) as run, + ): + assert host.run_compile(object(), config) is True + run.assert_called_once_with(config, CORE.verbose) + + +def test_run_compile_hook_raises_on_failure() -> None: + with ( + patch.object(host.sys, "platform", "linux"), + patch("esphome.host.toolchain.run_compile", return_value=2), + pytest.raises(EsphomeError, match="Host build failed"), + ): + host.run_compile(object(), {"esphome": {}}) + + +def test_run_compile_hook_refuses_windows() -> None: + """The host sources use POSIX interfaces; fail before building, by name.""" + with ( + patch.object(host.sys, "platform", "win32"), + patch("esphome.host.toolchain.run_compile") as run, + pytest.raises(EsphomeError, match="does not build on Windows"), + ): + host.run_compile(object(), {"esphome": {}}) + run.assert_not_called() diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 9f415344ae..ab11c41afb 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -112,17 +112,6 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: assert env["CCACHE_DIR"] == "x" -def test_ccache_env(tmp_path: Path) -> None: - assert framework.ccache_env(None) == {} - with patch.dict(os.environ, {"CCACHE_NOHASHDIR": "false"}, clear=True): - env = framework.ccache_env("/usr/bin/ccache") - # User-set values are respected; the rest get defaults - assert "CCACHE_NOHASHDIR" not in env - assert env["CCACHE_DEPEND"] == "1" - assert env["CCACHE_BASEDIR"] == str(Path(CORE.build_path).resolve()) - assert env["CCACHE_DIR"].endswith("ccache") - - def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: """Calling the installer below the floor fails before any download.""" with pytest.raises(EsphomeError, match=">= 3.1.1"): @@ -148,18 +137,20 @@ def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: assert env["PATH"].split(os.pathsep) == [str(tmp_path / "bin"), "/usr/bin", "/bin"] -def test_ccache_env_accepts_a_preresolved_path() -> None: - """The caller resolves ccache once and threads it through; None means - resolved-and-disabled.""" - with patch.dict(os.environ, {}, clear=True): - assert framework.ccache_env(None) == {} - env = framework.ccache_env("/usr/bin/ccache") - assert env["CCACHE_DIR"].endswith("ccache") - - def test_toolchain_tool_layout(tmp_path: Path) -> None: """One owner for the bin/xtensa-lx106-elf- layout.""" tool = framework.toolchain_tool(tmp_path, "addr2line") assert tool.parent == tmp_path / "bin" assert tool.name.startswith("xtensa-lx106-elf-addr2line") assert (tool.suffix == ".exe") is (os.name == "nt") + + +def test_get_build_env_uses_the_arduino8266_ccache_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ESPHOME_ARDUINO8266_PREFIX", str(tmp_path / "cache")) + monkeypatch.delenv("CCACHE_DIR", raising=False) + env = framework.get_build_env(tmp_path / "toolchain", "/usr/bin/ccache") + assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache") + # None means resolved and disabled + assert "CCACHE_DIR" not in framework.get_build_env(tmp_path / "toolchain", None) diff --git a/tests/unit_tests/test_arduino8266_toolchain.py b/tests/unit_tests/test_arduino8266_toolchain.py index a9c0546b4b..6286374418 100644 --- a/tests/unit_tests/test_arduino8266_toolchain.py +++ b/tests/unit_tests/test_arduino8266_toolchain.py @@ -16,6 +16,7 @@ from esphome.const import ( CONF_ESPHOME, KEY_CORE, KEY_FRAMEWORK_VERSION, + Toolchain, ) from esphome.core import CORE, EsphomeError @@ -80,7 +81,7 @@ def test_run_compile_build_failure(tmp_path: Path) -> None: patch.object( toolchain.subprocess, "run", return_value=MagicMock(returncode=2) ) as mock_run, - patch.object(toolchain, "_write_compile_commands") as mock_compdb, + patch.object(toolchain, "refresh_compile_commands") as mock_compdb, ): assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=True) == 2 cmd = mock_run.call_args[0][0] @@ -100,7 +101,7 @@ def test_run_compile_success(tmp_path: Path) -> None: "run", return_value=MagicMock(returncode=0, stdout="", stderr=""), ) as mock_run, - patch.object(toolchain, "_write_compile_commands") as mock_compdb, + patch.object(toolchain, "refresh_compile_commands") as mock_compdb, patch.object(toolchain, "_print_size_summary") as mock_size, patch.object(toolchain, "get_idedata") as mock_idedata, ): @@ -116,39 +117,12 @@ def test_run_compile_success(tmp_path: Path) -> None: # cwd, not -C, so ninja prints no "Entering directory" banner assert "-C" not in cmd assert call[1]["cwd"] is not None - mock_compdb.assert_called_once() + # An unchanged manifest is passed on, so the shared refresh can skip + assert mock_compdb.call_args.args[3] is False mock_size.assert_called_once() mock_idedata.assert_called_once() -def test_run_compile_regenerates_stale_compdb(tmp_path: Path) -> None: - """An interrupted run can leave build.ninja newer than the compile DB; - mere existence must not skip regeneration.""" - build_dir = toolchain.get_build_dir() - build_dir.mkdir(parents=True, exist_ok=True) - (build_dir / "build.ninja").write_text("") - compdb = build_dir / "compile_commands.json" - compdb.write_text("[]") - os.utime(compdb, ((build_dir / "build.ninja").stat().st_mtime - 5,) * 2) - with ( - patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), - patch.object(framework, "get_build_env", return_value={}), - patch("esphome.build_gen.arduino8266.write_project", return_value=False), - patch.object( - toolchain.subprocess, - "run", - return_value=MagicMock( - returncode=0, stdout="ninja: no work to do.\n", stderr="" - ), - ), - patch.object(toolchain, "_write_compile_commands") as mock_compdb, - patch.object(toolchain, "_print_size_summary"), - patch.object(toolchain, "get_idedata"), - ): - assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 - mock_compdb.assert_called_once() - - def test_run_compile_missing_artifact_fails( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: @@ -164,7 +138,7 @@ def test_run_compile_missing_artifact_fails( "run", return_value=MagicMock(returncode=0, stdout="", stderr=""), ), - patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "refresh_compile_commands"), patch.object(toolchain, "_print_size_summary") as mock_size, patch.object(toolchain, "get_idedata"), ): @@ -188,7 +162,7 @@ def test_run_compile_warns_when_idedata_fails( "run", return_value=MagicMock(returncode=0, stdout="", stderr=""), ), - patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "refresh_compile_commands"), patch.object(toolchain, "_print_size_summary"), patch.object(toolchain, "get_idedata", return_value=None), ): @@ -196,63 +170,6 @@ def test_run_compile_warns_when_idedata_fails( assert "No idedata was generated for this build" in caplog.text -def test_write_compile_commands(tmp_path: Path) -> None: - build_dir = tmp_path / "build" - build_dir.mkdir() - entries = '[{"file": "a.cpp", "command": "cc"}]\n' - with patch.object( - toolchain.subprocess, - "run", - return_value=MagicMock(returncode=0, stdout=entries), - ): - toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {}) - assert (build_dir / "compile_commands.json").read_text() == entries - - -@pytest.mark.parametrize( - ("stdout", "match"), - [ - ("[]\n", "empty compile database"), - # A parse failure names its cause, not the rule-name story - ("not json", "unparsable compile database.*not json"), - ], -) -def test_write_compile_commands_bad_db_raises( - tmp_path: Path, stdout: str, match: str -) -> None: - """An empty or unparsable compile database fails the build with its - actual cause and drops any stale database.""" - build_dir = tmp_path / "build" - build_dir.mkdir() - (build_dir / "compile_commands.json").write_text("[stale]") - with ( - patch.object( - toolchain.subprocess, - "run", - return_value=MagicMock(returncode=0, stdout=stdout), - ), - pytest.raises(EsphomeError, match=match), - ): - toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {}) - assert not (build_dir / "compile_commands.json").exists() - - -def test_write_compile_commands_failure_removes_stale_db(tmp_path: Path) -> None: - """A failed compdb run must not leave a stale database behind.""" - stale = tmp_path / "compile_commands.json" - stale.write_text("[]") - with ( - patch.object( - toolchain.subprocess, - "run", - return_value=MagicMock(returncode=1, stderr="boom"), - ), - pytest.raises(EsphomeError, match="compile_commands"), - ): - toolchain._write_compile_commands(tmp_path / "ninja", tmp_path, {}) - assert not stale.exists() - - def test_parse_app_size(tmp_path: Path) -> None: ld = tmp_path / "eagle.flash.4m.ld" ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0xfeff0\n}\n") @@ -342,7 +259,9 @@ def test_get_idedata_delegates(tmp_path: Path) -> None: "esphome.build_helpers.idedata.load_or_build_idedata", return_value={"cc_path": "x"}, ) as mock_load, - patch.object(toolchain, "resolve_ccache_path", return_value="/cc/ccache"), + patch.object( + toolchain, "resolve_absolute_ccache_path", return_value="/cc/ccache" + ), ): assert toolchain.get_idedata() == {"cc_path": "x"} compile_commands, elf, cache = mock_load.call_args[0] @@ -359,43 +278,12 @@ def test_get_idedata_no_ccache(tmp_path: Path) -> None: patch( "esphome.build_helpers.idedata.load_or_build_idedata", return_value={} ) as mock_load, - patch.object(toolchain, "resolve_ccache_path", return_value=None), + patch.object(toolchain, "resolve_absolute_ccache_path", return_value=None), ): toolchain.get_idedata() assert mock_load.call_args.kwargs["launcher"] is None -def test_run_compile_skips_compdb_when_ninja_unchanged(tmp_path: Path) -> None: - """An unchanged build.ninja means the compile DB is already current.""" - build_dir = toolchain.get_build_dir() - build_dir.mkdir(parents=True, exist_ok=True) - # write_project (stubbed below) always leaves a build.ninja behind - (build_dir / "build.ninja").write_text("# manifest") - - def run(regenerate_expected: bool) -> None: - with ( - patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), - patch.object(framework, "get_build_env", return_value={}), - patch("esphome.build_gen.arduino8266.write_project", return_value=False), - patch.object( - toolchain.subprocess, - "run", - return_value=MagicMock(returncode=0, stdout="", stderr=""), - ), - patch.object(toolchain, "_write_compile_commands") as mock_compdb, - patch.object(toolchain, "_print_size_summary"), - patch.object(toolchain, "get_idedata"), - ): - assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 - assert mock_compdb.called == regenerate_expected - - # Missing compile DB: regenerated even though build.ninja is unchanged - run(regenerate_expected=True) - # Present compile DB + unchanged build.ninja: skipped - (build_dir / "compile_commands.json").write_text("[]") - run(regenerate_expected=False) - - def test_print_size_summary_unparsable_section( tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -447,32 +335,6 @@ def test_print_size_summary_missing_section_skips_summary( assert "missing section(s) .bss" in caplog.text -def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None: - """Component-added options the native build drops are warned by name; - the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags, - flash_mode) stay quiet.""" - CORE.platformio_options = { - "board_build.ldscript": "eagle.flash.4m2m.ld", - "board_build.f_cpu": "160000000L", - "board_build.filesystem": "littlefs", - "board_build.flash_mode": "dio", - "build_src_flags": "-include throw_stubs.h", - "lib_ignore": ["Updater"], - "upload_speed": "460800", - } - toolchain._warn_ignored_platformio_options() - assert "platformio_options->board_build.filesystem is ignored" in caplog.text - assert "native 'arduino' toolchain" in caplog.text - assert "board_build.ldscript is ignored" not in caplog.text - assert "board_build.f_cpu is ignored" not in caplog.text - assert "lib_ignore" not in caplog.text - assert "build_src_flags" not in caplog.text - assert "flash_mode" not in caplog.text - # Component-added upload_speed never gets read under the native - # toolchain, so it must warn - assert "upload_speed" in caplog.text - - def test_run_compile_idedata_error_does_not_fail_build( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: @@ -486,7 +348,7 @@ def test_run_compile_idedata_error_does_not_fail_build( "run", return_value=MagicMock(returncode=0, stdout="", stderr=""), ), - patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "refresh_compile_commands"), patch.object(toolchain, "_print_size_summary"), patch.object( toolchain, @@ -512,7 +374,7 @@ def test_run_compile_skipped_size_summary_names_consequence( "run", return_value=MagicMock(returncode=0, stdout="", stderr=""), ), - patch.object(toolchain, "_write_compile_commands"), + patch.object(toolchain, "refresh_compile_commands"), patch.object(toolchain, "_print_size_summary", return_value=False), patch.object(toolchain, "get_idedata", return_value=None), ): @@ -541,8 +403,50 @@ def test_get_idedata_accepts_preresolved_ccache() -> None: "esphome.build_helpers.idedata.load_or_build_idedata", return_value={"ok": True}, ) as mock_build, - patch.object(toolchain, "resolve_ccache_path") as mock_resolve, + patch.object(toolchain, "resolve_absolute_ccache_path") as mock_resolve, ): assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True} mock_resolve.assert_not_called() assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache" + + +def test_run_compile_warns_about_dropped_platformio_options( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Component-added options the native build drops are warned by name; + the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags, + flash_mode) stay quiet.""" + CORE.toolchain = Toolchain.ARDUINO + CORE.platformio_options = { + "board_build.ldscript": "eagle.flash.4m2m.ld", + "board_build.f_cpu": "160000000L", + "board_build.filesystem": "littlefs", + "board_build.flash_mode": "dio", + "build_src_flags": "-include throw_stubs.h", + "lib_ignore": ["Updater"], + "upload_speed": "460800", + } + with ( + patch.object(framework, "check_and_install", return_value=_paths(tmp_path)), + patch.object(framework, "get_build_env", return_value={}), + patch("esphome.build_gen.arduino8266.write_project", return_value=True), + patch.object( + toolchain.subprocess, + "run", + return_value=MagicMock(returncode=0, stdout="", stderr=""), + ), + patch.object(toolchain, "refresh_compile_commands") as mock_compdb, + patch.object(toolchain, "_print_size_summary"), + patch.object(toolchain, "get_idedata"), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert "platformio_options->board_build.filesystem is ignored" in caplog.text + assert "platformio_options->upload_speed is ignored" in caplog.text + assert "native 'arduino' toolchain" in caplog.text + assert "board_build.ldscript is ignored" not in caplog.text + assert "board_build.f_cpu is ignored" not in caplog.text + assert "lib_ignore" not in caplog.text + assert "build_src_flags" not in caplog.text + assert "flash_mode" not in caplog.text + # A rewritten manifest is passed on, so the compile DB is regenerated + assert mock_compdb.call_args.args[3] is True diff --git a/tests/unit_tests/test_arduino_library.py b/tests/unit_tests/test_arduino_library.py index 87de28cf32..77b30363ec 100644 --- a/tests/unit_tests/test_arduino_library.py +++ b/tests/unit_tests/test_arduino_library.py @@ -1160,3 +1160,29 @@ def test_versionless_dependency_with_provider_stays_quiet( libs = _resolve(framework) assert "Wire" in [lib.name for lib in libs] assert "has no version to resolve" not in caplog.text + + +def test_resolve_libraries_without_framework_is_all_external() -> None: + """The host build has no core tree: nothing is bundled, the framework + check is skipped, and every name reaches the converter.""" + _add_library("Wire", None) + _add_library("lvgl/lvgl", "9.5.0") + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "native" + assert backend.framework is None + assert backend.cache_key == "host" + assert backend.provides("Wire") is False + return [] + + with patch.object(component, "convert_libraries", side_effect=fake_convert) as mock: + libs = component.resolve_libraries( + None, + pio_platform="native", + board_mcu="host", + cache_key="host", + framework=None, + ) + assert libs == [] + (libraries, _backend), _ = mock.call_args + assert [lib.name for lib in libraries] == ["Wire", "lvgl/lvgl"] diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py index 0c96f8c8c9..9445810975 100644 --- a/tests/unit_tests/test_core.py +++ b/tests/unit_tests/test_core.py @@ -976,6 +976,14 @@ class TestEsphomeCore: target.toolchain = const.Toolchain.SDK_NRF assert target.using_native_toolchain is False + def test_using_toolchain_host(self, target): + """The host's only toolchain: the system compiler driven by ninja.""" + target.toolchain = const.Toolchain.HOST + assert target.using_toolchain_host is True + assert target.using_native_toolchain is True + target.toolchain = const.Toolchain.PLATFORMIO + assert target.using_toolchain_host is False + def test_add_library__extracts_short_name_from_path(self, target): """Test add_library extracts short name from library paths like owner/lib.""" target.data[const.KEY_CORE] = { diff --git a/tests/unit_tests/test_host_toolchain.py b/tests/unit_tests/test_host_toolchain.py new file mode 100644 index 0000000000..4e4e444801 --- /dev/null +++ b/tests/unit_tests/test_host_toolchain.py @@ -0,0 +1,329 @@ +"""Tests for esphome.host.toolchain (the native host build driver).""" + +from __future__ import annotations + +from collections.abc import Generator +import logging +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from esphome.const import ( + CONF_COMPILE_PROCESS_LIMIT, + CONF_ESPHOME, + KEY_CORE, + KEY_TARGET_PLATFORM, + PLATFORM_HOST, + Toolchain, +) +from esphome.core import CORE, EsphomeError +from esphome.host import toolchain +from esphome.host.toolchain import PROGRAM_NAME + + +@pytest.fixture(autouse=True) +def _core(tmp_path: Path) -> None: + CORE.config_path = tmp_path / "dev.yaml" + CORE.build_path = tmp_path + CORE.name = "dev" + CORE.toolchain = Toolchain.HOST + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_HOST} + + +def _abs(path: str) -> str: + """What the toolchain makes of a tool path (a drive is added on Windows).""" + return str(Path(path).absolute()) + + +def test_find_command_prefers_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CXX", "/opt/clang++") + with patch("shutil.which", side_effect={"/opt/clang++": "/opt/clang++"}.get): + assert toolchain.find_command("CXX", ("g++",)) == (_abs("/opt/clang++"),) + + +def test_find_command_keeps_the_override_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``CC="gcc -m32"`` is a command, as make and CMake read it.""" + monkeypatch.setenv("CC", "gcc -m32 '--sysroot=/opt/my sdk'") + with patch("shutil.which", side_effect={"gcc": "/usr/bin/gcc"}.get): + assert toolchain.find_command("CC", ("cc",)) == ( + _abs("/usr/bin/gcc"), + "-m32", + "--sysroot=/opt/my sdk", + ) + + +@pytest.mark.parametrize( + ("override", "expected_args"), + [("ccache gcc", ()), ("/opt/bin/ccache gcc -m32", ("-m32",))], +) +def test_find_command_drops_a_ccache_prefix( + monkeypatch: pytest.MonkeyPatch, override: str, expected_args: tuple[str, ...] +) -> None: + """The build adds ccache itself; the compiler is the word after it.""" + monkeypatch.setenv("CC", override) + with patch("shutil.which", side_effect={"gcc": "/usr/bin/gcc"}.get): + assert toolchain.find_command("CC", ("cc",)) == ( + _abs("/usr/bin/gcc"), + *expected_args, + ) + + +def test_find_command_accepts_a_compiler_named_ccache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Alone, the word is the program: ccache's compiler links work that way.""" + monkeypatch.setenv("CC", "ccache") + with patch("shutil.which", side_effect={"ccache": "/usr/bin/ccache"}.get): + assert toolchain.find_command("CC", ("cc",)) == (_abs("/usr/bin/ccache"),) + + +def test_find_command_env_override_must_run(monkeypatch: pytest.MonkeyPatch) -> None: + """A broken override fails by name instead of silently using another compiler.""" + monkeypatch.setenv("CXX", "nope++") + with ( + patch("shutil.which", return_value=None), + pytest.raises(EsphomeError, match="CXX='nope\\+\\+' does not name"), + ): + toolchain.find_command("CXX", ("g++",)) + + +def test_find_command_blank_override_is_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CC", " ") + with patch("shutil.which", side_effect={"clang": "/usr/bin/clang"}.get): + assert toolchain.find_command("CC", ("gcc", "clang")) == ( + _abs("/usr/bin/clang"), + ) + + +def test_find_command_first_candidate_wins(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CC", raising=False) + table = {"gcc": "/usr/bin/gcc", "clang": "/usr/bin/clang"} + with patch("shutil.which", side_effect=table.get): + assert toolchain.find_command("CC", ("gcc", "clang")) == (_abs("/usr/bin/gcc"),) + + +def test_find_command_none_found(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CC", raising=False) + with ( + patch("shutil.which", return_value=None), + pytest.raises( + EsphomeError, + match=r"gcc not found on PATH \(tried gcc, clang\); install it or set CC", + ), + ): + toolchain.find_command("CC", ("gcc", "clang")) + + +def test_find_command_makes_a_relative_hit_absolute( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Anchor the path: ninja runs from the build directory.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("CC", "./toolchain/gcc") + with patch("shutil.which", return_value="./toolchain/gcc"): + (found,) = toolchain.find_command("CC", ("gcc",)) + assert Path(found) == Path("toolchain/gcc").absolute() + + +def test_find_tool_refuses_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AR", "ar --plugin x") + with ( + patch("shutil.which", return_value="/usr/bin/ar"), + pytest.raises(EsphomeError, match="AR must name a program without arguments"), + ): + toolchain.find_tool("AR", ("ar",)) + + +def test_find_compilers(monkeypatch: pytest.MonkeyPatch) -> None: + for var in ("CC", "CXX"): + monkeypatch.delenv(var, raising=False) + table = {"gcc": "/usr/bin/gcc", "g++": "/usr/bin/g++"} + with patch("shutil.which", side_effect=table.get): + assert toolchain.find_compilers() == toolchain.HostCompilers( + cc=(_abs("/usr/bin/gcc"),), cxx=(_abs("/usr/bin/g++"),) + ) + + +def test_build_paths(tmp_path: Path) -> None: + """The PlatformIO layout is kept: CORE.firmware_bin resolves the same file.""" + assert toolchain.get_build_dir() == tmp_path / ".pioenvs" / "dev" + assert toolchain.get_elf_path() == CORE.firmware_bin + + +def test_binutils_paths(monkeypatch: pytest.MonkeyPatch) -> None: + for var in ("OBJDUMP", "READELF"): + monkeypatch.delenv(var, raising=False) + table = {"objdump": "/usr/bin/objdump", "readelf": "/usr/bin/readelf"} + with patch("shutil.which", side_effect=table.get): + assert toolchain.get_objdump_path() == Path(_abs("/usr/bin/objdump")) + assert toolchain.get_readelf_path() == Path(_abs("/usr/bin/readelf")) + + +def test_get_build_env_merges_without_leaking( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ESPHOME_HOST_PREFIX", str(tmp_path / "cache")) + monkeypatch.delenv("CCACHE_DIR", raising=False) + monkeypatch.setenv("KEEP_ME", "1") + env = toolchain.get_build_env("/usr/bin/ccache") + assert env["KEEP_ME"] == "1" + assert env["CCACHE_DIR"] == str((tmp_path / "cache").resolve() / "ccache") + assert "CCACHE_DIR" not in os.environ + + +def test_get_build_env_without_ccache(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CCACHE_DIR", raising=False) + assert "CCACHE_DIR" not in toolchain.get_build_env(None) + + +@pytest.fixture +def compile_env(tmp_path: Path) -> Generator[SimpleNamespace]: + """Stub everything run_compile resolves; the build dir holds a manifest.""" + build_dir = tmp_path / ".pioenvs" / "dev" + build_dir.mkdir(parents=True) + (build_dir / "build.ninja").write_text("rule x\n") + compilers = toolchain.HostCompilers(("gcc",), ("g++",)) + with ( + patch.object(toolchain, "find_ninja", return_value=Path("/usr/bin/ninja")), + patch.object(toolchain, "find_compilers", return_value=compilers), + patch.object(toolchain, "resolve_absolute_ccache_path", return_value=None), + patch("esphome.build_gen.host.write_project", return_value=True) as project, + patch.object(toolchain, "refresh_compile_commands") as refresh, + patch.object( + toolchain, "_load_idedata", return_value={"cc_path": "gcc"} + ) as ide, + patch("subprocess.run") as run, + ): + yield SimpleNamespace( + build_dir=build_dir, + compilers=compilers, + write_project=project, + refresh=refresh, + idedata=ide, + run=run, + ) + + +def _completed(rc: int = 0) -> SimpleNamespace: + return SimpleNamespace(returncode=rc, stdout="", stderr="") + + +def test_run_compile_builds_and_reports_success(compile_env: SimpleNamespace) -> None: + elf = compile_env.build_dir / PROGRAM_NAME + + def build(cmd: list[str], **kwargs: object) -> SimpleNamespace: + elf.write_text("") + return _completed() + + compile_env.run.side_effect = build + config = {CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 4}} + assert toolchain.run_compile(config, verbose=True) == 0 + + compile_env.write_project.assert_called_once_with(compile_env.compilers, None) + compile_env.refresh.assert_called_once() + assert compile_env.refresh.call_args.args[3] is True + compile_env.run.assert_called_once() + assert compile_env.run.call_args.args[0] == [ + str(Path("/usr/bin/ninja")), + "-v", + "-j", + "4", + PROGRAM_NAME, + ] + assert compile_env.run.call_args.kwargs["cwd"] == compile_env.build_dir + compile_env.idedata.assert_called_once_with(None) + + +def test_run_compile_defaults(compile_env: SimpleNamespace) -> None: + """No verbosity and no process limit: just the ninja target.""" + (compile_env.build_dir / PROGRAM_NAME).write_text("") + compile_env.run.return_value = _completed() + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert compile_env.run.call_args.args[0] == [ + str(Path("/usr/bin/ninja")), + PROGRAM_NAME, + ] + # The build inherits stdio and never captures: progress must stream + kwargs = compile_env.run.call_args.kwargs + assert kwargs["check"] is False + assert kwargs["close_fds"] is False + assert "capture_output" not in kwargs + + +def test_run_compile_warns_about_dropped_platformio_options( + compile_env: SimpleNamespace, caplog: pytest.LogCaptureFixture +) -> None: + (compile_env.build_dir / PROGRAM_NAME).write_text("") + compile_env.run.return_value = _completed() + CORE.platformio_options = {"lib_ignore": ["x"], "board_build.f_cpu": "1"} + with caplog.at_level(logging.WARNING): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + assert "platformio_options->board_build.f_cpu is ignored" in caplog.text + assert "native 'host' toolchain" in caplog.text + assert "lib_ignore" not in caplog.text + + +def test_run_compile_passes_the_resolved_ccache(compile_env: SimpleNamespace) -> None: + (compile_env.build_dir / PROGRAM_NAME).write_text("") + compile_env.run.return_value = _completed() + with patch.object( + toolchain, "resolve_absolute_ccache_path", return_value="/usr/bin/ccache" + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + compile_env.write_project.assert_called_once_with( + compile_env.compilers, "/usr/bin/ccache" + ) + compile_env.idedata.assert_called_once_with("/usr/bin/ccache") + + +def test_run_compile_build_failure_returns_code(compile_env: SimpleNamespace) -> None: + compile_env.run.return_value = _completed(rc=3) + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 3 + compile_env.idedata.assert_not_called() + + +def test_run_compile_missing_program_fails( + compile_env: SimpleNamespace, caplog: pytest.LogCaptureFixture +) -> None: + """A green ninja run that produced no program fails by name.""" + compile_env.run.return_value = _completed() + with caplog.at_level(logging.ERROR): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 1 + assert "Build produced no" in caplog.text + compile_env.idedata.assert_not_called() + + +@pytest.mark.parametrize("ccache", ["/usr/bin/ccache", None]) +def test_get_idedata_resolves_ccache(tmp_path: Path, ccache: str | None) -> None: + with ( + patch.object(toolchain, "resolve_absolute_ccache_path", return_value=ccache), + patch( + "esphome.build_helpers.idedata.load_or_build_idedata", return_value={"x": 1} + ) as load, + ): + assert toolchain.get_idedata() == {"x": 1} + load.assert_called_once_with( + tmp_path / ".pioenvs" / "dev" / "compile_commands.json", + tmp_path / ".pioenvs" / "dev" / PROGRAM_NAME, + CORE.relative_internal_path("idedata", "dev.json"), + launcher=ccache, + ) + + +@pytest.mark.parametrize("platform", ["darwin", "win32"]) +def test_check_analysis_supported_refuses_non_elf(platform: str) -> None: + with ( + patch.object(toolchain.sys, "platform", platform), + pytest.raises(EsphomeError, match=f"the host build on {platform}"), + ): + toolchain.check_analysis_supported() + + +def test_check_analysis_supported_accepts_linux() -> None: + with patch.object(toolchain.sys, "platform", "linux"): + toolchain.check_analysis_supported() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 4f11be0cb8..2ed61e5d26 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -7805,52 +7805,206 @@ async def test_wrap_to_code_comment_is_insertion_order_independent() -> None: assert second.index("a: 2") < second.index("z: 1") -def test_host_program_path_platformio_toolchain() -> None: - """Host + PlatformIO toolchain reads the memoized idedata path.""" - setup_core(platform=PLATFORM_HOST) - idedata = SimpleNamespace(firmware_elf_path="/build/x/.pioenvs/x/program") - with patch( - "esphome.platformio.toolchain.get_idedata", return_value=idedata - ) as mock_get: - assert main._host_program_path({}) == "/build/x/.pioenvs/x/program" - mock_get.assert_called_once_with({}) - - -def test_host_program_path_esp_idf_toolchain() -> None: - """Host + native ESP-IDF toolchain asks the espidf toolchain for the ELF.""" - setup_core(platform=PLATFORM_HOST) - CORE.toolchain = Toolchain.ESP_IDF - with patch( - "esphome.espidf.toolchain.get_elf_path", return_value=Path("/b/app.elf") - ): - assert main._host_program_path({}) == str(Path("/b/app.elf")) - - def test_command_compile_host_logs_program_path( + tmp_path: Path, caplog: pytest.LogCaptureFixture, ) -> None: """command_compile on host logs the compiled program path.""" - setup_core(platform=PLATFORM_HOST) + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") with ( patch.object(main, "write_cpp", return_value=0), patch.object(main, "compile_program", return_value=0), - patch.object(main, "_host_program_path", return_value="/b/program"), caplog.at_level(logging.INFO), ): assert main.command_compile(SimpleNamespace(only_generate=False), {}) == 0 - assert "Successfully compiled program to path '/b/program'" in caplog.text + assert f"Successfully compiled program to path '{CORE.firmware_bin}'" in caplog.text -def test_command_run_host_executes_program(caplog: pytest.LogCaptureFixture) -> None: +def test_command_run_host_executes_program( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: """command_run on host logs and executes the compiled program directly.""" - setup_core(platform=PLATFORM_HOST) + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") with ( patch.object(main, "write_cpp", return_value=0), patch.object(main, "compile_program", return_value=0), - patch.object(main, "_host_program_path", return_value="/b/program"), patch.object(main, "run_external_process", return_value=0) as mock_run, caplog.at_level(logging.INFO), ): assert main.command_run(SimpleNamespace(), {}) == 0 - mock_run.assert_called_with("/b/program") - assert "Running program from path '/b/program'" in caplog.text + program = str(CORE.firmware_bin) + mock_run.assert_called_with(program) + assert f"Running program from path '{program}'" in caplog.text + + +def test_write_cpp_file_project_generation_follows_host_toolchain() -> None: + """Only PlatformIO gets a platformio.ini; ESP-IDF writes its CMake project + here, and the other native builds generate theirs at compile time.""" + setup_core(platform=PLATFORM_HOST) + with ( + patch("esphome.writer.write_cpp"), + patch("esphome.build_gen.platformio.write_project") as mock_pio, + patch("esphome.build_gen.espidf.write_project") as mock_idf, + ): + CORE.toolchain = Toolchain.HOST + assert main.write_cpp_file() == 0 + mock_pio.assert_not_called() + mock_idf.assert_not_called() + CORE.toolchain = Toolchain.PLATFORMIO + assert main.write_cpp_file() == 0 + mock_pio.assert_called_once() + CORE.toolchain = Toolchain.ESP_IDF + assert main.write_cpp_file() == 0 + mock_idf.assert_called_once() + + +def test_compile_program_host_uses_the_platform_hook() -> None: + """The host component's run_compile claims the build.""" + setup_core(platform=PLATFORM_HOST) + CORE.toolchain = Toolchain.HOST + with ( + patch("esphome.components.host.run_compile", return_value=True) as hook, + patch("esphome.__main__._check_and_emit_build_info") as build_info, + ): + assert compile_program(MagicMock(), {}) == 0 + hook.assert_called_once() + build_info.assert_called_once() + + +def test_compile_program_native_toolchain_needs_a_backend() -> None: + """A native toolchain no hook claims must not fall through to PlatformIO.""" + setup_core(platform=PLATFORM_HOST) + CORE.toolchain = Toolchain.HOST + with ( + patch("esphome.platformio.toolchain.run_compile") as mock_pio, + # A platform package without a run_compile hook + patch.dict(sys.modules, {"esphome.components.host": SimpleNamespace()}), + pytest.raises(EsphomeError, match="no platform backend claimed the build"), + ): + compile_program(MagicMock(), {}) + mock_pio.assert_not_called() + + +def test_native_backend_resolves_host() -> None: + from esphome.build_helpers.native import native_backend + from esphome.host import toolchain as host_toolchain + + setup_core(platform=PLATFORM_HOST) + CORE.toolchain = Toolchain.PLATFORMIO + assert native_backend() is None + CORE.toolchain = Toolchain.HOST + assert native_backend() is host_toolchain + # A native toolchain without a backend for the platform must not degrade + CORE.toolchain = Toolchain.ARDUINO + with pytest.raises( + EsphomeError, match="has no native build backend module for platform host" + ): + native_backend() + + +def test_command_idedata_host_prints_json(capsys: CaptureFixture) -> None: + setup_core(platform=PLATFORM_HOST) + CORE.toolchain = Toolchain.HOST + data = {"cxx_path": "g++", "prog_path": "/build/program"} + with patch("esphome.host.toolchain.get_idedata", return_value=data) as mock_get: + assert command_idedata(MagicMock(), CORE.config) == 0 + mock_get.assert_called_once_with() + assert json.loads(capsys.readouterr().out) == data + + +def _native_host_tools(tmp_path: Path) -> tuple[Path, Path, Path]: + objdump, readelf, elf = (tmp_path / n for n in ("objdump", "readelf", "program")) + for tool in (objdump, readelf, elf): + tool.write_text("") + return objdump, readelf, elf + + +def test_command_analyze_memory_native_toolchain( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_write_cpp: Mock, + mock_compile_program: Mock, + mock_get_esphome_components: Mock, + mock_memory_analyzer_cli: Mock, + mock_ram_strings_analyzer: Mock, +) -> None: + """A native backend supplies its own binutils and ELF, with no idedata.""" + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") + CORE.toolchain = Toolchain.HOST + objdump, readelf, elf = _native_host_tools(tmp_path) + config = {CONF_ESPHOME: {CONF_NAME: "dev"}} + with ( + patch("esphome.host.toolchain.get_objdump_path", return_value=objdump), + patch("esphome.host.toolchain.get_readelf_path", return_value=readelf), + patch("esphome.host.toolchain.get_elf_path", return_value=elf), + patch("esphome.host.toolchain.check_analysis_supported"), + ): + assert command_analyze_memory(MockArgs(), config) == 0 + mock_memory_analyzer_cli.assert_called_once_with( + str(elf), str(objdump), str(readelf), set(), idedata=None + ) + mock_ram_strings_analyzer.assert_called_once_with( + str(elf), objdump_path=str(objdump), platform="host" + ) + assert "Mock Memory Report" in capfd.readouterr().out + + +def test_command_analyze_memory_native_missing_tool_fails( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + mock_write_cpp: Mock, + mock_compile_program: Mock, +) -> None: + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") + CORE.toolchain = Toolchain.HOST + objdump, readelf, elf = _native_host_tools(tmp_path) + readelf.unlink() + with ( + patch("esphome.host.toolchain.get_objdump_path", return_value=objdump), + patch("esphome.host.toolchain.get_readelf_path", return_value=readelf), + patch("esphome.host.toolchain.get_elf_path", return_value=elf), + patch("esphome.host.toolchain.check_analysis_supported"), + caplog.at_level(logging.ERROR), + ): + assert command_analyze_memory(MockArgs(), {}) == 1 + assert f"{readelf} is missing; the toolchain install may be incomplete" in ( + caplog.text + ) + + +def test_command_analyze_memory_host_missing_elf_fails( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + mock_write_cpp: Mock, + mock_compile_program: Mock, +) -> None: + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") + CORE.toolchain = Toolchain.HOST + objdump, readelf, elf = _native_host_tools(tmp_path) + elf.unlink() + with ( + patch("esphome.host.toolchain.get_objdump_path", return_value=objdump), + patch("esphome.host.toolchain.get_readelf_path", return_value=readelf), + patch("esphome.host.toolchain.get_elf_path", return_value=elf), + patch("esphome.host.toolchain.check_analysis_supported"), + caplog.at_level(logging.ERROR), + ): + assert command_analyze_memory(MockArgs(), {}) == 1 + assert f"{elf} is missing; compile the configuration first" in caplog.text + + +def test_command_analyze_memory_host_refuses_before_compiling( + tmp_path: Path, + mock_write_cpp: Mock, + mock_compile_program: Mock, +) -> None: + """A machine whose host program is not ELF fails before the compile.""" + setup_core(platform=PLATFORM_HOST, tmp_path=tmp_path, name="dev") + CORE.toolchain = Toolchain.HOST + with ( + patch("esphome.host.toolchain.sys.platform", "darwin"), + pytest.raises(EsphomeError, match="analyze-memory reads ELF files"), + ): + command_analyze_memory(MockArgs(), {}) + mock_write_cpp.assert_not_called() + mock_compile_program.assert_not_called() diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 512c883c37..dba8977f1a 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -407,6 +407,28 @@ def test_convert_libraries_redownloads_when_manifest_missing( assert top[0].data["name"] == "A" +def test_convert_libraries_manifest_optional_uses_default_layout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A backend accepting manifest-less libraries (the host build, like + PlatformIO's native platform) gets a default manifest instead of a + re-download that would repeat every build.""" + calls = _patch_download_without_manifest( + monkeypatch, tmp_path, manifest_on_force=True + ) + emitted: list[ConvertedLibrary] = [] + backend = _backend(emit=emitted.append) + backend.manifest_optional = True + + with caplog.at_level(logging.DEBUG, logger="esphome.platformio.library"): + top = convert_libraries([Library("esphome/A", "1.0.0", None)], backend) + + assert calls == [False] + assert top[0].data == {"name": "esphome/A"} + assert emitted == top + assert "has no manifest; using PlatformIO's default layout" in caplog.text + + def test_convert_libraries_raises_when_manifest_missing_after_retry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1251,3 +1273,50 @@ def test_versionless_dependency_matching_resolved_manifest_name_stays_quiet( _backend(), ) assert "has no version to resolve" not in caplog.text + + +def test_convert_libraries_symlink_url_resolves_as_local(setup_core: Path) -> None: + """symlink:// is PlatformIO's other spelling for a local library folder.""" + src = setup_core / "lib_dev" + (src / "src").mkdir(parents=True) + (src / "library.json").write_text(json.dumps({"name": "benchmark"})) + url = src.as_uri().replace("file://", "symlink://", 1) + + # Both the name=URL form and an explicit repository take the scheme + for library in ( + Library(f"benchmark={url}", None, None), + Library("benchmark", None, url), + ): + top = convert_libraries([library], _backend()) + assert isinstance(top[0].source, LocalSource) + assert top[0].source_path == src + + +def test_convert_libraries_incompatible_names_the_platform_without_a_framework( + setup_core: Path, +) -> None: + """The host has no framework; the error must not read 'compatible with None'.""" + src = setup_core / "lib_dev" + src.mkdir() + (src / "library.json").write_text( + json.dumps({"name": "Only32", "platforms": "espressif32"}) + ) + backend = _backend() + backend.platform = "native" + backend.framework = None + with pytest.raises(RuntimeError, match="Only32 is not compatible with native"): + convert_libraries([Library("Only32", None, src.as_uri())], backend) + + +@pytest.mark.parametrize( + ("url", "message"), + [ + ("symlink://host/lib", "Unsupported host in symlink:// library URL"), + ("symlink:lib_dev", "symlink:// library URL .* must be an absolute"), + ], +) +def test_convert_libraries_symlink_url_errors_name_the_scheme( + setup_core: Path, url: str, message: str +) -> None: + with pytest.raises(RuntimeError, match=message): + convert_libraries([Library("benchmark", None, url)], _backend())