Merge branch 'esp8266-native-build-infra' into esp8266-native-framework-installer

This commit is contained in:
J. Nick Koston
2026-08-22 11:38:49 -05:00
28 changed files with 99 additions and 285 deletions
+6 -16
View File
@@ -865,11 +865,8 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
RuntimeError,
ValueError,
) as err:
# The firmware already built; idedata is a bonus artifact here.
# Broad on purpose: a vanished compiler (OSError), a failed
# include probe (RuntimeError), or a truncated compile DB
# (ValueError/LookupError) must not fail a successful build
# either.
# Broad on purpose: the firmware already built; an idedata
# failure must not fail a successful build.
_LOGGER.warning("Could not generate idedata: %s", err)
else:
from esphome.platformio import toolchain
@@ -2737,12 +2734,8 @@ def run_esphome(argv):
cache_write_eligible = (
args.command in ("upload", "logs") and not command_line_substitutions
)
# An explicit CLI toolchain must run the per-platform validators; the
# cache was validated under whatever the last compile used. Only the
# read is gated: the refresh below still saves the freshly validated
# config. The sidecar is only written when none exists; a
# compile-written one keeps the compile's toolchain (the firmware on
# disk was built by it), which upload/logs then restore.
# An explicit --toolchain must re-run the per-platform validators, so
# gate only the cache read; the refresh below still saves the result.
cache_read_eligible = cache_write_eligible and args.toolchain is None
if cache_read_eligible:
from esphome.compiled_config import load_compiled_config
@@ -2768,11 +2761,8 @@ def run_esphome(argv):
return 2
CORE.config = config
# Every platform resolves the toolchain during validation now, but the
# compiled-config cache fast path skips validation entirely and a
# sidecar written before the toolchain field existed restores nothing;
# this fallback covers that path. Must run before the cache refresh
# below so its sidecar records the same toolchain a compile would.
# The cache fast path skips validation, and legacy sidecars lack the
# toolchain field. Must run before the cache refresh below.
if CORE.toolchain is None:
CORE.toolchain = Toolchain.PLATFORMIO
+6 -19
View File
@@ -1,13 +1,5 @@
"""Shared ccache policy for build backends.
``ccache_defaults_env`` serves the backends that export ``CCACHE_*`` into a
build subprocess (native ESP-IDF and Arduino); ``resolve_ccache_path``
carries the probe and enable rules (PlatformIO and the native Arduino
build). The ESP-IDF backend keeps ``IDF_CCACHE_ENABLE`` as a
higher-precedence override and falls back to the shared resolver (probe
included) when it is unset; PlatformIO feeds its SCons wrapper script
through env channels instead of ``CCACHE_*`` defaults.
"""
"""Shared ccache policy for build backends: env-knob parsing, binary
resolution, and default ``CCACHE_*`` values."""
from __future__ import annotations
@@ -50,12 +42,8 @@ def parse_enable_env(name: str) -> bool | None:
def resolve_ccache_path() -> str | None:
"""The ccache binary to wrap compiles with, or None when disabled.
Shared policy for every backend: on by default when a runnable ccache is
on PATH, ``ESPHOME_CCACHE_ENABLE=0`` opts out, and an explicit ``=1``
warns when no binary is found and skips the runnability probe;
any other value warns and is treated as unset. The
Windows extended-length prefix is stripped before probing so the probe
validates the exact string the build will execute (#18399).
An explicit ``ESPHOME_CCACHE_ENABLE=1`` skips the runnability probe; the
Windows extended-length prefix is stripped before probing (#18399).
"""
import shutil
@@ -85,9 +73,8 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
"""
from esphome.core import CORE
# build_path is set during preload for every config-loading command; unset
# means the caller built the environment too early. Fail loudly rather
# than silently drop CCACHE_BASEDIR (losing cross-device cache hits).
# An unset build_path means the env was built before preload; fail loudly
# rather than silently drop CCACHE_BASEDIR.
if CORE.build_path is None:
raise ValueError(
"CORE.build_path must be set before constructing the build environment"
+4 -9
View File
@@ -123,10 +123,8 @@ def _pick_entry(entries: list[dict]) -> dict:
raise ValueError("no C++ translation unit found in compile_commands.json")
# Compiler launchers that may prefix a compile command. A closed denylist is
# sturdier than trying to enumerate compiler names: launchers are few and
# stable, while compilers (cross prefixes, versioned names, icx, armcc, ...)
# are an open set.
# Compiler launchers that may prefix a compile command; a closed launcher
# denylist beats enumerating compiler names, an open set.
_LAUNCHER_STEMS = frozenset({"ccache", "sccache", "distcc", "icecc", "buildcache"})
@@ -300,17 +298,14 @@ def idedata_from_build(compile_commands: Path, launcher: str | None = None) -> d
representative = _pick_entry(entries)
cxx_path, defines, rep_includes, cxx_flags = parse_entry(representative, launcher)
if _is_launcher(cxx_path):
# Checked before the toolchain probe (which would fail opaquely on
# a launcher) so the unusable compile DB is named, and never
# cached or conflated with "nothing built yet"
# Reject before the toolchain probe, which would fail opaquely on
# a launcher; never cache the unusable compile DB
raise EsphomeError(
f"compile_commands.json names the launcher {cxx_path} as the "
"compiler; the compile database is unusable"
)
# Seed with the representative's includes so it is not parsed twice
# (per-file -c/-o arguments make every command distinct, so memoizing
# whole commands would never hit)
build_includes: dict[str, None] = dict.fromkeys(
rep_includes if _is_esphome_src(representative["file"]) else ()
)
+8 -21
View File
@@ -25,11 +25,7 @@ def _ninja_runs(binary: str) -> bool:
def find_ninja() -> Path:
"""Locate the ninja binary: a runnable PATH hit first, else the ninja
PyPI wheel.
The wheel is a requirements.txt dependency, so pip has already
integrity-checked it; no download logic is needed here.
"""
PyPI wheel."""
if binary := shutil.which("ninja"):
binary = strip_win_long_path_prefix(binary)
if _ninja_runs(binary):
@@ -58,13 +54,9 @@ def escape(value: Path | str) -> str:
def quote_arg(tok: str) -> str:
"""Wrap a token in double quotes with the Windows argv rule.
Same escaping rule as ``subprocess.list2cmdline``: a backslash run
doubles only immediately before a quote (or the closing quote), and the
quote itself is escaped. CreateProcess-only; POSIX sh collapses
backslash runs inside double quotes, so shell_token single-quotes
there instead. ``$`` must already be doubled for ninja.
"""Quote with the CreateProcess argv rule (as ``subprocess.list2cmdline``):
backslash runs double only before a quote. Windows-only; ``$`` must
already be doubled for ninja.
"""
quoted = re.sub(r'(\\*)"', lambda m: m.group(1) * 2 + '\\"', tok)
quoted = re.sub(r"(\\+)\Z", lambda m: m.group(1) * 2, quoted)
@@ -78,16 +70,11 @@ _NEEDS_QUOTE = re.compile(r"[^\w@%+=:,./-]")
def shell_token(tok: str, force: bool = False) -> str:
"""Quote a lexed token only when needed; ``force`` always quotes.
"""Re-quote a lexed token for the platform shell; ``force`` always quotes.
Lexing strips the quoting a user wrote (``-DX="a b"`` becomes the single
token ``-DX=a b``); re-quote on the way out so the compiler receives the
same argv element SCons would pass under PlatformIO. Ninja hands POSIX
commands to ``/bin/sh -c`` and Windows commands to CreateProcess, so the
quoting style is chosen per platform: single quotes on POSIX (sh expands
nothing inside them, matching SCons's no-shell spawn) and the argv rule
on Windows. ``$`` is doubled first in either case because ninja expands
``$`` before the command reaches the shell.
Single quotes on POSIX (/bin/sh), the argv rule on Windows
(CreateProcess). ``$`` is doubled first because ninja expands it before
the command reaches the shell.
"""
tok = tok.replace("$", "$$") # ninja would expand a bare $ to nothing
if not (force or not tok or _NEEDS_QUOTE.search(tok)):
+1 -5
View File
@@ -4,11 +4,7 @@ from __future__ import annotations
def format_bar(used: int, total: int) -> str:
"""Match PlatformIO's ``_format_availale_bytes`` (pioupload.py) exactly.
The upstream helper's name really is spelled that way; keep the citation
verbatim so it stays greppable in the PlatformIO source.
"""
"""Match PlatformIO's ``_format_availale_bytes`` (sic, pioupload.py) exactly."""
pct_raw = used / total if total else 0
blocks = 10
filled = min(int(round(blocks * pct_raw)), blocks)
+3 -4
View File
@@ -105,10 +105,9 @@ def _refresh_sidecar() -> bool:
and CORE.toolchain is not None
and old.toolchain != CORE.toolchain.value
):
# The config was validated under a different toolchain than
# the compile's, and platforms normalize toolchain-sensitive
# keys (e.g. the esp32 board name) differently; caching it
# would disagree with the sidecar until the next compile
# Platforms normalize toolchain-sensitive keys differently;
# never cache a config validated under a different toolchain
# than the compile's
_LOGGER.debug(
"Not caching: config validated with toolchain %r but the "
"last compile used %r",
+3 -8
View File
@@ -362,14 +362,9 @@ BOARDS = {
}
# Per-board Arduino core build metadata for the native (PlatformIO-free)
# toolchain: the variant directory (supplies pins_arduino.h) and the
# board-identity defines the PlatformIO builder passes via build.extra_flags.
# Valid for platform 4.x only (older tags differ, e.g. esp8285's variant);
# the native toolchain's validator enforces that pairing by requiring core
# >= 3.1.1 and rejecting a custom platform_version.
# -DESP8266 and -DARDUINO_ARCH_ESP8266 are shared by every board and added by
# the generator; only the per-board defines are listed here.
# Per-board variant dir + identity defines from platform-espressif8266 4.x
# build.extra_flags; the shared -DESP8266/-DARDUINO_ARCH_ESP8266 are added
# by the generator.
#
# Regenerate ESP8266_BOARD_BUILD with (v4.2.1 is the platform version the
# native toolchain mirrors; regenerate against the tag when bumping it):
+2 -8
View File
@@ -84,8 +84,6 @@ def apply_testing_memory_patches(content: str, segments: Collection[str]) -> str
"""
for segment in _TESTING_SEGMENT_SIZES:
if segment not in segments and _segment_line_re(segment).search(content):
# A known segment left unpatched would keep its real memory limit
# and silently under-provision the testing build
raise RuntimeError(
f"Testing-mode segment {segment} is present in the linker "
"script but was not selected for patching"
@@ -111,12 +109,8 @@ def segment_length(content: str, segment_name: str) -> int | None:
def surgery_fingerprint() -> str:
"""Fingerprint of this module's source, covering every behavioral input.
Linker-script caches include it so an edit here invalidates them; hashing
the source over-invalidates on comment edits, which is the safe direction.
Native-toolchain-only, like ``segment_length``; no script twin.
"""
"""Hash of this module's source; linker-script caches include it so an
edit here invalidates them."""
import inspect
import sys
+4 -14
View File
@@ -2540,12 +2540,8 @@ def platformio_version_constraint(value):
def _check_supported_toolchain(
platform_name: str, supported: tuple[Toolchain, ...]
) -> None:
"""Raise when the resolved ``CORE.toolchain`` is not in ``supported``.
One message shape for every platform, so a ``--toolchain`` a platform
cannot serve always fails by name instead of silently building with a
different backend.
"""
"""Raise when the resolved ``CORE.toolchain`` is not in ``supported``
(one message shape for every platform)."""
toolchain = CORE.toolchain
if toolchain is None:
# A caller ran the check before resolving; an ordering bug, not a
@@ -2591,14 +2587,8 @@ def resolve_toolchain(
def require_platformio_toolchain(
platform_name: str,
) -> Callable[[ConfigType], ConfigType]:
"""Reject a CLI-selected toolchain other than PlatformIO.
For platforms with only the PlatformIO backend. Without this a
``--toolchain`` they cannot serve would either build with PlatformIO
while claiming another backend, or (for a toolchain another platform
owns, like ``esp-idf``) dispatch to a native backend that cannot
build this platform at all.
"""
"""Reject a CLI-selected toolchain other than PlatformIO, for platforms
with only the PlatformIO backend."""
return resolve_toolchain(
platform_name, (Toolchain.PLATFORMIO,), Toolchain.PLATFORMIO
)
+2 -6
View File
@@ -985,12 +985,8 @@ class EsphomeCore:
@property
def using_toolchain_arduino(self):
"""The native (PlatformIO-free) ESP8266 Arduino build backend.
Unlike ``using_arduino`` (the target *framework*, true for any
platform compiling Arduino code), this is a build *toolchain*
choice, like its ``using_toolchain_*`` siblings.
"""
"""The native ESP8266 Arduino build toolchain (unlike
``using_arduino``, which is the target framework)."""
return self.toolchain == Toolchain.ARDUINO
@property
+3 -11
View File
@@ -566,11 +566,7 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No
if CORE.using_native_toolchain:
# The native builds don't read platformio.ini; honor the options
# with a native equivalent and warn about the rest, which would
# otherwise be silently ignored. Every dispatch site that tests a
# specific using_toolchain_* as a stand-in for "native" (project
# writing, compile, upload, firmware paths) must agree with this
# gate: a toolchain treated as native here must never fall through
# to a PlatformIO code path there.
# otherwise be silently ignored.
for key, val in pio_options.items():
vals = [val] if isinstance(val, str) else val
if key == CONF_BUILD_FLAGS:
@@ -600,12 +596,8 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No
# discovered dependencies
cg.add_platformio_option(key, vals)
elif key in NATIVE_ARDUINO_PIO_OPTIONS and CORE.using_toolchain_arduino:
# Real-world knobs many published ESP8266 configs rely on:
# f_cpu 160000000L for timing-sensitive integrations, and a
# custom ldscript to reserve a filesystem region or correct
# a board's flash size. The esp8266 native generator reads
# both; other native toolchains have no equivalent and fall
# through to the warning.
# The esp8266 native generator reads these; other native
# toolchains have no equivalent and fall through to the warning.
cg.add_platformio_option(key, val)
elif key != "upload_speed":
# upload_speed needs no handling: it is read from the raw
+2 -4
View File
@@ -92,10 +92,8 @@ def generate_cmakelists_txt(component: IDFComponent) -> str:
build_flags = ensure_list(
component.data.get("build", {}).get("flags", DEFAULT_BUILD_FLAGS)
)
# PlatformIO shell-lexes each build.flags entry, so one entry can carry a
# flag and its argument (e.g. "-include cp_custom_alloc.h"); bare
# -I/-L/-l/-D tokens re-glue to their argument ("-I foo" -> "-Ifoo") so
# prefix classifiers below still route them.
# PlatformIO shell-lexes each build.flags entry; bare -I/-L/-l/-D tokens
# re-glue to their argument so the prefix classifiers below route them.
# Joined per entry, as SCons's ParseFlags lexes each string
# independently: a dangling -I ending one entry must warn, not absorb
# the next entry's first token.
+2 -7
View File
@@ -1155,11 +1155,8 @@ def _ccache_env() -> dict[str, str]:
Only values the user has not already set in the environment are returned, so
a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected.
"""
# Honor an explicit choice already in the environment (opt-out or opt-in).
# IDF_CCACHE_ENABLE (this backend's native knob) wins over the shared
# ESPHOME_CCACHE_ENABLE, which resolve_ccache_path parses; without it a
# user disabling ccache to debug a miscompile would silently keep it
# enabled here.
# IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared
# ESPHOME_CCACHE_ENABLE.
idf_knob = parse_enable_env("IDF_CCACHE_ENABLE")
if idf_knob is False:
return {}
@@ -1167,8 +1164,6 @@ def _ccache_env() -> dict[str, str]:
# ESP-IDF silently skips ccache without the binary; don't enable it.
return {}
# ccache is enabled past here; the shared helper carries the CCACHE_*
# policy (and the fail-loud build_path guard).
env = ccache_defaults_env(get_idf_tools_path() / "ccache")
if idf_knob is None:
# An unparsable IDF_CCACHE_ENABLE must not leak to idf.py as truthy
+2 -4
View File
@@ -199,10 +199,8 @@ def run_command(
def tool_version_runs(binary: str, warning: str) -> bool:
"""Probe ``binary --version``; on failure warn with ``warning`` % binary.
``shutil.which`` proves existence, not runnability: on Windows it also
matches ``.bat``/``.cmd`` wrappers and stale package-manager shims whose
target is gone. Callers probe once and fall back instead of failing
every build step with an opaque OS error.
``shutil.which`` proves existence, not runnability (Windows .bat/.cmd
shims, stale package-manager shims).
"""
try:
subprocess.run(
+3 -6
View File
@@ -141,9 +141,7 @@ class _FakeSConsEnv:
def Append(self, **kwargs) -> None: # noqa: N802 (SCons API name)
for key, value in kwargs.items():
if key not in _CAPTURED_KEYS:
# Diagnosable from the build log when a script configures
# something this shim does not translate; once per key so a
# loop of Appends cannot spam
# Warn once per key so a loop of Appends cannot spam
if key not in self._warned_keys:
self._warned_keys.add(key)
_LOGGER.warning(
@@ -239,9 +237,8 @@ def run_extra_script(
)
return ExtraScriptResult()
except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught
# Discard any partial capture: folding half a script's flags into the
# build could produce wrong-output firmware that links cleanly. The
# warning plus the resulting loud link error point back here.
# Discard any partial capture: half-applied flags could build wrong
# firmware that links cleanly.
_LOGGER.warning(
"PIO extra-script %s (in %s) raised %r; ignoring its output",
script_path,
+2 -5
View File
@@ -47,11 +47,8 @@ DEFAULT_BUILD_SRC_FILTER = (
DEFAULT_BUILD_SRC_DIRS = "src"
DEFAULT_BUILD_INCLUDE_DIR = "include"
DEFAULT_BUILD_FLAGS = []
# Source suffix -> compiler kind, PlatformIO's CSUFFIXES/CXXSUFFIXES/ASSUFFIXES
# split. Native build generators map the kind to their compile rules. "asm"
# deliberately merges SCons's AS (.s/.asm) and ASPP (.S/.spp/.sx) sets: the
# ninja rules compile all of them as assembler-with-cpp, whose asm-mode
# preprocessor passes non-directive text through unchanged.
# Suffix -> compiler kind (PlatformIO's CSUFFIXES/CXXSUFFIXES/ASSUFFIXES).
# "asm" merges SCons's AS and ASPP sets: all compile as assembler-with-cpp.
SOURCE_KIND_FOR_SUFFIX: dict[str, str] = {
".c": "c",
".cpp": "cxx",
+13 -30
View File
@@ -1,9 +1,5 @@
"""Install packages from the PlatformIO registry without PlatformIO.
Native toolchains install the exact registry packages the PlatformIO backend
uses, so the bits are identical, but resolve and verify them with esphome's
own download machinery instead of importing the platformio package.
"""
"""Install packages from the PlatformIO registry without importing the
platformio package (identical bits, esphome's own download machinery)."""
from __future__ import annotations
@@ -33,11 +29,9 @@ _REGISTRY_URL = (
def get_systype() -> str:
"""The registry system tag for the current host.
A transliteration of ``platformio.util.get_systype()``, honoring the same
``PLATFORMIO_SYSTEM_TYPE`` override, so this module never imports the
platformio package. One deviation: windows-arm64 maps straight to
``windows_amd64``: the registry ships no arm64 toolchains and those hosts
run x86 binaries via emulation, which upstream leaves to the override.
Transliterates ``platformio.util.get_systype()`` (same
``PLATFORMIO_SYSTEM_TYPE`` override). Deviation: windows-arm64 maps to
``windows_amd64`` (no arm64 toolchains; x86 emulation).
"""
if systype := os.environ.get("PLATFORMIO_SYSTEM_TYPE"):
return systype
@@ -100,10 +94,8 @@ def registry_download(package: str, version: str) -> tuple[str, str, int | None]
f"Unexpected package registry response for {package}: "
f"{str(ver)[:200]}"
)
# Only a MISSING key means "any system"; an explicitly empty
# list must not match (a wrong-architecture download would be
# cached as a good install). A bare string would make ``in`` a
# substring test.
# Only a missing key means "any system"; an empty list must not
# match, and a bare string would make ``in`` a substring test.
systems = file.get("system")
if systems is None:
systems = ["*"]
@@ -138,12 +130,8 @@ def registry_download(package: str, version: str) -> tuple[str, str, int | None]
def _check_layout(name: str, dest: Path, expect: Collection[str]) -> None:
"""Raise when an install tree is missing an expected directory.
Runs on fresh extracts and on marker hits: a marked tree that later
lost files (manual deletion, antivirus quarantine) must fail by name
instead of surfacing as an opaque toolchain error.
"""
"""Raise when an install tree is missing an expected directory (runs on
fresh extracts and on marker hits)."""
for rel in expect:
if not (dest / rel).is_dir():
raise EsphomeError(
@@ -177,21 +165,16 @@ def install_package(
return
from filelock import FileLock
# The cache is machine-global; serialize concurrent cold builds so one
# process cannot wipe the directory another is extracting into (same
# filelock pattern as platformio/toolchain.py and git.py).
# Serialize concurrent cold builds (same filelock pattern as git.py).
dest.parent.mkdir(parents=True, exist_ok=True)
# fallback_to_soft would silently degrade to an existence lock on a
# flock-less filesystem; a hard-killed run would then hang every later
# build forever (same hazard git.py documents).
# A soft-lock fallback would turn a hard-killed run into a permanent
# hang (see git.py).
with FileLock(f"{dest}.lock", fallback_to_soft=False):
if marker.is_file():
# Another process finished the install while we waited
return
rmdir(dest, msg=f"Clean up incomplete {name} install")
# A persistent download location (not a temp dir) so an interrupted
# download resumes across esphome runs via download_with_resume's
# .part file, mirroring the espidf dist/ convention.
# Persistent location so an interrupted download resumes across runs.
downloads_dir.mkdir(parents=True, exist_ok=True)
archive = downloads_dir / f"{name}-{version}"
_LOGGER.info("Downloading %s %s ...", name, version)
@@ -429,8 +429,7 @@ def test_load_or_build_idedata_corrupted_cache_is_logged(
def test_load_or_build_idedata_never_caches_a_launcher(tmp_path: Path) -> None:
"""A compile DB naming a launcher as the compiler is rejected by name,
before the toolchain probe could fail opaquely, and never cached."""
"""A compile DB naming a launcher as the compiler is rejected, never cached."""
compile_commands = tmp_path / "compile_commands.json"
compile_commands.write_text(
json.dumps(
+1 -1
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import os
from pathlib import Path
import subprocess
import sys
from unittest.mock import MagicMock, patch
@@ -84,7 +85,6 @@ def test_shell_token_quotes_shell_metacharacters() -> None:
def test_shell_token_posix_roundtrips_through_sh() -> None:
"""Backslash runs, $, backticks, and quotes must reach the compiler
exactly as lexed once ninja un-doubles $$ and /bin/sh strips quotes."""
import subprocess
if sys.platform == "win32":
pytest.skip("POSIX sh quoting")
@@ -2,8 +2,13 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
import pytest
from esphome.components.esp8266 import build_surgery
from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD
from esphome.components.esp8266.build_surgery import (
RATETABLE_RULE,
@@ -110,11 +115,6 @@ def test_board_build_covers_every_board() -> None:
def test_surgery_fingerprint_is_stable_and_sensitive(tmp_path) -> None:
"""The properties the linker-script cache depends on: the fingerprint is
stable across calls and changes when the module's source changes."""
import importlib.util
from pathlib import Path as _Path
import sys
from esphome.components.esp8266 import build_surgery
first = build_surgery.surgery_fingerprint()
assert first == build_surgery.surgery_fingerprint()
@@ -124,7 +124,7 @@ def test_surgery_fingerprint_is_stable_and_sensitive(tmp_path) -> None:
# A modified copy of the module must fingerprint differently
copy = tmp_path / "build_surgery_variant.py"
copy.write_text(
_Path(build_surgery.__file__).read_text(encoding="utf-8")
Path(build_surgery.__file__).read_text(encoding="utf-8")
+ "\nEXTRA_BEHAVIORAL_INPUT = 1\n",
encoding="utf-8",
)
+1 -2
View File
@@ -643,8 +643,7 @@ def test_save_compiled_config_and_sidecar_toolchain_mismatch(
tmp_path: Path, sidecar_toolchain: str | None, saved: bool
) -> None:
"""A config validated under a different toolchain than the compile's
must not overwrite the cache: platforms normalize toolchain-sensitive
keys differently and the sidecar keeps the compile's toolchain."""
must not overwrite the cache."""
yaml_path = _bare_yaml(tmp_path)
_prime_core(tmp_path)
CORE.config = {CONF_ESPHOME: {CONF_NAME: "lite_test"}}
+3 -14
View File
@@ -1,3 +1,4 @@
import importlib
import json
import logging
from pathlib import Path
@@ -48,6 +49,7 @@ from esphome.const import (
TYPE_GIT,
TYPE_LOCAL,
Framework,
Toolchain,
)
from esphome.core import (
CORE,
@@ -3169,9 +3171,6 @@ def test_file__remapped_path_is_directory_raises(setup_core: Path) -> None:
def test_require_platformio_toolchain() -> None:
"""Platforms with only the PlatformIO backend reject other toolchains."""
from esphome.const import Toolchain
from esphome.core import CORE
validator = cv.require_platformio_toolchain("RP2")
CORE.toolchain = None
config: dict = {}
@@ -3186,9 +3185,6 @@ def test_require_platformio_toolchain() -> None:
def test_check_supported_toolchain_unresolved_is_an_ordering_bug() -> None:
"""Calling the check before resolution fails naming the ordering bug,
not a user-facing unsupported-toolchain error."""
from esphome.const import Toolchain
from esphome.core import CORE
CORE.toolchain = None
with pytest.raises(Invalid, match="not resolved before RP2 validation"):
cv._check_supported_toolchain("RP2", (Toolchain.PLATFORMIO,))
@@ -3209,14 +3205,7 @@ def test_check_supported_toolchain_unresolved_is_an_ordering_bug() -> None:
def test_every_platformio_only_platform_rejects_arduino_toolchain(
platform: str, minimal_config: dict
) -> None:
"""The invariant every native-toolchain gate relies on: a platform that
cannot serve a CLI toolchain rejects it at validation (esp32, esp8266,
and nrf52 pin this in their own suites)."""
import importlib
from esphome.const import Toolchain
from esphome.core import CORE
"""A platform that cannot serve a CLI toolchain rejects it at validation."""
module = importlib.import_module(f"esphome.components.{platform}")
CORE.toolchain = Toolchain.ARDUINO
with pytest.raises(Invalid, match="Unsupported toolchain 'arduino'"):
+2 -2
View File
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
import pytest
from esphome.components import esp32 as esp32_module
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
@@ -15,6 +16,7 @@ from esphome.const import (
)
from esphome.core import CORE, Library
from esphome.espidf.component import (
_emit_idf_component,
generate_cmakelists_txt,
generate_idf_component_yml,
generate_idf_components,
@@ -1072,8 +1074,6 @@ def test_idf_component_download_passes_salt() -> None:
def test_emit_idf_component_wires_esp32_target(tmp_path, monkeypatch):
"""Emitting a component resolves the esp32 variant into the shared
extraScript helper."""
from esphome.components import esp32 as esp32_module
from esphome.espidf.component import _emit_idf_component
monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
(tmp_path / "src").mkdir()
@@ -1395,8 +1395,6 @@ def test_get_framework_env_without_python_env_uses_os_path(tmp_path: Path) -> No
def _ccache_patches(tmp_path: Path, which: str | None, build_path: Path | None):
return (
# The gate defers to the shared resolver (which carries the PATH
# lookup, ESPHOME_CCACHE_ENABLE parse, and runnability probe)
patch("esphome.espidf.framework.resolve_ccache_path", return_value=which),
patch(
"esphome.espidf.framework.get_idf_tools_path",
+2 -29
View File
@@ -101,6 +101,7 @@ from esphome.const import (
CONF_WEB_SERVER,
CONF_WIFI,
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
PLATFORM_BK72XX,
PLATFORM_ESP32,
@@ -7139,7 +7140,7 @@ def test_warn_source_tree_mismatch_falls_back_when_stat_fails(
RuntimeError("Could not query builtin include dirs"),
ValueError("no C++ translation unit found"),
KeyError("command"),
None, # replaced with EsphomeError inside (import is function-local)
None, # replaced with EsphomeError inside
],
)
def test_compile_program_espidf_idedata_failure_does_not_fail_build(
@@ -7147,14 +7148,6 @@ def test_compile_program_espidf_idedata_failure_does_not_fail_build(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A post-compile idedata error is a warning: the firmware already built."""
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
Toolchain,
)
from esphome.core import CORE, EsphomeError
if error is None:
error = EsphomeError("compile database is unusable")
CORE.toolchain = Toolchain.ESP_IDF
@@ -7178,14 +7171,6 @@ def test_compile_program_espidf_idedata_success_is_silent(
caplog: pytest.LogCaptureFixture,
) -> None:
"""The healthy path: idedata generated, nothing to warn about."""
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
Toolchain,
)
from esphome.core import CORE
CORE.toolchain = Toolchain.ESP_IDF
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp32",
@@ -7207,14 +7192,6 @@ def test_compile_program_espidf_idedata_none_warns(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A silent None from the post-compile idedata refresh is made visible."""
from esphome.const import (
KEY_CORE,
KEY_TARGET_FRAMEWORK,
KEY_TARGET_PLATFORM,
Toolchain,
)
from esphome.core import CORE
CORE.toolchain = Toolchain.ESP_IDF
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp32",
@@ -7235,8 +7212,6 @@ def test_compile_program_espidf_idedata_none_warns(
def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None:
"""An explicit --toolchain must run the per-platform validators, so the
upload/logs fast path becomes a cache miss."""
from esphome.__main__ import run_esphome
conf = tmp_path / "device.yaml"
conf.write_text("esphome:\n name: t\n")
argv = ["esphome", "--toolchain", "arduino", "logs", str(conf)]
@@ -7255,8 +7230,6 @@ def test_cli_toolchain_still_refreshes_the_validated_config_cache(
"""An explicit --toolchain gates only the cache read; the freshly
validated config is still saved so a later plain run keeps the fast
path (an existing compile-written sidecar keeps its toolchain)."""
from esphome.__main__ import run_esphome
conf = tmp_path / "device.yaml"
conf.write_text("esphome:\n name: t\n")
argv = ["esphome", "--toolchain", "platformio", "logs", str(conf)]
+4 -6
View File
@@ -7,8 +7,10 @@ import sys
from types import SimpleNamespace
from unittest.mock import patch
import platformdirs
import pytest
from esphome.components.nrf52 import _resolve_toolchain
from esphome.components.nrf52.framework import (
_PLATFORMIO_PENV_REQUIREMENTS,
_REQUIREMENTS,
@@ -22,8 +24,9 @@ from esphome.components.nrf52.framework import (
get_sdk_nrf_tools_path,
setup_platformio_python_env,
)
import esphome.config_validation as cv
from esphome.config_validation import Version
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain
from esphome.core import CORE, EsphomeError
from esphome.framework_helpers import get_python_env_executable_path
@@ -558,7 +561,6 @@ def testget_tools_path_blank_env_falls_back_to_default(
Path("") would resolve to the working directory, which clean-all could
then delete by accident.
"""
import platformdirs
monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", value)
expected = (
@@ -570,7 +572,6 @@ def testget_tools_path_blank_env_falls_back_to_default(
def testget_tools_path_default_is_global_cache(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import platformdirs
monkeypatch.delenv("ESPHOME_SDK_NRF_PREFIX", raising=False)
expected = (
@@ -623,9 +624,6 @@ def test_needs_venv_rebuild_on_dangling_interpreter_symlink(tmp_path: Path) -> N
def test_resolve_toolchain_rejects_unsupported() -> None:
"""A --toolchain nRF52 cannot serve fails instead of degrading silently."""
from esphome.components.nrf52 import _resolve_toolchain
import esphome.config_validation as cv
from esphome.const import Toolchain
CORE.toolchain = Toolchain.ARDUINO
with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"):
@@ -2,19 +2,25 @@
from __future__ import annotations
import logging
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.core import EsphomeError
from esphome.platformio.extra_script import (
ExtraScriptResult,
_FakeSConsEnv,
apply_extra_script,
captured_as_build_flags,
run_extra_script,
)
from esphome.platformio.library import ConvertedLibrary as IDFComponent, URLSource
def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
from esphome.platformio.extra_script import (
captured_as_build_flags,
run_extra_script,
)
(tmp_path / "src" / "esp32").mkdir(parents=True)
script = tmp_path / "extra_script.py"
@@ -58,10 +64,6 @@ def test_extra_script_libpath_relative_resolves_against_library_dir(
"""Relative LIBPATH entries must resolve against ``library_dir``, not the
caller's CWD (the shim restores CWD before ``captured_as_build_flags``
runs)."""
from esphome.platformio.extra_script import (
ExtraScriptResult,
captured_as_build_flags,
)
(tmp_path / "lib" / "esp32").mkdir(parents=True)
elsewhere = tmp_path.parent / "not_the_library_dir"
@@ -76,10 +78,6 @@ def test_extra_script_libpath_relative_resolves_against_library_dir(
def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
from esphome.platformio.extra_script import (
ExtraScriptResult,
captured_as_build_flags,
)
outside = tmp_path.parent / "system_lib"
outside.mkdir(exist_ok=True)
@@ -90,7 +88,6 @@ def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "broken.py"
script.write_text("raise RuntimeError('boom')\n")
@@ -106,7 +103,6 @@ def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
from esphome.platformio.extra_script import apply_extra_script
library_dir = tmp_path / "lib"
library_dir.mkdir()
@@ -117,8 +113,6 @@ def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
c.path = library_dir
c.data = {"build": {"extraScript": "../evil.py"}}
from esphome.core import EsphomeError
with pytest.raises(EsphomeError, match="escapes the library directory"):
apply_extra_script(c, board_mcu=lambda: "esp32", pio_platform="espressif32")
# Nothing was folded into flags: the traversal was rejected before
@@ -127,7 +121,6 @@ def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
def test_apply_extra_script_merges_into_existing_flags(tmp_path):
from esphome.platformio.extra_script import apply_extra_script
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
@@ -146,8 +139,6 @@ def test_apply_extra_script_merges_into_existing_flags(tmp_path):
def test_apply_extra_script_malformed_flags_raises(tmp_path) -> None:
"""A null/dict build.flags fails naming the library instead of injecting
a non-string into the compiler command line."""
from esphome.core import EsphomeError
from esphome.platformio.extra_script import apply_extra_script
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
@@ -164,7 +155,6 @@ def test_apply_extra_script_malformed_flags_raises(tmp_path) -> None:
def test_apply_extra_script_callable_target_and_str_flags(tmp_path) -> None:
"""The shared helper resolves the board_mcu callable lazily and normalizes
a string ``build.flags`` value into a list before extending it."""
from esphome.platformio.extra_script import apply_extra_script
(tmp_path / "src").mkdir()
script = tmp_path / "extra.py"
@@ -180,7 +170,6 @@ def test_apply_extra_script_callable_target_and_str_flags(tmp_path) -> None:
def test_apply_extra_script_no_script_and_no_flags(tmp_path) -> None:
from esphome.platformio.extra_script import apply_extra_script
# No extraScript declared: nothing happens, the target is never resolved
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
@@ -203,9 +192,6 @@ def test_apply_extra_script_no_script_and_no_flags(tmp_path) -> None:
def test_apply_extra_script_ignores_uncaptured_env_calls(tmp_path, caplog) -> None:
"""Un-captured env vars and unsupported env methods are skipped but
diagnosable from the build log."""
import logging
from esphome.platformio.extra_script import apply_extra_script
caplog.set_level(logging.DEBUG)
script = tmp_path / "extra.py"
@@ -223,7 +209,6 @@ def test_apply_extra_script_ignores_uncaptured_env_calls(tmp_path, caplog) -> No
def test_apply_extra_script_swallows_script_errors(tmp_path, caplog) -> None:
"""A raising extra-script is best-effort: logged and skipped."""
from esphome.platformio.extra_script import apply_extra_script
script = tmp_path / "extra.py"
script.write_text("raise RuntimeError('boom')\n")
@@ -237,7 +222,6 @@ def test_apply_extra_script_swallows_script_errors(tmp_path, caplog) -> None:
def test_apply_extra_script_pio_platform(tmp_path) -> None:
"""The backend's platform token is exposed to the script as PIOPLATFORM."""
from esphome.platformio.extra_script import apply_extra_script
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=[env.get('PIOPLATFORM')])\n")
@@ -251,8 +235,6 @@ def test_apply_extra_script_pio_platform(tmp_path) -> None:
def test_apply_extra_script_missing_script_raises(tmp_path) -> None:
"""A declared but absent extraScript is a broken package and fails by
name, as it would under PlatformIO."""
from esphome.core import EsphomeError
from esphome.platformio.extra_script import apply_extra_script
c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
c.path = tmp_path
@@ -264,7 +246,6 @@ def test_apply_extra_script_missing_script_raises(tmp_path) -> None:
def test_run_extra_script_failure_discards_partial_capture(tmp_path, caplog) -> None:
"""A crashed script yields an empty result: half-applied flags could
build wrong-output firmware that links cleanly."""
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_text("env.Append(LIBS=['algobsec'])\nraise RuntimeError('boom')\n")
@@ -278,7 +259,6 @@ def test_run_extra_script_failure_discards_partial_capture(tmp_path, caplog) ->
def test_run_extra_script_syntax_error_is_best_effort(tmp_path, caplog) -> None:
"""A vendored script that does not even compile warns and skips instead
of aborting the build."""
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_text("def broken(:\n")
@@ -291,7 +271,6 @@ def test_run_extra_script_syntax_error_is_best_effort(tmp_path, caplog) -> None:
def test_unsupported_env_method_warns_once(caplog) -> None:
"""Repeated calls to the same unsupported method warn only once."""
from esphome.platformio.extra_script import _FakeSConsEnv
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
@@ -304,7 +283,6 @@ def test_unsupported_env_method_warns_once(caplog) -> None:
def test_run_extra_script_sys_exit_is_best_effort(tmp_path, caplog) -> None:
"""A nonzero sys.exit() in a vendored script must not kill the esphome
run, and its output is discarded."""
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_text("import sys\nenv.Append(LIBS=['x'])\nsys.exit(3)\n")
@@ -317,7 +295,6 @@ def test_run_extra_script_sys_exit_is_best_effort(tmp_path, caplog) -> None:
def test_run_extra_script_sys_exit_zero_is_success(tmp_path, caplog) -> None:
"""sys.exit(0) is a normal PlatformIO script ending: the capture is kept."""
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_text("import sys\nenv.Append(LIBS=['algobsec'])\nsys.exit(0)\n")
@@ -330,10 +307,6 @@ def test_run_extra_script_sys_exit_zero_is_success(tmp_path, caplog) -> None:
def test_run_extra_script_unreadable_raises(tmp_path) -> None:
"""An unreadable declared script is a broken package, like a missing one."""
from unittest.mock import patch
from esphome.core import EsphomeError
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_text("")
@@ -348,7 +321,6 @@ def test_run_extra_script_unreadable_raises(tmp_path) -> None:
def test_run_extra_script_bad_encoding_is_best_effort(tmp_path, caplog) -> None:
"""Undecodable content warns and skips, like a SyntaxError."""
from esphome.platformio.extra_script import run_extra_script
script = tmp_path / "extra.py"
script.write_bytes(b"\xff\xfe\x00bad")
@@ -361,7 +333,6 @@ def test_run_extra_script_bad_encoding_is_best_effort(tmp_path, caplog) -> None:
def test_uncaptured_append_key_warns_once(caplog) -> None:
"""A loop of Appends to the same uncaptured key warns once."""
from esphome.platformio.extra_script import _FakeSConsEnv
env = _FakeSConsEnv(
board_mcu="esp8266", pio_env="esphome_esp8266", pio_platform="espressif8266"
+3 -5
View File
@@ -13,6 +13,7 @@ import pytest
from esphome.core import EsphomeError, Library
import esphome.platformio.library as lib
from esphome.platformio.library import (
SOURCE_KIND_FOR_SUFFIX,
ConvertedLibrary,
GitSource,
InvalidLibrary,
@@ -23,6 +24,8 @@ from esphome.platformio.library import (
_resolve_registry_version,
check_library_data,
convert_libraries,
join_flag_args,
split_flag_entry,
)
@@ -535,7 +538,6 @@ def test_convert_libraries_skips_incompatible_dependency(tmp_path, monkeypatch):
def test_split_flag_entry_unbalanced_quote_is_clean() -> None:
"""A malformed flags entry raises EsphomeError, not a raw ValueError."""
from esphome.platformio.library import split_flag_entry
assert split_flag_entry('-DX="a b"', "library x") == ["-DX=a b"]
with pytest.raises(EsphomeError, match=r"Malformed build flag.*library x"):
@@ -544,7 +546,6 @@ def test_split_flag_entry_unbalanced_quote_is_clean() -> None:
def test_join_flag_args_reglues_spaced_define() -> None:
"""A spaced -D re-glues to its argument, as ParseFlags does."""
from esphome.platformio.library import join_flag_args
assert join_flag_args(["-D", "FOO=1", "-Os"], "x") == ["-DFOO=1", "-Os"]
@@ -552,7 +553,6 @@ def test_join_flag_args_reglues_spaced_define() -> None:
def test_join_flag_args_trailing_bare_flag_warns(
caplog: pytest.LogCaptureFixture,
) -> None:
from esphome.platformio.library import join_flag_args
assert join_flag_args(["-Os", "-l"], "library x") == ["-Os"]
assert "Ignoring trailing '-l'" in caplog.text
@@ -561,7 +561,6 @@ def test_join_flag_args_trailing_bare_flag_warns(
def test_split_flag_entry_non_string_is_clean() -> None:
"""A dict or number from a third-party manifest fails naming the entry,
not with an opaque shlex traceback."""
from esphome.platformio.library import split_flag_entry
with pytest.raises(EsphomeError, match="Malformed build flag"):
split_flag_entry({"esp32": ["-DX"]}, "lib x")
@@ -572,7 +571,6 @@ def test_split_flag_entry_non_string_is_clean() -> None:
def test_source_kind_map_shape() -> None:
"""The kind values the native compile rules key on, and the deliberate
AS/ASPP merge (.s and .S both map to asm)."""
from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX
assert set(SOURCE_KIND_FOR_SUFFIX.values()) == {"c", "cxx", "asm"}
assert SOURCE_KIND_FOR_SUFFIX[".s"] == "asm"