Compare commits

..
60 changed files with 1140 additions and 8140 deletions
@@ -1,39 +0,0 @@
name: Cache Arduino ESP8266
description: >
Resolve the pinned Arduino core and xtensa toolchain versions and cache the
native ESP8266 install (~110 MB framework + toolchain; no ccache store, the
seed job saves before any compile runs). Exports
ESPHOME_ARDUINO8266_PREFIX to the job so every later step installs into
the cached path; the Python venv must already be restored. Mirrors
cache-esp-idf: only dev-branch pushes write the shared cache, everything
else restores.
runs:
using: composite
steps:
- name: Resolve the native toolchain cache key
# Versions are pinned in code, not a hashable file; resolve them so a
# bump changes the cache key. Assignment form so errexit catches a
# resolver failure.
id: version
shell: bash
run: |
# One owner for the install prefix: exported here and referenced by
# the cache steps below via env, so the caller's install and the
# cached path cannot diverge.
echo "ESPHOME_ARDUINO8266_PREFIX=$HOME/.esphome-arduino8266" >> "$GITHUB_ENV"
. venv/bin/activate
key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")')
[ -n "$key" ] || exit 1
echo "key=$key" >> "$GITHUB_OUTPUT"
- name: Cache the native toolchain (write on dev)
if: github.ref == 'refs/heads/dev'
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.ESPHOME_ARDUINO8266_PREFIX }}
key: ${{ runner.os }}-esp8266-native-${{ steps.version.outputs.key }}
- name: Restore the native toolchain (off dev)
if: github.ref != 'refs/heads/dev'
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ env.ESPHOME_ARDUINO8266_PREFIX }}
key: ${{ runner.os }}-esp8266-native-${{ steps.version.outputs.key }}
+1 -67
View File
@@ -102,8 +102,6 @@ jobs:
device-builder: ${{ steps.determine.outputs.device-builder }}
esp32-platformio: ${{ steps.determine.outputs.esp32-platformio }}
esp32-platformio-components: ${{ steps.determine.outputs.esp32-platformio-components }}
esp8266-native: ${{ steps.determine.outputs.esp8266-native }}
esp8266-native-components: ${{ steps.determine.outputs.esp8266-native-components }}
changed-components: ${{ steps.determine.outputs.changed-components }}
changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }}
directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }}
@@ -167,8 +165,6 @@ jobs:
echo "device-builder=$(echo "$output" | jq -r '.device_builder')" >> $GITHUB_OUTPUT
echo "esp32-platformio=$(echo "$output" | jq -r '.esp32_platformio')" >> $GITHUB_OUTPUT
echo "esp32-platformio-components=$(echo "$output" | jq -r '.esp32_platformio_components')" >> $GITHUB_OUTPUT
echo "esp8266-native=$(echo "$output" | jq -r '.esp8266_native')" >> $GITHUB_OUTPUT
echo "esp8266-native-components=$(echo "$output" | jq -r '.esp8266_native_components')" >> $GITHUB_OUTPUT
echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT
echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT
echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT
@@ -187,32 +183,6 @@ jobs:
path: .temp/components_graph.json
key: components-graph-${{ hashFiles('esphome/components/**/*.py') }}
seed-esp8266-native-cache:
name: Seed the esp8266 native toolchain cache
runs-on: ubuntu-24.04
needs:
- common
# PR-branch cache saves are invisible to other PRs, so dev pushes seed
# the shared entry test-esp8266-native restores. Only dev: the composite
# action saves nowhere else, so a beta/release push would download the
# toolchain and discard it.
if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
timeout-minutes: 15
steps:
- name: Check out code from GitHub
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Cache the native toolchain
uses: ./.github/actions/cache-arduino8266
- name: Install the native toolchain
run: |
. venv/bin/activate
python -c "from esphome.arduino8266.framework import check_and_install; from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION; check_and_install(RECOMMENDED_ARDUINO_FRAMEWORK_VERSION)"
ci-custom:
name: Run script/ci-custom
runs-on: ubuntu-24.04
@@ -1267,7 +1237,7 @@ jobs:
# compile validates config first, so a separate config pass is
# redundant for this smoke test. ESP-IDF framework via PlatformIO:
python3 script/test_build_components.py -e compile -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain platformio --fail-on-no-tests
python3 script/test_build_components.py -e compile -t esp32-idf -c "$TEST_COMPONENTS" -f --toolchain platformio
echo ""
echo "ESP-IDF-via-PlatformIO build passed! Starting Arduino smoke test..."
@@ -1276,40 +1246,6 @@ jobs:
# Arduino framework via PlatformIO (only components with an esp32-ard test are built):
python3 script/test_build_components.py -e compile -t esp32-ard -c "$TEST_COMPONENTS" -f --toolchain platformio
test-esp8266-native:
name: Test esp8266 components with the native toolchain
runs-on: ubuntu-24.04
needs:
- common
- determine-jobs
if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.esp8266-native == 'true'
env:
# Computed by script/determine-jobs.py (ESP8266_NATIVE_TEST_COMPONENTS)
TEST_COMPONENTS: ${{ needs.determine-jobs.outputs.esp8266-native-components }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Restore Python
uses: ./.github/actions/restore-python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
cache-key: ${{ needs.common.outputs.cache-key }}
- name: Cache the native toolchain
uses: ./.github/actions/cache-arduino8266
- name: Run native toolchain compile test
run: |
. venv/bin/activate
echo "Testing components: $TEST_COMPONENTS"
echo ""
# ESP8266 Arduino built directly (no PlatformIO); compile validates
# config first, so a separate config pass is redundant.
python3 script/test_build_components.py -e compile -t esp8266-ard -c "$TEST_COMPONENTS" -f --toolchain arduino --fail-on-no-tests
device-builder:
name: Test downstream esphome/device-builder
runs-on: ubuntu-24.04
@@ -1672,7 +1608,6 @@ jobs:
needs:
- common
- seed-apt-cache
- seed-esp8266-native-cache
- determine-jobs
- ci-custom
- pylint
@@ -1688,7 +1623,6 @@ jobs:
- clang-tidy-esp32-variants
- test-build-components-split
- test-esp32-platformio
- test-esp8266-native
- device-builder
- memory-impact-target-branch
- memory-impact-pr-branch
+31 -83
View File
@@ -817,9 +817,7 @@ def write_cpp_file() -> int:
from esphome.build_gen import espidf
espidf.write_project()
elif not CORE.using_native_toolchain:
# Other native builds generate their project at compile time;
# never write a platformio.ini for them
else:
from esphome.build_gen import platformio
platformio.write_project()
@@ -861,14 +859,20 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
toolchain.create_factory_bin()
toolchain.create_ota_bin()
toolchain.create_elf_copy()
from esphome.build_helpers.idedata import warn_if_idedata_missing
from esphome.build_helpers.idedata import IDEDATA_BEST_EFFORT_ERRORS
warn_if_idedata_missing(toolchain.get_idedata)
elif CORE.using_native_toolchain:
raise EsphomeError(
f"Toolchain '{CORE.toolchain.value}' resolved but no platform "
"backend claimed the build"
)
try:
if toolchain.get_idedata() is None:
_LOGGER.warning("No idedata was generated for this build")
except IDEDATA_BEST_EFFORT_ERRORS as err:
# The firmware already built; an idedata failure must not fail
# a successful build.
_LOGGER.warning(
"Could not generate idedata: %s (IDE, clang-tidy, and "
"memory-analysis data will be unavailable for this build)",
err,
)
_LOGGER.debug("Idedata failure detail", exc_info=True)
else:
from esphome.platformio import toolchain
@@ -971,15 +975,12 @@ def upload_using_esptool(
if file is not None:
flash_images = [FlashImage(path=file, offset="0x0")]
elif (native := _native_toolchain_module()) is not None:
# Every native backend supplies its own 0x0 flash image (bootloader
# and partitions included where the target needs them)
image = native.get_factory_firmware_path()
if not image.is_file():
raise EsphomeError(
f"{image} does not exist; compile the configuration first"
)
flash_images = [FlashImage(path=image, offset="0x0")]
elif CORE.using_toolchain_esp_idf:
from esphome.espidf import toolchain
flash_images = [
FlashImage(path=toolchain.get_factory_firmware_path(), offset="0x0")
]
else:
from esphome.platformio import toolchain
@@ -1961,39 +1962,15 @@ def command_update_all(args: ArgsProtocol) -> int | None:
return run_multiple_configs(files, build_command)
# Native build backend per (target platform, toolchain). Keyed here rather
# than through a platform hook so the serial upload/logs fast path never
# imports the platform component package (see the esp32 variant comment in
# upload_using_esptool); the platform half comes from CORE.data the same way.
_NATIVE_TOOLCHAIN_MODULES = {
("esp32", Toolchain.ESP_IDF): "esphome.espidf.toolchain",
("esp8266", Toolchain.ARDUINO): "esphome.arduino8266.toolchain",
}
def _native_toolchain_module():
"""The native build backend module for the resolved toolchain."""
if not CORE.using_native_toolchain:
return None
key = (CORE.target_platform, CORE.toolchain)
if (module_path := _NATIVE_TOOLCHAIN_MODULES.get(key)) is None:
# Degrading to the PlatformIO path would build with the wrong backend
raise EsphomeError(
f"Toolchain '{CORE.toolchain.value}' has no native build backend "
f"module for platform {CORE.target_platform}"
)
return importlib.import_module(module_path)
def command_idedata(args: ArgsProtocol, config: ConfigType) -> int:
import json
native_toolchain = _native_toolchain_module()
if CORE.using_toolchain_esp_idf:
# Native ESP-IDF derives idedata from the build's compile_commands.json,
# so the configuration must already be compiled.
from esphome.espidf import toolchain as espidf_toolchain
if native_toolchain is not None:
# Native toolchains derive idedata from the build's
# compile_commands.json, so the configuration must already be compiled.
idedata = native_toolchain.get_idedata()
idedata = espidf_toolchain.get_idedata()
if idedata is None:
_LOGGER.error(
"No idedata available; compile the configuration first",
@@ -2032,17 +2009,6 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int:
from esphome.analyze_memory.cli import MemoryAnalyzerCLI
from esphome.analyze_memory.ram_strings import RamStringsAnalyzer
# Refuse an unsupported toolchain before paying for a full compile
native_toolchain = _native_toolchain_module()
if native_toolchain is None and not CORE.using_toolchain_platformio:
_LOGGER.error(
"analyze-memory is not supported with the '%s' toolchain on %s; "
"re-run with --toolchain platformio",
CORE.toolchain.value if CORE.toolchain else "unresolved",
CORE.target_platform,
)
return 1
# Always compile to ensure fresh data (fast if no changes - just relinks)
exit_code = write_cpp(config)
if exit_code != 0:
@@ -2054,31 +2020,13 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int:
# Get idedata for analysis
idedata = None
if native_toolchain is not None:
objdump = native_toolchain.get_objdump_path()
readelf = native_toolchain.get_readelf_path()
for tool in (objdump, readelf):
if not tool.is_file():
# The analyzer would silently fall back to host binutils,
# which cannot read the target ELF. clean-all is heavy for
# ESP-IDF, so suggest a recompile first.
_LOGGER.error(
"%s is missing; the toolchain install may be incomplete "
"(recompile, or run 'esphome clean-all' if it persists)",
tool,
)
return 1
objdump_path = str(objdump)
readelf_path = str(readelf)
if CORE.using_toolchain_esp_idf:
from esphome.espidf import toolchain
firmware_elf = native_toolchain.get_elf_path()
if not firmware_elf.is_file():
# The analyzer swallows tool failures, so a missing ELF would
# produce an exit-0 zeroed report
_LOGGER.error(
"%s is missing; compile the configuration first", firmware_elf
)
return 1
objdump_path = str(toolchain.get_objdump_path())
readelf_path = str(toolchain.get_readelf_path())
firmware_elf = toolchain.get_elf_path()
else:
from esphome.platformio import toolchain
+1 -4
View File
@@ -19,7 +19,6 @@ from typing import NamedTuple
from esphome.build_helpers.ccache import ccache_defaults_env
from esphome.build_helpers.ninja import find_ninja
from esphome.build_helpers.pch import ccache_pch_env
from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path
from esphome.core import EsphomeError, Version
from esphome.framework_helpers import str_to_lst_of_str
@@ -157,6 +156,4 @@ def ccache_env(ccache: str | None) -> dict[str, str]:
"""
if ccache is None:
return {}
env = ccache_defaults_env(get_arduino8266_tools_path() / "ccache")
env.update(ccache_pch_env())
return env
return ccache_defaults_env(get_arduino8266_tools_path() / "ccache")
-329
View File
@@ -1,329 +0,0 @@
"""Native Arduino ESP8266 build driver (the PlatformIO ``run`` equivalent)."""
from __future__ import annotations
import json
import logging
from pathlib import Path
import subprocess
from typing import Any
from esphome.arduino8266 import framework
from esphome.build_helpers.ccache import resolve_ccache_path
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
CONF_ESPHOME,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
)
from esphome.core import CORE, EsphomeError
from esphome.helpers import write_file_if_changed
from esphome.types import ConfigType
_LOGGER = logging.getLogger(__name__)
# ESP8266 user RAM (matches upload.maximum_ram_size in every board manifest)
_MAX_RAM_SIZE = 81920
def _warn_ignored_platformio_options() -> None:
"""Warn for component-added platformio options the native build drops.
The consumed set is exported by core/config.py next to the routing that
stores these options, so the two cannot drift; YAML upload_speed never
reaches CORE.platformio_options here.
"""
from esphome.core.config import NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS
consumed = NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS
for key in sorted(CORE.platformio_options or {}):
if key not in consumed:
_LOGGER.warning(
"platformio_options->%s is ignored when building with the "
"native 'arduino' toolchain",
key,
)
_RAM_SECTIONS = (".data", ".rodata", ".bss")
_FLASH_SECTIONS = (".irom0.text", ".text", ".text1", ".data", ".rodata")
def get_build_dir() -> Path:
return CORE.relative_pioenvs_path(CORE.name)
def get_elf_path() -> Path:
return get_build_dir() / "firmware.elf"
def _toolchain_tool(name: str) -> Path:
return framework.toolchain_tool(framework.get_toolchain_path(), name)
def get_factory_firmware_path() -> Path:
"""The image to serial-flash at 0x0 (same bytes as firmware.bin: the
8266 factory copy exists for artifact-contract parity, not content)."""
return get_build_dir() / "firmware.factory.bin"
def get_addr2line_path() -> Path:
return _toolchain_tool("addr2line")
def get_objdump_path() -> Path:
return _toolchain_tool("objdump")
def get_readelf_path() -> Path:
return _toolchain_tool("readelf")
def run_compile(config: ConfigType, verbose: bool) -> int:
from esphome.build_gen import arduino8266 as build_gen
_warn_ignored_platformio_options()
paths = framework.check_and_install(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION])
# Resolved once per build: the resolution probes PATH and spawns the
# runnability check, and three consumers need the same answer
ccache = resolve_ccache_path()
ninja_changed = build_gen.write_project(paths, ccache)
build_dir = get_build_dir()
env = framework.get_build_env(paths.toolchain, ccache)
# Regenerate the compile DB before the build (a pure function of
# build.ninja); skip only when it is at least as fresh as build.ninja
# (an interrupted previous run may have rewritten the manifest without
# regenerating the DB).
compdb = build_dir / "compile_commands.json"
compdb_stamp = build_dir / ".compile_commands.stamp"
ninja_file = build_dir / "build.ninja"
# Freshness rides a stamp: the DB itself is written through
# write_file_if_changed (its mtime feeds get_idedata's cache), so a
# regeneration with identical content would stay "stale" forever
if (
ninja_changed
or not compdb.is_file()
or not compdb_stamp.is_file()
or compdb_stamp.stat().st_mtime < ninja_file.stat().st_mtime
):
_write_compile_commands(paths.ninja, build_dir, env)
compdb_stamp.touch()
cmd = [str(paths.ninja)]
if verbose:
cmd.append("-v")
if jobs := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT):
cmd += ["-j", str(jobs)]
# Explicit targets, not the default statement: a generator defect that
# drops them fails loudly with "unknown target" instead of a green
# no-op run that leaves stale artifacts in place
targets = ["firmware.factory.bin", "firmware.ota.bin"]
cmd += targets
# A dry-run probe keeps a no-op rebuild quiet: ninja would only print
# "no work to do". A freshly rewritten manifest all but guarantees work,
# so skip the probe (and its full stat pass) on that path. cwd instead
# of -C also drops the "Entering directory" banner on real builds.
skip_build = False
if not ninja_changed:
probe = subprocess.run(
[str(paths.ninja), "-n", *targets],
cwd=build_dir,
env=env,
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if probe.stderr.strip():
# A load-time diagnostic (e.g. "multiple rules generate X")
# flags a generator bug; the skip branch would otherwise
# swallow it forever
_LOGGER.warning("ninja: %s", probe.stderr.strip())
if probe.returncode != 0:
# An unknown target here is the defective-manifest case; fall
# through to the real build so the error prints attributably
_LOGGER.debug("ninja probe failed; running the full build")
skip_build = probe.returncode == 0 and "no work to do" in probe.stdout
if skip_build:
_LOGGER.debug("ninja: nothing to rebuild")
else:
_LOGGER.debug("Running: %s", " ".join(cmd))
rc = subprocess.run(
cmd, cwd=build_dir, env=env, check=False, close_fds=False
).returncode
if rc != 0:
return rc
# ninja already refused a manifest missing the explicit targets above;
# existence covers the remaining hole (a rule that ran but wrote
# elsewhere). The factory/ota copies are what upload and OTA consume.
build_dir_artifacts = (
get_elf_path(),
build_dir / "firmware.bin",
get_factory_firmware_path(),
build_dir / "firmware.ota.bin",
)
for artifact in build_dir_artifacts:
if not artifact.is_file():
_LOGGER.error("Build produced no %s", artifact)
return 1
if not _print_size_summary(build_dir, paths):
# The cause was already warned; name the consequence so a build
# contributing no RAM/Flash metric is visible to CI harnesses
_LOGGER.warning("Firmware size summary unavailable for this build")
from esphome.build_helpers.idedata import warn_if_idedata_missing
warn_if_idedata_missing(lambda: get_idedata(ccache))
return 0
def _write_compile_commands(
ninja_path: Path, build_dir: Path, env: dict[str, str]
) -> None:
compdb = build_dir / "compile_commands.json"
result = subprocess.run(
[str(ninja_path), "-C", str(build_dir), "-t", "compdb", "c", "cxx", "asm"],
env=env,
capture_output=True,
text=True,
check=False,
close_fds=False,
)
if result.returncode != 0:
# Drop any stale database so consumers (IDE integration, clang-tidy,
# the memory analyzer) can't silently read outdated data.
compdb.unlink(missing_ok=True)
raise EsphomeError(f"Could not generate compile_commands.json: {result.stderr}")
try:
entries = json.loads(result.stdout)
except ValueError as err:
compdb.unlink(missing_ok=True)
raise EsphomeError(
f"ninja produced an unparsable compile database: {err} "
f"(output starts {result.stdout[:120]!r})"
) from err
if not entries:
# compdb exits 0 with [] for unknown rule names; a renamed compile
# rule must fail the build, not silently strand every consumer
compdb.unlink(missing_ok=True)
raise EsphomeError(
"ninja produced an empty compile database; the generator's rule "
"names no longer match"
)
# write_file_if_changed keeps the mtime stable on no-op builds so the
# idedata cache in get_idedata() stays valid.
write_file_if_changed(compdb, result.stdout)
def _parse_app_size(build_dir: Path, paths: framework.InstalledPaths) -> int | None:
"""Read the app flash budget (irom0_0_seg length) from the linker script."""
from esphome.build_gen.arduino8266 import get_flash_ld_path
from esphome.components.esp8266.build_surgery import segment_length
# Warnings, not debug: without the app size the Flash summary line is
# dropped and CI's memory-impact extraction loses its flash metric.
ld_path = get_flash_ld_path(build_dir, paths)
try:
ld_text = ld_path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as err:
# UnicodeDecodeError: a truncated/corrupt script must degrade to
# the same warning, never abort an already-linked build
_LOGGER.warning("Cannot read linker script for the Flash summary: %s", err)
return None
app_size = segment_length(ld_text, "irom0_0_seg")
if app_size is None:
_LOGGER.warning("irom0_0_seg not found in %s; skipping Flash summary", ld_path)
return None
if app_size == 0:
_LOGGER.warning(
"irom0_0_seg has zero length in %s; skipping Flash summary", ld_path
)
return None
return app_size
def _print_size_summary(build_dir: Path, paths: framework.InstalledPaths) -> bool:
"""Print the PlatformIO-shaped RAM/Flash lines; False when skipped.
The exact shape (including the bar) is parsed by
``script/ci_memory_impact_extract.py``; ``print_size_line`` matches it.
"""
from esphome.build_helpers.size_summary import print_size_line
size_tool = _toolchain_tool("size")
try:
result = subprocess.run(
[str(size_tool), "-A", "-d", str(get_elf_path())],
capture_output=True,
text=True,
check=False,
close_fds=False,
)
except OSError as err:
# The summary is a bonus artifact like idedata; a truncated
# toolchain extraction must not discard an already-linked build
_LOGGER.warning("Could not summarize firmware size: %s", err)
return False
if result.returncode != 0:
_LOGGER.warning("Could not summarize firmware size: %s", result.stderr)
return False
sections: dict[str, int] = {}
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0].startswith("."):
try:
sections[parts[0]] = int(parts[1])
except ValueError:
# An unparsed RAM/Flash section trips the missing-sections
# guard below, so no total is built on a dropped value
_LOGGER.warning("Unparsable size output for section %s", parts[0])
if missing := set(_RAM_SECTIONS + _FLASH_SECTIONS) - set(sections):
# A defaulted 0 would print a confidently wrong total for CI's metric
_LOGGER.warning(
"Size output is missing section(s) %s; skipping the size summary",
", ".join(sorted(missing)),
)
return False
# Resolve the flash budget before printing anything: a RAM line without
# its Flash line would let CI's memory-impact extraction sum the two
# metrics over different build counts (_parse_app_size already warned).
app_size = _parse_app_size(build_dir, paths)
if not app_size:
return False
ram = sum(sections[s] for s in _RAM_SECTIONS)
flash = sum(sections[s] for s in _FLASH_SECTIONS)
print_size_line("RAM", ram, _MAX_RAM_SIZE)
print_size_line("Flash", flash, app_size)
return True
# Sentinel: "resolve for me"; None is a real value meaning disabled.
_CCACHE_UNRESOLVED: Any = object()
def get_idedata(ccache: str | None = _CCACHE_UNRESOLVED) -> dict | None:
"""Derive idedata from the build's compile_commands.json.
Same contract as ``espidf.toolchain.get_idedata``: the fields IDE
integrations, clang-tidy, and the memory analyzer expect.
"""
from esphome.build_helpers.idedata import load_or_build_idedata
if ccache is _CCACHE_UNRESOLVED:
# Deliberately uncached: env/PATH can change between builds in a
# long-lived host process
ccache = resolve_ccache_path()
return load_or_build_idedata(
get_build_dir() / "compile_commands.json",
get_elf_path(),
# Suffixed so a platformio->arduino->platformio round trip on one
# config never serves the other toolchain's cache shape
CORE.relative_internal_path("idedata", f"{CORE.name}.arduino.json"),
# The compile DB's commands carry the same ccache prefix the ninja
# rules were generated with
launcher=str(ccache) if ccache else None,
)
File diff suppressed because it is too large Load Diff
+1 -14
View File
@@ -87,19 +87,6 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
"CCACHE_DIR": str(cache_dir),
"CCACHE_NOHASHDIR": "true",
"CCACHE_DEPEND": "1",
# A user value wins via the filter below
"CCACHE_BASEDIR": effective_ccache_basedir(),
"CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
}
return {k: v for k, v in defaults.items() if k not in os.environ}
def effective_ccache_basedir() -> str:
"""The prefix ccache rewrites out of hashed paths: a user CCACHE_BASEDIR
wins, else the resolved build path (matching ccache_defaults_env)."""
from esphome.core import CORE
raw = os.environ.get("CCACHE_BASEDIR")
if raw is not None and Path(raw).is_absolute() and len(Path(raw).parts) > 1:
return raw
# Unset or degenerate ("", "/", relative): fall back to the build path
return str(Path(CORE.build_path).resolve())
+6 -61
View File
@@ -11,7 +11,6 @@ consumers (IDE integration, clang-tidy) expect:
from __future__ import annotations
from collections.abc import Callable
import json
import logging
import os
@@ -22,8 +21,6 @@ import subprocess
from esphome.core import EsphomeError
from esphome.helpers import write_file
_LOGGER = logging.getLogger(__name__)
# Everything idedata generation may raise after a successful link; idedata
# is a bonus artifact, so consumers warn instead of failing the build
IDEDATA_BEST_EFFORT_ERRORS = (
@@ -34,36 +31,13 @@ IDEDATA_BEST_EFFORT_ERRORS = (
ValueError,
)
_LOGGER = logging.getLogger(__name__)
def warn_if_idedata_missing(get_idedata: Callable[[], dict | None]) -> None:
"""Run an idedata generator, downgrading any failure to a warning.
Shared by the native backends: the firmware already built, so a missing
or broken idedata must not fail a successful build.
"""
try:
if get_idedata() is None:
_LOGGER.warning("No idedata was generated for this build")
except IDEDATA_BEST_EFFORT_ERRORS as err:
_LOGGER.warning(
"Could not generate idedata: %s (IDE, clang-tidy, and "
"memory-analysis data will be unavailable for this build)",
err,
)
if isinstance(err, (EsphomeError, OSError)):
# Routine environmental failures keep the detail at debug
_LOGGER.debug("Idedata failure detail", exc_info=True)
else:
# LookupError/ValueError/RuntimeError smell like a parsing bug;
# a permanently masked traceback would hide it on every build
_LOGGER.warning("Idedata failure detail", exc_info=True)
# C++ translation-unit suffixes.
CXX_SOURCE_SUFFIXES = (".cpp", ".cc", ".cxx")
# C++ translation-unit suffixes used to identify ESPHome source files.
_CXX_SUFFIXES = (".cpp", ".cc")
# Suffixes of input/output files that appear bare on the command line (and so
# must not be mistaken for compiler flags).
_INPUT_FILE_SUFFIXES = (*CXX_SOURCE_SUFFIXES, ".c", ".o", ".S", ".s")
_INPUT_FILE_SUFFIXES = (*_CXX_SUFFIXES, ".c", ".o", ".S", ".s")
# Path marker identifying an ESPHome source translation unit.
_ESPHOME_SRC_MARKER = "/src/esphome/"
@@ -72,7 +46,7 @@ def _is_esphome_src(file: str) -> bool:
"""Whether ``file`` is an ESPHome C++ translation unit; normalized to
``/`` first since Windows compile DBs use backslashes."""
return _ESPHOME_SRC_MARKER in file.replace("\\", "/") and file.endswith(
CXX_SOURCE_SUFFIXES
_CXX_SUFFIXES
)
@@ -147,7 +121,7 @@ def _pick_entry(entries: list[dict]) -> dict:
if _is_esphome_src(entry["file"]):
return entry
for entry in entries:
if entry["file"].endswith(CXX_SOURCE_SUFFIXES):
if entry["file"].endswith(_CXX_SUFFIXES):
return entry
raise ValueError("no C++ translation unit found in compile_commands.json")
@@ -161,15 +135,6 @@ def _is_launcher(token: str) -> bool:
return Path(token).stem.lower() in _LAUNCHER_STEMS
def is_joined_include(tok: str) -> bool:
"""The joined ``-includefoo.h`` spelling; excludes clang's -include-pch."""
return (
tok.startswith("-include")
and tok != "-include"
and not tok.startswith("-include-")
)
def parse_entry(
entry: dict, launcher: str | None = None
) -> tuple[str, list[str], list[str], list[str]]:
@@ -204,23 +169,11 @@ def parse_entry(
defines: list[str] = []
includes: list[str] = []
cxx_flags: list[str] = []
unresolved_force_includes: list[str] = []
it = iter(tokens[1:])
for tok in it:
if tok in ("-c", "-o"):
next(it, None) # drop the flag and its argument (input/output)
elif tok == "-include" or is_joined_include(tok):
# Re-anchor only names next to the compile (the pch); a name
# meant for the -I chain must stay untouched
raw = next(it, "") if tok == "-include" else tok[len("-include") :]
if not raw:
_LOGGER.warning("Dropping -include with no argument")
elif Path(resolved := _include(raw)).is_file():
cxx_flags.extend(("-include", resolved))
else:
unresolved_force_includes.append(raw)
cxx_flags.extend(("-include", raw))
elif tok.startswith("-D"):
# ``.strip()`` handles tokens like ``-D CONFIGURED=1`` (a single
# quoted arg with a space after -D) that some flags arrive as.
@@ -239,14 +192,6 @@ def parse_entry(
pass # input/output files
else:
cxx_flags.append(tok)
for raw in unresolved_force_includes:
# A deleted build artifact would otherwise surface only downstream
if not any((Path(inc) / raw).is_file() for inc in includes):
_LOGGER.warning(
"-include %s found neither next to the compile nor on the "
"include path; cached idedata may not resolve it",
raw,
)
return cxx_path, defines, includes, cxx_flags
-178
View File
@@ -1,178 +0,0 @@
"""Shared precompiled-header policy for the build backends.
The prefix either mirrors the TUs' own force-includes (ESP8266) or is a
curated core-header set (ESP-IDF). ``esphome: includes:`` sources receive
it too; Arduino.h visibility there is intended (esphome#8693).
"""
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
import hashlib
import logging
import os
from pathlib import Path
import posixpath
import re
import stat
from esphome.build_helpers.ccache import parse_enable_env
_DOMAIN = "pch"
@dataclass
class _PCHData:
emitted: bool = False
def _pch_data() -> _PCHData:
from esphome.core import CORE
if _DOMAIN not in CORE.data:
CORE.data[_DOMAIN] = _PCHData()
return CORE.data[_DOMAIN]
def mark_pch_emitted() -> None:
"""Record that this build's consumers reference the pch."""
_pch_data().emitted = True
_LOGGER = logging.getLogger(__name__)
# The header and its .gch/.sum sidecars live in the build directory.
PCH_HEADER_NAME = "esphome_pch.h"
# Every artifact the pch machinery can leave behind, for cleanup.
PCH_ARTIFACT_NAMES = (
PCH_HEADER_NAME,
f"{PCH_HEADER_NAME}.gch",
f"{PCH_HEADER_NAME}.gch.sum",
f"{PCH_HEADER_NAME}.gch.failed",
)
# The core defines header every backend anchors its prefix on.
PCH_CORE_HEADER = "esphome/core/defines.h"
# ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the
# .sum sidecar instead of the .gch bytes, which are not reproducible.
# Keep in sync with the literals in platformio/pch.py.script.
_CCACHE_PCH_ENV = {
"CCACHE_SLOPPINESS": "pch_defines,time_macros",
"CCACHE_PCH_EXTSUM": "true",
}
# Both include forms: an angle include resolving under src/ must enter the
# digest too; ones that do not resolve simply end the walk
_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE)
def pch_enabled() -> bool:
"""Precompiled-header knob: default on, ``ESPHOME_PCH_ENABLE=0`` opts out."""
return parse_enable_env("ESPHOME_PCH_ENABLE") is not False
def ccache_pch_env() -> dict[str, str]:
"""Settings ccache needs to cache compiles that consume the .gch;
empty unless this build actually emitted one. User-set values win.
Native backends export these process-wide; only time_macros affects
non-pch TUs."""
if not (pch_enabled() and _pch_data().emitted):
return {}
extsum = os.environ.get("CCACHE_PCH_EXTSUM")
if extsum is not None and extsum.strip().lower() not in ("1", "true", "yes", "on"):
# ccache then hashes the non-reproducible .gch bytes: permanent misses
_LOGGER.warning("CCACHE_PCH_EXTSUM=%s disables pch caching", extsum)
env = {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ}
user_sloppiness = os.environ.get("CCACHE_SLOPPINESS")
if user_sloppiness is not None and (
missing := [
t
for t in ("pch_defines", "time_macros")
if t not in {tok.strip() for tok in user_sloppiness.split(",")}
]
):
# Without these ccache declines every pch-consuming compile
env["CCACHE_SLOPPINESS"] = ",".join((user_sloppiness, *missing))
_LOGGER.warning(
"Adding %s to CCACHE_SLOPPINESS so ccache can cache compiles "
"that use the precompiled header",
",".join(missing),
)
return env
def pch_extra_scripts() -> list[str]:
"""The extra_scripts entries a PlatformIO platform registers for the
pch; empty when disabled (the script itself has no enable check)."""
return ["post:pch.py"] if pch_enabled() else []
def pch_header_text(include_headers: Iterable[str]) -> str:
"""The prefix-header source: exactly these includes, in order."""
return "".join(f'#include "{name}"\n' for name in include_headers)
def _resolves(path: Path) -> bool:
"""False when missing; other stat failures propagate (identity unknown,
unlike is_file(), which would silently drop the header)."""
try:
return stat.S_ISREG(path.stat().st_mode)
except (FileNotFoundError, NotADirectoryError):
return False
def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
"""Include closure of ``roots``: src-relative name -> contents.
Resolution mirrors the compiler (includer's dir, then src root); names
outside ``src_dir`` end the walk and are versioned by the caller. No
#ifdef evaluation: over-approximating is the safe direction.
"""
seen: dict[str, bytes] = {}
stack: list[tuple[str, str]] = [(name, "") for name in roots]
while stack:
name, from_dir = stack.pop()
for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
rel = posixpath.normpath(candidate)
if not rel.startswith("..") and _resolves(src_dir / rel):
break
else:
continue
if rel in seen:
continue
try:
data = (src_dir / rel).read_bytes()
except OSError as err:
# A marker would truncate the transitive walk; fail closed
_LOGGER.warning("Could not read %s for the pch checksum: %s", rel, err)
raise
seen[rel] = data
parent = posixpath.dirname(rel)
stack.extend(
# surrogateescape: a non-UTF-8 name just fails to resolve
(inc.decode(errors="surrogateescape"), parent)
for inc in _INCLUDE_RE.findall(data)
)
return seen
def pch_checksum(
src_dir: Path, include_headers: Iterable[str], extra: Iterable[str]
) -> str:
"""Digest standing in for the .gch in ccache's hash: the include closure
of the prefix header plus caller-supplied identity strings (versioned
install paths, flags). Raises OSError when a header's identity cannot
be established at all; callers must then compile without a pch."""
digest = hashlib.sha256()
closure = _include_closure(src_dir, include_headers)
for name in sorted(closure):
digest.update(name.encode(errors="surrogateescape"))
digest.update(closure[name])
digest.update(b"\0")
for item in extra:
digest.update(item.encode(errors="surrogateescape"))
digest.update(b"\0")
return digest.hexdigest()
+41 -191
View File
@@ -3,10 +3,8 @@ from pathlib import Path
import platform
import re
import subprocess
import time
from typing import Any
from esphome.build_helpers.pch import pch_extra_scripts
import esphome.codegen as cg
import esphome.config_validation as cv
from esphome.const import (
@@ -16,7 +14,6 @@ from esphome.const import (
CONF_FRAMEWORK,
CONF_PLATFORM_VERSION,
CONF_SOURCE,
CONF_TOOLCHAIN,
CONF_VERSION,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
@@ -24,7 +21,6 @@ from esphome.const import (
KEY_TARGET_PLATFORM,
PLATFORM_ESP8266,
ThreadModel,
Toolchain,
)
from esphome.core import (
CORE,
@@ -35,13 +31,12 @@ from esphome.core import (
)
from esphome.core.config import BOARD_MAX_LENGTH
from esphome.helpers import IS_MACOS, copy_file_if_changed
from esphome.platformio.toolchain import copy_ccache_script, copy_pch_script
from esphome.platformio.toolchain import copy_ccache_script
from esphome.storage_json import StorageJSON
from esphome.types import ConfigType
from .boards import BOARDS, ESP8266_BOARD_BUILD, board_ld_script
from .boards import BOARDS, board_ld_script
from .const import (
BUILD_FLASH_MODES,
CONF_EARLY_PIN_INIT,
CONF_ENABLE_SERIAL,
CONF_ENABLE_SERIAL1,
@@ -49,7 +44,6 @@ from .const import (
KEY_BOARD,
KEY_ESP8266,
KEY_PIN_INITIAL_STATES,
KEY_SCANF_FLOAT,
KEY_SERIAL1_REQUIRED,
KEY_SERIAL_REQUIRED,
KEY_WAVEFORM_REQUIRED,
@@ -109,53 +103,6 @@ def set_core_data(config: ConfigType) -> ConfigType:
return config
_TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ARDUINO)
_validate_toolchain = cv.toolchain_enum(_TOOLCHAINS)
_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.PLATFORMIO)
def _validate_native_toolchain(config: ConfigType) -> ConfigType:
"""Constraints of the native (non-PlatformIO) Arduino toolchain."""
if not CORE.using_toolchain_arduino:
return config
from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION
conf = config[CONF_FRAMEWORK]
version = cv.Version.parse(conf[CONF_VERSION])
if version < MIN_FRAMEWORK_VERSION:
raise cv.Invalid(
"'toolchain: arduino' requires framework version "
f"{MIN_FRAMEWORK_VERSION} or newer"
)
# platform_version is a PlatformIO concept; drop it (as esp32's native
# toolchain does), warning when a custom pin is discarded. The floor
# above guarantees the schema-derived default is the ARDUINO_4 spec.
if (
conf.pop(CONF_PLATFORM_VERSION, _ARDUINO_4_PLATFORM_SPEC)
!= _ARDUINO_4_PLATFORM_SPEC
):
_LOGGER.warning(
"'platform_version' is ignored by 'toolchain: arduino'; the native "
"toolchain downloads the framework and compiler directly"
)
if conf[CONF_SOURCE] != _format_framework_arduino_version(version):
raise cv.Invalid(
"'toolchain: arduino' does not support a custom framework source; "
"use 'toolchain: platformio'"
)
# BOARDS is a subset of ESP8266_BOARD_BUILD today; the second clause is
# a drift guard for the independently regenerated tables
if (
config[CONF_BOARD] not in BOARDS
or config[CONF_BOARD] not in ESP8266_BOARD_BUILD
):
raise cv.Invalid(
f"Board '{config[CONF_BOARD]}' is not supported by "
"'toolchain: arduino'; use 'toolchain: platformio'"
)
return config
def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]:
"""Binary-download entries for a built ESP8266 firmware.
@@ -185,7 +132,7 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
# a PIO platformio/framework-arduinoespressif8266 value
# List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266
# Same encoding the native toolchain uses for its package download, so a
# custom-source check against this value cannot drift from what it fetches.
# version bump cannot drift between the two paths.
from esphome.arduino8266.framework import framework_package_version
try:
@@ -247,7 +194,7 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType:
platform_version = value.get(CONF_PLATFORM_VERSION)
if platform_version is None:
if version >= cv.Version(3, 1, 0):
platform_version = _ARDUINO_4_PLATFORM_SPEC
platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION))
else:
platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION))
value[CONF_PLATFORM_VERSION] = platform_version
@@ -270,10 +217,6 @@ def _parse_platform_version(value: Any) -> str:
return value
# The platform_version derived for every core >= 3.1.0 config
_ARDUINO_4_PLATFORM_SPEC = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION))
ARDUINO_FRAMEWORK_SCHEMA = cv.All(
cv.Schema(
{
@@ -290,6 +233,7 @@ ARDUINO_FRAMEWORK_SCHEMA = cv.All(
)
BUILD_FLASH_MODES = ["qio", "qout", "dio", "dout"]
CONFIG_SCHEMA = cv.All(
cv.Schema(
{
@@ -306,30 +250,15 @@ CONFIG_SCHEMA = cv.All(
cv.Optional(CONF_ENABLE_SERIAL1): cv.boolean,
cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean,
cv.Optional(CONF_ENABLE_SCANF_FLOAT): cv.boolean,
cv.Optional(
CONF_TOOLCHAIN, visibility=cv.Visibility.ADVANCED
): _validate_toolchain,
}
),
_resolve_toolchain,
_validate_native_toolchain,
# Until the native toolchain lands, PlatformIO is the only backend;
# reject a --toolchain this platform cannot serve yet.
cv.require_platformio_toolchain("ESP8266"),
set_core_data,
)
def native_toolchain_module():
"""The native build backend for the resolved toolchain, if any.
``__main__`` dispatches from its own toolchain-keyed table; this helper
serves the component's internal callers.
"""
if not CORE.using_toolchain_arduino:
return None
from esphome.arduino8266 import toolchain
return toolchain
def check_rosetta() -> None:
"""Fail fast when the x86_64 ESP8266 toolchain cannot run on this Mac.
@@ -365,13 +294,12 @@ def _choose_ld_script(board: str) -> str:
@coroutine_with_priority(CoroPriority.PLATFORM)
async def to_code(config: ConfigType) -> None:
use_platformio = CORE.using_toolchain_platformio
cg.add(esp8266_ns.setup_preferences())
if use_platformio:
cg.add_platformio_option("lib_ldf_mode", "off")
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("board", config[CONF_BOARD])
cg.add_platformio_option("lib_ldf_mode", "off")
cg.add_platformio_option("lib_compat_mode", "strict")
cg.add_platformio_option("board", config[CONF_BOARD])
cg.add_build_flag("-DUSE_ESP8266")
cg.set_cpp_standard("gnu++20")
cg.add_define("ESPHOME_BOARD", config[CONF_BOARD])
@@ -387,33 +315,28 @@ async def to_code(config: ConfigType) -> None:
"enabling scanf float support (~8KB flash)"
)
# The native generator reads the same decision (KEY_SCANF_FLOAT)
CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = bool(enable_scanf_float)
if use_platformio:
extra_scripts = [
"pre:ccache.py",
"pre:testing_mode.py",
"pre:exclude_updater.py",
"pre:exclude_waveform.py",
"pre:relocate_ratetable.py",
]
if not enable_scanf_float:
extra_scripts.append("pre:remove_float_scanf.py")
extra_scripts.extend(pch_extra_scripts())
extra_scripts.append("post:post_build.py")
cg.add_platformio_option("extra_scripts", extra_scripts)
extra_scripts = [
"pre:ccache.py",
"pre:testing_mode.py",
"pre:exclude_updater.py",
"pre:exclude_waveform.py",
"pre:relocate_ratetable.py",
]
if not enable_scanf_float:
extra_scripts.append("pre:remove_float_scanf.py")
extra_scripts.append("post:post_build.py")
cg.add_platformio_option("extra_scripts", extra_scripts)
conf = config[CONF_FRAMEWORK]
cg.add_platformio_option("framework", "arduino")
cg.add_build_flag("-DUSE_ARDUINO")
cg.add_build_flag("-DUSE_ESP8266_FRAMEWORK_ARDUINO")
cg.add_build_flag("-Wno-nonnull-compare")
if use_platformio:
cg.add_platformio_option("framework", "arduino")
cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
cg.add_platformio_option(
"platform_packages",
[f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"],
)
cg.add_platformio_option("platform", conf[CONF_PLATFORM_VERSION])
cg.add_platformio_option(
"platform_packages",
[f"platformio/framework-arduinoespressif8266@{conf[CONF_SOURCE]}"],
)
# Default for platformio is LWIP2_LOW_MEMORY with:
# - MSS=536
@@ -451,8 +374,7 @@ async def to_code(config: ConfigType) -> None:
# Force-include inline std::__throw_* overrides so GCC dead-strips the unused
# libstdc++ error message strings (e.g. "basic_string::_M_create") from DRAM.
# See throw_stubs.h for details. Must be prepended before <string>, so this
# uses build_src_flags with -include. Unconditional: the native build
# generator reads the same option, keeping one source of truth.
# uses build_src_flags with -include.
cg.add_platformio_option(
"build_src_flags", "-include esphome/components/esp8266/throw_stubs.h"
)
@@ -482,8 +404,6 @@ async def to_code(config: ConfigType) -> None:
# implementation in the Arduino ESP8266 core.
cg.add_build_flag("-Wl,--wrap=millis")
# Unconditional: the native build generator reads the same option,
# keeping one source of truth
cg.add_platformio_option("board_build.flash_mode", config[CONF_BOARD_FLASH_MODE])
ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
@@ -492,7 +412,7 @@ async def to_code(config: ConfigType) -> None:
cg.RawExpression(f"VERSION_CODE({ver.major}, {ver.minor}, {ver.patch})"),
)
if use_platformio and config[CONF_BOARD] in BOARDS:
if config[CONF_BOARD] in BOARDS:
cg.add_platformio_option(
"board_build.ldscript", _choose_ld_script(config[CONF_BOARD])
)
@@ -533,24 +453,8 @@ async def finalize_serial_config() -> None:
cg.add_build_flag("-DNO_GLOBAL_SERIAL1")
# Called by __main__.compile_program; returning False falls through to the
# PlatformIO toolchain.
def run_compile(args, config: ConfigType) -> bool:
# Positive check: the native backend only runs when explicitly resolved
toolchain = native_toolchain_module()
if toolchain is None:
return False
if toolchain.run_compile(config, CORE.verbose) != 0:
raise EsphomeError("ESP8266 native build failed")
return True
# Called by writer.py
def copy_files() -> None:
# Native builds skip the PlatformIO extra scripts; the build generator
# carries their logic
if CORE.using_toolchain_arduino:
return
dir = Path(__file__).parent
for script in (
"post_build",
@@ -565,7 +469,6 @@ def copy_files() -> None:
CORE.relative_build_path(f"{script}.py"),
)
copy_ccache_script()
copy_pch_script()
# ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder
@@ -608,75 +511,22 @@ ESP8266_EXCEPTION_CODES = {
}
_DECODE_WARNED_AT: dict[str, float] = {}
def _decode_pc(config: ConfigType, addr: str) -> None:
from esphome.platformio import toolchain
def _warn_decode_problem(key: str, message: str, *args) -> bool:
"""Warn, deduplicated briefly so a burst of stack-dump addresses warns
once but a later dump warns again; returns whether it warned so the
caller can mark suppressed addresses individually."""
now = time.monotonic()
last = _DECODE_WARNED_AT.get(key)
if last is not None and now - last < 30:
return False
_DECODE_WARNED_AT[key] = now
_LOGGER.warning(message, *args)
return True
def _decode_pc(config: ConfigType, addr: str, *, bulk: bool = False) -> None:
"""Decode one crash address. ``bulk``: the caller is scanning every
8-hex stack word, most of which are not code addresses -- unmappable
ones log at debug so real frames are not buried."""
if (native_toolchain := native_toolchain_module()) is not None:
addr2line = native_toolchain.get_addr2line_path()
elf = native_toolchain.get_elf_path()
for path in (addr2line, elf):
if not path.is_file():
_warn_decode_problem(
str(path), "Cannot decode crash addresses: %s missing", path
)
# The detailed warning names no address; mark named
# registers, but bulk stack words at debug (~150 per dump)
log = _LOGGER.debug if bulk else _LOGGER.warning
log("Not decoded %s (toolchain file missing)", addr)
return
addr2line, elf = str(addr2line), str(elf)
else:
from esphome.platformio import toolchain
idedata = toolchain.get_idedata(config)
if not idedata.addr2line_path or not idedata.firmware_elf_path:
_warn_decode_problem(
"no-addr2line",
"Cannot decode crash addresses: no addr2line or ELF in idedata",
)
log = _LOGGER.debug if bulk else _LOGGER.warning
log("Not decoded %s (no addr2line or ELF)", addr)
return
addr2line, elf = idedata.addr2line_path, idedata.firmware_elf_path
command = [addr2line, "-pfiaC", "-e", elf, addr]
idedata = toolchain.get_idedata(config)
if not idedata.addr2line_path or not idedata.firmware_elf_path:
_LOGGER.debug("decode_pc no addr2line")
return
command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr]
try:
translation = subprocess.check_output(command, close_fds=False).decode().strip()
except Exception as err: # noqa: BLE001 # pylint: disable=broad-except
# Warn, not debug: a failing addr2line must be visible. The warning
# is rate-limited across a dump, so mark every undecoded address
# inline or the rest read as merely unmappable
if not _warn_decode_problem(
"addr2line-failed", "Could not decode crash address %s (%s)", addr, err
):
# The detailed warning already named this address; mark only
# the rate-limited ones, and bulk stack words at debug
log = _LOGGER.debug if bulk else _LOGGER.warning
log("Not decoded %s (addr2line failed)", addr)
except Exception: # noqa: BLE001 # pylint: disable=broad-except
_LOGGER.debug("Caught exception for command %s", command, exc_info=1)
return
if "?? ??:0" in translation:
# A named register that fails to decode is confusing silence; a
# bulk stack word failing is the expected common case
log = _LOGGER.debug if bulk else _LOGGER.warning
log("Not decoded %s (address not in %s)", addr, elf)
# Nothing useful
return
translation = translation.replace(" at ??:?", "").replace(":?", "")
_LOGGER.warning("Decoded %s", translation)
@@ -746,6 +596,6 @@ def process_stacktrace(config: ConfigType, line: str, backtrace_state: bool) ->
if backtrace_state:
for addr in re.finditer(STACKTRACE_ESP8266_BACKTRACE_PC_RE, line):
_decode_pc(config, addr.group(), bulk=True)
_decode_pc(config, addr.group())
return backtrace_state
+1 -3
View File
@@ -16,6 +16,7 @@ KEY_WAVEFORM_REQUIRED = "waveform_required"
KEY_SERIAL_REQUIRED = "serial_required"
KEY_SERIAL1_REQUIRED = "serial1_required"
# Set for the native (non-PlatformIO) toolchain's build generator
KEY_FLASH_MODE = "flash_mode"
KEY_SCANF_FLOAT = "scanf_float"
# Per-board flash-layout override consumed by board_ld_script()
KEY_LDSCRIPT = "ldscript"
@@ -72,6 +73,3 @@ def enable_serial1() -> None:
enable_serial1()
"""
CORE.data.setdefault(KEY_ESP8266, {})[KEY_SERIAL1_REQUIRED] = True
BUILD_FLASH_MODES = ("qio", "qout", "dio", "dout")
@@ -446,9 +446,9 @@ void MicroWakeWord::loop() {
xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING);
}
if ((event_group_bits & EventGroupBits::TASK_STOPPED)) {
// Retries on a subsequent loop if the task is still running on the other core
if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) {
ESP_LOGD(TAG, "Inference task is finished, freeing task resources");
this->inference_task_.deallocate();
xEventGroupClearBits(this->event_group_, ALL_BITS);
xQueueReset(this->detection_queue_);
this->set_state_(State::STOPPED);
@@ -382,8 +382,8 @@ void MixerSpeaker::loop() {
ESP_LOGV(TAG, "Stopping");
xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING);
}
if (event_group_bits & MIXER_TASK_STATE_STOPPED) {
this->task_.deallocate();
// Retries on a subsequent loop if the task is still running on the other core
if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) {
ESP_LOGD(TAG, "Stopped");
xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS);
this->all_stopped_since_ms_ = 0;
@@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() {
ESP_LOGV(TAG, "Stopping");
xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING);
}
if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) {
this->task_.deallocate();
// Retries on a subsequent loop if the task is still running on the other core
if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) {
ESP_LOGD(TAG, "Stopped");
xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS);
}
@@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() {
if (!this->is_playing_) {
// The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks
if (this->read_task_.is_created() || this->decode_task_.is_created()) {
this->read_task_.deallocate();
this->decode_task_.deallocate();
// Both are attempted every time; a task that is still running on the other core is freed by a
// subsequent call, and freeing an already freed task succeeds without doing anything
bool read_task_freed = this->read_task_.deallocate();
bool decode_task_freed = this->decode_task_.deallocate();
if (!read_task_freed || !decode_task_freed) {
// A task is still running on the other core, so keep the pipeline in its current state and try
// again on the next call
return AudioPipelineState::PLAYING;
}
if (this->hard_stop_) {
// Stop command was sent, so immediately end the playback
this->speaker_->stop();
+1 -8
View File
@@ -555,14 +555,7 @@ NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscri
# that is stored rather than translated away. Consumed by the esp8266 native
# backend (later in this chain) for its ignored-option warning; defined here
# so it stays adjacent to the routing.
# build_src_flags and board_build.flash_mode: set unconditionally by
# esp8266/__init__ and read by the native generator; not user-routable, so
# not in the set above
NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | {
"lib_ignore",
"build_src_flags",
"board_build.flash_mode",
}
NATIVE_ARDUINO_CONSUMED_PIO_OPTIONS = NATIVE_ARDUINO_PIO_OPTIONS | {"lib_ignore"}
@coroutine_with_priority(CoroPriority.FINAL)
+23 -7
View File
@@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size
return true;
}
void StaticTask::destroy() {
if (this->handle_ != nullptr) {
TaskHandle_t handle = this->handle_;
this->handle_ = nullptr;
vTaskDelete(handle);
bool StaticTask::destroy() {
if (this->handle_ == nullptr) {
return true;
}
// Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks
// the other core to yield though, so the task may still be running on it for a moment.
vTaskSuspend(this->handle_);
if (eTaskGetState(this->handle_) != eSuspended) {
// The task is still running on the other core and using its stack. Deleting it now would only put it on
// the termination list and return, so the caller has to try again once it has been swapped out.
return false;
}
// The task cannot run again, so the delete completes right away instead of being left to the idle task.
TaskHandle_t handle = this->handle_;
this->handle_ = nullptr;
vTaskDelete(handle);
return true;
}
void StaticTask::deallocate() {
this->destroy();
bool StaticTask::deallocate() {
if (!this->destroy()) {
return false;
}
if (this->stack_buffer_ != nullptr) {
RAMAllocator<StackType_t> allocator(this->use_psram_ ? RAMAllocator<StackType_t>::ALLOC_EXTERNAL
: RAMAllocator<StackType_t>::ALLOC_INTERNAL);
@@ -57,6 +72,7 @@ void StaticTask::deallocate() {
this->stack_buffer_ = nullptr;
this->stack_size_ = 0;
}
return true;
}
} // namespace esphome
+12 -5
View File
@@ -11,6 +11,7 @@ namespace esphome {
/** Helper for FreeRTOS static task management.
* Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods.
* Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on.
*/
class StaticTask {
public:
@@ -23,7 +24,7 @@ class StaticTask {
/// @brief Allocate stack and create task.
/// @param fn Task function
/// @param name Task name (for debug)
/// @param stack_size Stack size in StackType_t words
/// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF)
/// @param param Parameter passed to task function
/// @param priority FreeRTOS task priority
/// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM
@@ -31,11 +32,17 @@ class StaticTask {
bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority,
bool use_psram);
/// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call.
void destroy();
/// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call.
/// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is
/// suspended here so that it cannot be scheduled again, and it is given no chance to clean up.
/// @return true if the task was deleted; false if it is still running on another core, in which case the
/// caller should try again later.
bool destroy();
/// @brief Delete the task (if running) and free the stack buffer.
void deallocate();
/// @brief Delete the task (if created) and free the stack buffer.
/// @return true if the stack buffer was freed; false if the task is still running on another core, in
/// which case the caller should try again later.
bool deallocate();
protected:
TaskHandle_t handle_{nullptr};
-364
View File
@@ -1,364 +0,0 @@
import hashlib
import os
from pathlib import Path
import posixpath
import re
import shlex
import stat
import subprocess
import traceback
# pylint: disable=E0602
Import("env") # noqa: F821
_projenv_error = None
try:
Import("projenv") # noqa: F821
except Exception as err: # noqa: BLE001 -- not exported under -t nobuild
projenv = None
_projenv_error = err
# Precompile the src force-includes plus defines.h and force-include the
# result into C++ src compiles only; their preprocessed output is unchanged.
# Registration is gated host-side (pch_enabled()). Keep the closure, ccache
# values, and stamp flow in sync with build_helpers/pch.py.
# Compiler failures that clear on their own must not latch the .failed marker
_TRANSIENT_ERRORS = ("No space left", "Cannot allocate", "Resource temporarily")
_INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE)
_CORE_HEADER = "esphome/core/defines.h"
def _raise_walk_error(err: OSError) -> None:
raise err
def _resolves_dir(path: Path) -> bool:
"""False when missing; other stat failures propagate."""
try:
return stat.S_ISDIR(path.stat().st_mode)
except (FileNotFoundError, NotADirectoryError):
return False
def _resolves(path: Path) -> bool:
"""False when missing; other stat failures propagate (identity unknown)."""
try:
return stat.S_ISREG(path.stat().st_mode)
except (FileNotFoundError, NotADirectoryError):
return False
def _include_closure(src_dir: Path, roots: list) -> dict:
"""Quoted-include closure: src-relative name -> contents (mirror of
build_helpers/pch.py)."""
seen = {}
stack = [(name, "") for name in roots]
while stack:
name, from_dir = stack.pop()
for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
rel = posixpath.normpath(candidate)
if not rel.startswith("..") and _resolves(src_dir / rel):
break
else:
continue
if rel in seen:
continue
try:
data = (src_dir / rel).read_bytes()
except OSError as err:
# A marker would truncate the transitive walk; fail closed
print(f"ESPHome: could not read {rel} for the pch checksum: {err}")
raise
seen[rel] = data
parent = posixpath.dirname(rel)
stack.extend(
(inc.decode(errors="surrogateescape"), parent)
for inc in _INCLUDE_RE.findall(data)
)
return seen
def _shell_arg(element) -> str | None:
"""One compiler argv from one SCons element, matching the real spawn:
spaced elements pass whole, the rest get one shell unquote (skipped on
Windows, where shlex would eat path backslashes)."""
arg = str(element)
if " " in arg or os.name == "nt":
return arg.replace('\\"', '"')
if not arg.strip():
return arg
try:
tokens = shlex.split(arg)
except ValueError as err:
print(f"ESPHome: could not lex flag {arg!r} for the pch: {err}")
return None
if len(tokens) != 1:
# A flag the model cannot reproduce would diverge the .gch's flags
print(f"ESPHome: cannot model flag {arg!r} for the pch")
return None
return tokens[0]
def _compile_gch(cxx, flags, header: Path, gch: Path, proj_dir: Path):
"""Compile the .gch, then probe that the toolchain can load it back
(GCC 10 on macOS arm64 rejects its own per-process). Returns an error
string or None; OSError propagates as transient."""
result = subprocess.run( # noqa: PLW1510
[cxx, "-x", "c++-header", *flags, "-c", str(header), "-o", str(gch)],
cwd=proj_dir,
# C locale keeps diagnostics matchable by _TRANSIENT_ERRORS
env={**os.environ, "LC_ALL": "C"},
capture_output=True,
text=True,
)
if result.returncode < 0:
# Signal-killed (OOM, ^C): route to the transient no-marker path
raise OSError(f"compiler killed by signal {-result.returncode}")
if result.returncode != 0:
return result.stderr
# -MF is only legal alongside a dependency flag; pass it solely to
# redirect a depfile that -MD/-MMD in the flags would otherwise write
dep_redirect = (
["-MF", os.devnull]
if any(f in ("-MD", "-MMD", "-M", "-MM") for f in flags)
else []
)
probe = subprocess.run( # noqa: PLW1510
[
cxx,
*flags,
*dep_redirect,
"-Winvalid-pch",
"-include",
str(header),
"-fsyntax-only",
"-x",
"c++",
"-",
],
cwd=proj_dir,
env={**os.environ, "LC_ALL": "C"},
input="",
capture_output=True,
text=True,
)
if probe.returncode < 0:
raise OSError(f"probe killed by signal {-probe.returncode}")
if probe.returncode != 0 or ".gch" in probe.stderr:
return f"toolchain cannot load the pch: {probe.stderr.strip()}"
return None
def _read_stamp(path: Path) -> str:
"""A corrupt sidecar must read as stale, not kill the pch forever."""
try:
return path.read_text(encoding="utf-8").strip()
except (OSError, UnicodeDecodeError):
return ""
def _setup_pch() -> None:
if projenv is None:
# Expected under -t nobuild; anything else must leave a trail
print(f"ESPHome: projenv unavailable ({_projenv_error}); skipping pch")
return
# Project root: SCons compiles run here, so the relative -include
# resolves; an absolute path would break cross-device ccache sharing.
proj_dir = Path(env.subst("$PROJECT_DIR")) # noqa: F821
src_dir = Path(env.subst("$PROJECT_SRC_DIR")) # noqa: F821
header = proj_dir / "esphome_pch.h"
gch = Path(f"{header}.gch")
sum_path = Path(f"{gch}.sum")
cxx = projenv.subst("$CXX") # noqa: F821
# The header holds the -include entries itself, so the .gch compile must
# not see them; consumers keep theirs, which the .gch then satisfies.
flags = []
include_headers = []
raw_args = [
_shell_arg(element)
for element in projenv.subst_list("$CXXFLAGS $CCFLAGS $_CCCOMCOM")[0] # noqa: F821
]
if any(arg is None for arg in raw_args):
print("ESPHome: skipping precompiled header: unmodelable flag")
return
flag_it = iter(raw_args)
for tok in flag_it:
if tok == "-include":
include_headers.append(next(flag_it, ""))
elif tok.startswith("-include") and not tok.startswith("-include-"):
include_headers.append(tok[len("-include") :])
else:
flags.append(tok)
if any(not name for name in include_headers):
print("ESPHome: build_src_flags has a trailing -include; skipping pch")
return
# Fold only relative names resolving under src/: consumers keep their
# own -include entries, so folding an unguarded user header would
# include it twice; unfolded ones stay consumer-only.
try:
folded = [
name
for name in include_headers
if not Path(name).is_absolute() and _resolves(src_dir / name)
]
except OSError as err:
print(f"ESPHome: skipping precompiled header: {err}")
return
if unfolded := [n for n in include_headers if n not in folded]:
print(f"ESPHome: not precompiling non-src force-includes: {unfolded}")
content = "".join(f'#include "{name}"\n' for name in (*folded, _CORE_HEADER))
digest = hashlib.sha256()
digest.update(content.encode(errors="surrogateescape"))
digest.update(cxx.encode(errors="surrogateescape"))
# Mirror CCACHE_BASEDIR: strip the per-device build path so identical
# configs produce identical .sum files and share cache entries
flags_id = " ".join(flags)
if basedir := os.environ.get("CCACHE_BASEDIR"):
flags_id = flags_id.replace(basedir, "")
digest.update(flags_id.encode(errors="surrogateescape"))
# GCC never validates a .gch against its source headers, and PlatformIO
# package paths carry no version, so a package bump must invalidate here
platform = env.PioPlatform() # noqa: F821
for package in sorted(platform.packages):
try:
version = platform.get_package_version(package)
except KeyError:
# An unresolved manifest must not hash as a constant
if platform.get_package(package) is not None:
print(f"ESPHome: skipping precompiled header: no version for {package}")
return
version = None # absent optional package
except Exception as err: # noqa: BLE001
# No trustworthy package identity: a stale .gch could survive
print(f"ESPHome: skipping precompiled header: {err}")
return
digest.update(f"{package}={version}".encode())
digest.update(b"\0")
try:
closure = _include_closure(src_dir, [*folded, _CORE_HEADER])
except OSError as err:
print(f"ESPHome: skipping precompiled header: {err}")
return
for rel in sorted(closure):
digest.update(rel.encode(errors="surrogateescape"))
digest.update(closure[rel])
digest.update(b"\0")
# Project-local -I dirs (e.g. rp2's lwip_override) hold generated
# headers the src closure cannot see; hash them too
prev = ""
try:
for tok in flags:
inc = tok[2:] if tok.startswith("-I") and len(tok) > 2 else ""
if prev == "-I":
inc = tok
prev = tok
if not inc:
continue
inc_dir = Path(inc)
if not (
_resolves_dir(inc_dir)
and inc_dir.is_relative_to(proj_dir)
and not inc_dir.is_relative_to(src_dir)
# Library trees never enter the prefix closure; walking them
# would read every library file each build
and not inc_dir.is_relative_to(proj_dir / ".piolibdeps")
and not inc_dir.is_relative_to(proj_dir / ".pioenvs")
):
continue
local_headers = []
# os.walk with onerror: rglob would swallow unlistable subtrees
for root, _dirs, files in os.walk(inc_dir, onerror=_raise_walk_error):
local_headers.extend(
Path(root) / f
for f in files
if f.endswith((".h", ".hpp", ".hh", ".inc"))
)
for local in sorted(local_headers):
digest.update(str(local.relative_to(proj_dir)).encode())
digest.update(local.read_bytes())
digest.update(b"\0")
except OSError as err:
print(f"ESPHome: skipping precompiled header: {err}")
return
checksum = digest.hexdigest()
# The ccache .sum sidecar doubles as the freshness stamp
if (
not header.is_file()
or not gch.is_file()
or not sum_path.is_file()
or (_read_stamp(sum_path) != checksum)
):
failed_marker = Path(f"{gch}.failed")
if _read_stamp(failed_marker) == checksum:
print(
"ESPHome: skipping precompiled header (previous attempt "
f"failed); delete {failed_marker.name} to retry"
)
return
header.write_text(content, encoding="utf-8")
try:
error = _compile_gch(cxx, flags, header, gch, proj_dir)
except OSError as err:
# Transient spawn/IO failure: no marker, retry next build
print(f"ESPHome: precompiled header compile did not run: {err}")
gch.unlink(missing_ok=True)
sum_path.unlink(missing_ok=True)
return
if error is not None:
print("ESPHome: precompiled header failed; compiling without it")
print(error)
gch.unlink(missing_ok=True)
sum_path.unlink(missing_ok=True)
if any(m in error for m in _TRANSIENT_ERRORS):
# Resource exhaustion clears on its own; retry next build
return
# Skip retries until a flag/header/platform change alters the checksum
failed_marker.write_text(checksum + "\n", encoding="utf-8")
return
failed_marker.unlink(missing_ok=True)
sum_path.write_text(checksum + "\n", encoding="utf-8")
# Computed first so the flags and env land together: a raise between
# them would leave a pch-consuming build without its ccache settings.
# projenv["ENV"] aliases os.environ, so these reach all TUs; only
# time_macros affects non-pch TUs. User values win.
ccache_updates = {
key: value
for key, value in (
("CCACHE_SLOPPINESS", "pch_defines,time_macros"),
("CCACHE_PCH_EXTSUM", "true"),
)
if key not in os.environ
}
sloppiness = os.environ.get("CCACHE_SLOPPINESS")
if sloppiness is not None:
tokens = {tok.strip() for tok in sloppiness.split(",")}
missing = [t for t in ("pch_defines", "time_macros") if t not in tokens]
if missing:
# Without these ccache declines every pch-consuming compile
ccache_updates["CCACHE_SLOPPINESS"] = ",".join((sloppiness, *missing))
print(f"ESPHome: adding {','.join(missing)} to CCACHE_SLOPPINESS for the pch")
extsum = os.environ.get("CCACHE_PCH_EXTSUM")
if extsum is not None and extsum.strip().lower() not in ("1", "true", "yes", "on"):
# ccache then hashes the non-reproducible .gch bytes: permanent misses
print(f"ESPHome: CCACHE_PCH_EXTSUM={extsum} disables pch caching")
# Prepended: GCC only uses a .gch while no other tokens precede it.
# The relative name also reaches "pio run -t idedata" output.
# -Wno-error: the per-process probe can pass while a later cc1plus
# rejects the .gch; that must stay a warning under user -Werror.
projenv.Prepend( # noqa: F821
CXXFLAGS=["-Winvalid-pch", "-Wno-error=invalid-pch", "-include", header.name]
)
projenv["ENV"].update(ccache_updates) # noqa: F821
print("ESPHome: Compiling with precompiled header")
try:
_setup_pch()
except Exception: # noqa: BLE001 -- a speedup must never break the build
print("ESPHome: pch internal error; compiling without it")
traceback.print_exc()
-9
View File
@@ -289,15 +289,6 @@ def copy_ccache_script() -> None:
)
def copy_pch_script() -> None:
"""Copy the shared precompiled-header SCons post-script into the build
dir; platform components pair it with ``post:pch.py`` in extra_scripts."""
copy_file_if_changed(
Path(__file__).parent / "pch.py.script",
CORE.relative_build_path("pch.py"),
)
def default_libdeps_dir() -> str:
"""The PLATFORMIO_LIBDEPS_DIR value a pio run defaults to; the package
prefetch must resolve installed libraries against the same dir."""
+4 -14
View File
@@ -7,7 +7,6 @@ import re
import time
from esphome import loader
from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES
from esphome.compiled_config import save_compiled_config
from esphome.config import iter_component_configs, iter_components
from esphome.const import (
@@ -610,20 +609,11 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False):
if idf_path.is_dir():
_LOGGER.info("Deleting %s", idf_path)
rmtree(idf_path)
# The PlatformIO pch artifacts live at the project root so the
# relative -include resolves; a partial clean must drop them too
for name in PCH_ARTIFACT_NAMES:
CORE.relative_build_path(name).unlink(missing_ok=True)
# The idedata caches are derived from the build but live under the data
# dir, not the build path, so they must be removed separately in both
# modes. Globbed (name.json plus name.<backend>.json) so a future
# backend suffix cannot silently drift out of clean-all.
idedata_dir = CORE.relative_internal_path("idedata")
for idedata_cache in (
*idedata_dir.glob(f"{CORE.name}.json"),
*idedata_dir.glob(f"{CORE.name}.*.json"),
):
# The idedata cache is derived from the build but lives under the data dir,
# not the build path, so it must be removed separately in both modes.
idedata_cache = CORE.relative_internal_path("idedata", f"{CORE.name}.json")
if idedata_cache.is_file():
_LOGGER.info("Deleting %s", idedata_cache)
idedata_cache.unlink()
+1 -1
View File
@@ -10,7 +10,7 @@ tzlocal==5.4.4 # from time
tzdata>=2026.3 # from time
pyserial==3.5
platformio==6.1.19
esptool==5.3.1
esptool==5.4.0
click==8.3.3
aioesphomeapi==46.3.0
aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi
+24 -132
View File
@@ -50,7 +50,6 @@ from __future__ import annotations
import argparse
from collections import Counter
from collections.abc import Callable
from enum import StrEnum
from functools import cache
import json
@@ -532,53 +531,37 @@ ESP32_PLATFORMIO_TRIGGER_PATH_PREFIXES = ("esphome/platformio/",)
# - esphome/build_gen/platformio.py -- the PlatformIO build generator
# - script/test_build_components.py -- the harness the job invokes
# - .github/workflows/ci.yml -- the job's own definition
# Shared by every toolchain smoke-test job: the harness it invokes and the
# workflow that defines it
_SMOKE_HARNESS_TRIGGER_FILES = frozenset(
ESP32_PLATFORMIO_TRIGGER_FILES = frozenset(
{
"esphome/build_gen/platformio.py",
"script/test_build_components.py",
".github/workflows/ci.yml",
}
)
ESP32_PLATFORMIO_TRIGGER_FILES = _SMOKE_HARNESS_TRIGGER_FILES | {
"esphome/build_gen/platformio.py",
}
def _path_or_file_trigger(
files: list[str],
trigger_files: frozenset[str],
trigger_prefixes: tuple[str, ...],
) -> bool:
"""Whether any changed file matches the given infrastructure triggers."""
return any(
file in trigger_files or file.startswith(trigger_prefixes) for file in files
)
@cache
def _cached_components_closure(files: tuple[str, ...]) -> frozenset[str]:
"""Dependency closure of the changed components, from the changed files.
The walk is expensive and every toolchain smoke-test job asks for the
same file list, so compute it once per run."""
component_files = [f for f in files if filter_component_and_test_files(f)]
return frozenset(get_components_with_dependencies(component_files, True))
def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool:
"""Whether any changed file is a PlatformIO infrastructure / harness trigger."""
return _path_or_file_trigger(
files, ESP32_PLATFORMIO_TRIGGER_FILES, ESP32_PLATFORMIO_TRIGGER_PATH_PREFIXES
)
for file in files:
if file in ESP32_PLATFORMIO_TRIGGER_FILES:
return True
if any(
file.startswith(prefix) for prefix in ESP32_PLATFORMIO_TRIGGER_PATH_PREFIXES
):
return True
return False
def _esp_idf_infra_changed(files: list[str]) -> bool:
"""Whether any changed file is ESP-IDF build/runner infrastructure."""
return _path_or_file_trigger(
files, ESP_IDF_INFRA_TRIGGER_FILES, ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES
)
for file in files:
if file in ESP_IDF_INFRA_TRIGGER_FILES:
return True
if any(
file.startswith(prefix) for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES
):
return True
return False
def esp32_platformio_components_to_test(branch: str | None = None) -> list[str]:
@@ -616,23 +599,15 @@ def esp32_platformio_components_to_test(branch: str | None = None) -> list[str]:
Returns:
Sorted list of component names to compile.
"""
return _toolchain_components_to_test(
branch, ESP32_PLATFORMIO_TEST_COMPONENTS, _esp32_platformio_path_or_file_trigger
)
def _toolchain_components_to_test(
branch: str | None,
test_set: frozenset[str],
infra_trigger: Callable[[list[str]], bool],
) -> list[str]:
"""The shared narrowing rule for the per-toolchain smoke-test jobs."""
files = changed_files(branch)
if core_changed(files) or infra_trigger(files):
return sorted(test_set)
if core_changed(files) or _esp32_platformio_path_or_file_trigger(files):
return sorted(ESP32_PLATFORMIO_TEST_COMPONENTS)
return sorted(test_set & _cached_components_closure(tuple(files)))
component_files = [f for f in files if filter_component_and_test_files(f)]
changed = get_components_with_dependencies(component_files, True)
return sorted(ESP32_PLATFORMIO_TEST_COMPONENTS & set(changed))
def should_run_esp32_platformio(branch: str | None = None) -> bool:
@@ -653,83 +628,6 @@ def should_run_esp32_platformio(branch: str | None = None) -> bool:
return bool(esp32_platformio_components_to_test(branch))
# The `--toolchain arduino` smoke-test set: covers the core, the bundled and
# converted registry libraries, and the waveform path.
ESP8266_NATIVE_TEST_COMPONENTS = frozenset(
{
"esp8266",
"api",
"web_server",
"captive_portal",
"mqtt",
"esp8266_pwm",
"neopixelbus",
"bme280_i2c",
"uart",
}
)
# Infrastructure whose changes always trigger the native ESP8266 compile
# test. esphome/build_helpers/ holds the idedata and size-summary helpers
# the backend shares with the native ESP-IDF build.
ESP8266_NATIVE_TRIGGER_PATH_PREFIXES = (
"esphome/arduino8266/",
"esphome/arduino/",
"esphome/build_helpers/",
)
# Shared library-conversion modules every native build imports; espidf-only
# infra (build_gen/espidf.py) deliberately stays out of the esp8266 set.
_NATIVE_SHARED_TRIGGER_FILES = frozenset(
{
"esphome/framework_helpers.py",
"esphome/platformio/library.py",
"esphome/platformio/extra_script.py",
}
)
# Tripwire: the shared modules must stay in the ESP-IDF trigger set too
# (now defined in clang_tidy_hash), or its smoke test silently skips them
assert _NATIVE_SHARED_TRIGGER_FILES <= ESP_IDF_INFRA_TRIGGER_FILES
ESP8266_NATIVE_TRIGGER_FILES = (
_NATIVE_SHARED_TRIGGER_FILES
| _SMOKE_HARNESS_TRIGGER_FILES
| {
"esphome/build_gen/arduino8266.py",
"esphome/build_gen/build_tool.py",
"esphome/components/esp8266/build_surgery.py",
"esphome/components/esp8266/boards.py",
"esphome/platformio/registry.py",
# esp8266/__init__.py imports copy_ccache_script from it
"esphome/platformio/toolchain.py",
".github/actions/cache-arduino8266/action.yml",
}
)
def _esp8266_native_path_or_file_trigger(files: list[str]) -> bool:
"""Whether any changed file is native-ESP8266 infrastructure / harness."""
# base_python_changed covers the top-level esphome/*.py modules the
# native backend imports directly (framework_helpers, helpers, writer,
# __main__); without it a change there would silently skip this job.
# base_python_changed is deliberately broad (any top-level esphome/*.py)
# as belt-and-braces while the backend is new; narrow it to the modules
# the backend imports once the toolchain has soaked a few releases
return base_python_changed(files) or _path_or_file_trigger(
files, ESP8266_NATIVE_TRIGGER_FILES, ESP8266_NATIVE_TRIGGER_PATH_PREFIXES
)
def esp8266_native_components_to_test(branch: str | None = None) -> list[str]:
"""Subset of ``ESP8266_NATIVE_TEST_COMPONENTS`` the job needs to compile.
Same narrowing logic as ``esp32_platformio_components_to_test``: the full
list on core or infrastructure changes, otherwise the intersection with
the changed-component dependency closure (empty list skips the job).
"""
return _toolchain_components_to_test(
branch, ESP8266_NATIVE_TEST_COMPONENTS, _esp8266_native_path_or_file_trigger
)
def determine_cpp_unit_tests(
branch: str | None = None,
) -> tuple[bool, list[str]]:
@@ -1328,8 +1226,6 @@ def main() -> None:
run_device_builder = True
esp32_platformio_components = sorted(ESP32_PLATFORMIO_TEST_COMPONENTS)
run_esp32_platformio = True
esp8266_native_components = sorted(ESP8266_NATIVE_TEST_COMPONENTS)
run_esp8266_native = True
else:
integration_run_all, integration_test_files = determine_integration_tests(
args.branch
@@ -1341,8 +1237,6 @@ def main() -> None:
run_device_builder = should_run_device_builder(args.branch)
esp32_platformio_components = esp32_platformio_components_to_test(args.branch)
run_esp32_platformio = bool(esp32_platformio_components)
esp8266_native_components = esp8266_native_components_to_test(args.branch)
run_esp8266_native = bool(esp8266_native_components)
run_integration, integration_test_buckets = _compute_integration_test_buckets(
integration_run_all, integration_test_files
)
@@ -1538,8 +1432,6 @@ def main() -> None:
"device_builder": run_device_builder,
"esp32_platformio": run_esp32_platformio,
"esp32_platformio_components": ",".join(esp32_platformio_components),
"esp8266_native": run_esp8266_native,
"esp8266_native_components": ",".join(esp8266_native_components),
"changed_components": changed_components,
"changed_components_with_tests": changed_components_with_tests,
"directly_changed_components_with_tests": list(directly_changed_with_tests),
+17
View File
@@ -1104,6 +1104,10 @@ def get_components_per_integration_fixture() -> dict[str, set[str]]:
_TEST_FUNC_RE = re.compile(r"async def (test_\w+)")
# Any usage form (decorator, pytestmark assignment or list element); only
# test_*.py files are scanned, so the marker docs elsewhere cannot false-hit
_SHARED_YAML_USE_RE = re.compile(r"\bmark\.shared_yaml")
_SHARED_YAML_ARG_RE = re.compile(r"\(\s*[\"'](\w+)[\"']\s*\)")
@cache
@@ -1123,6 +1127,19 @@ def get_fixture_to_test_files() -> dict[str, frozenset[str]]:
for func in _TEST_FUNC_RE.findall(content):
base_name = func.replace("test_", "").partition("[")[0]
result.setdefault(base_name, set()).add(rel_path)
# Shared fixtures are named by marker, not by a test function; each
# decorator must carry a string literal or its fixture would silently
# map to no tests
for use in _SHARED_YAML_USE_RE.finditer(content):
arg = _SHARED_YAML_ARG_RE.match(content, use.end())
if arg is None:
line = content.count("\n", 0, use.start()) + 1
raise ValueError(
f"{rel_path}:{line}: shared_yaml marker must take a "
"single-line string literal so CI test selection can map "
"its fixture"
)
result.setdefault(arg.group(1), set()).add(rel_path)
return {k: frozenset(v) for k, v in result.items()}
+8 -50
View File
@@ -1027,7 +1027,6 @@ def test_components(
isolated_components: set[str] | None = None,
base_only: bool = False,
toolchain: str | None = None,
fail_on_no_tests: bool = False,
) -> int:
"""Test components with optional intelligent grouping.
@@ -1062,34 +1061,20 @@ def test_components(
# toolchain build.
include_validate = esphome_command != "compile"
# Find all component tests; remember which components each pattern
# (wildcards included) matched, for the deferred no-tests accounting
# Find all component tests
all_tests = {}
pattern_components: dict[str, set[str]] = {}
for pattern in component_patterns:
# Skip empty patterns (happens when components list is empty string)
if not pattern:
continue
found = find_component_tests(
tests_dir, pattern, base_only, include_validate=include_validate
all_tests.update(
find_component_tests(
tests_dir, pattern, base_only, include_validate=include_validate
)
)
pattern_components[pattern] = set(found)
all_tests.update(found)
# The flag's contract is "no test matched fails": a fully blank pattern
# list would otherwise slide into the reference-baseline fallback and
# exit green while building nothing a caller asked for
if fail_on_no_tests and not any(component_patterns):
print("No components requested (blank component list)")
return 1
if fail_on_no_tests and not all_tests:
# Nothing matched at all: fail before the synthetic baseline build,
# which would spend a compile reporting success on nothing. Partial
# matches defer to the per-pattern accounting after the summary.
print(f"No components found matching: {component_patterns}")
return 1
# If no components found, build a reference configuration for baseline comparison
# Create a synthetic "empty" component test that will build just the base config
if not all_tests:
print(f"No components found matching: {component_patterns}")
print(
@@ -1193,26 +1178,6 @@ def test_components(
toolchain=toolchain,
)
silent: list[str] = []
if fail_on_no_tests:
# A green run that built nothing for a requested pattern (renamed
# fixture, missing base file, version-suffix mismatch, a wildcard
# matching no component) must not pass CI. Per pattern: an
# all-or-nothing check would let one silent pattern hide behind the
# others. Opt-in: some legs (the esp32-ard smoke subset)
# legitimately match nothing. Failing is deferred past the summary
# so a real failure's reproduce commands still print.
built = {c for r in test_results for c in r.components}
# A pattern is silent when it matched no fixture, or when none of
# its matched components produced a build (wildcards included)
silent = [
p
for p in component_patterns
if p and not (pattern_components.get(p, set()) & built)
]
if silent:
print(f"No tests ran for requested pattern(s): {', '.join(silent)}")
# Separate results into passed and failed
passed_results = [r for r in test_results if r.success]
failed_results = [r for r in test_results if not r.success]
@@ -1244,7 +1209,7 @@ def test_components(
if os.environ.get("GITHUB_STEP_SUMMARY"):
write_github_summary(test_results, toolchain=toolchain)
if failed_results or silent:
if failed_results:
return 1
return 0
@@ -1299,12 +1264,6 @@ def main() -> int:
"--toolchain",
help="Select toolchain for compiling.",
)
parser.add_argument(
"--fail-on-no-tests",
action="store_true",
help="Exit non-zero when no test matched (for CI legs whose "
"components must all have fixtures)",
)
args = parser.parse_args()
@@ -1323,7 +1282,6 @@ def main() -> int:
continue_on_fail=args.continue_on_fail,
enable_grouping=not args.no_grouping,
isolated_components=isolated_components,
fail_on_no_tests=args.fail_on_no_tests,
base_only=args.base_only,
toolchain=args.toolchain,
)
+7
View File
@@ -21,6 +21,13 @@ The `yaml_config` fixture automatically loads YAML configurations based on the t
- The fixture file must exist or the test will fail with a clear error message
- The fixture automatically injects a dynamic port number into the API configuration
Tests marked `@pytest.mark.shared_yaml("name")` load `fixtures/name.yaml` instead
of the test-named file and compile it in a shared, hash-keyed build directory, so
the whole group pays one full compile and each test only a relink. The marker
argument must be a single-line string literal (CI test selection maps fixtures to
test files by scanning for it), and marked tests must hand the `yaml_config`
content to `run_compiled` unmodified.
### Key Fixtures
- `run_compiled` - Combines write, compile, and run operations into a single context manager
+335 -81
View File
@@ -4,17 +4,22 @@ from __future__ import annotations
import asyncio
from collections.abc import AsyncGenerator, Callable, Generator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress
import fcntl
from functools import cache
import hashlib
import logging
import os
from pathlib import Path
import platform
import re
import shutil
import signal
import socket
import subprocess
import sys
import tempfile
import time
from typing import TextIO
from aioesphomeapi import APIClient, APIConnectionError, LogParser, ReconnectLogic
@@ -23,7 +28,13 @@ import pytest_asyncio
import esphome.config
from esphome.core import CORE
from esphome.helpers import get_usable_cpu_count
from esphome.helpers import (
get_usable_cpu_count,
read_file,
rmtree,
write_file,
write_file_if_changed,
)
from esphome.platformio.toolchain import get_idedata
from .const import (
@@ -56,6 +67,21 @@ import pty # not available on Windows
pytest.register_assert_rewrite("tests.integration.entity_utils")
def pytest_configure(config: pytest.Config) -> None:
config.addinivalue_line(
"markers",
"shared_yaml(name): load fixtures/<name>.yaml and compile it in a shared, "
"hash-keyed incremental build directory",
)
FIXTURES_DIR = Path(__file__).parent / "fixtures"
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
# CI caches parts of this path; keep in sync with ci.yml integration-tests.
INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests"
def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
"""Get environment variables for PlatformIO with shared cache."""
env = os.environ.copy()
@@ -78,7 +104,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
)
# Compile with THIS tree's esphome sources, not wherever the venv's editable
# install points (which may be a different git worktree or checkout).
repo_root = str(Path(__file__).resolve().parent.parent.parent)
repo_root = str(REPO_ROOT)
existing = env.get("PYTHONPATH")
env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{existing}" if existing else repo_root
return env
@@ -88,8 +114,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]:
def shared_platformio_cache() -> Generator[Path]:
"""Initialize a shared PlatformIO cache for all integration tests."""
# Use a dedicated directory for integration tests to avoid conflicts.
# CI caches parts of this path; keep in sync with ci.yml integration-tests.
test_cache_dir = Path.home() / ".esphome-integration-tests"
test_cache_dir = INTEGRATION_TESTS_ROOT
cache_dir = test_cache_dir / "platformio"
# Use a lock file in the home directory to ensure only one process initializes the cache
@@ -112,7 +137,9 @@ def shared_platformio_cache() -> Generator[Path]:
init_dir = Path(tmpdir)
fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml"
config_path = init_dir / "cache_init.yaml"
config_path.write_text(fixture_path.read_text())
config_path.write_text(
fixture_path.read_text(encoding="utf-8"), encoding="utf-8"
)
# Run compilation to populate the cache
# We must succeed here to avoid race conditions where multiple
@@ -162,13 +189,6 @@ def integration_test_dir() -> Generator[Path]:
yield Path(tmpdir)
@pytest.fixture
def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Host preferences persist per device name; give the test its own so a
provisioned key never leaks into another run."""
monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs"))
@pytest.fixture
def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]:
"""Reserve an unused TCP port by holding the socket open."""
@@ -188,21 +208,29 @@ def unused_tcp_port(reserved_tcp_port: tuple[int, socket.socket]) -> int:
return reserved_tcp_port[0]
@pytest.fixture(autouse=True)
def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Give every test its own host prefs dir; prefs are keyed only by device
name, which tests sharing a fixture also share."""
prefdir = tmp_path / "prefs"
monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir))
return prefdir
@pytest_asyncio.fixture
async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> str:
"""Load YAML configuration based on test name."""
# Get the test function name
test_name: str = request.node.name
# Extract the base test name (remove test_ prefix and any parametrization)
base_name = test_name.replace("test_", "").partition("[")[0]
shared_name = _shared_yaml_name(request)
# Base test name: test_ prefix and any parametrization stripped
base_name = shared_name or request.node.name.replace("test_", "").partition("[")[0]
# Load the fixture file
fixture_path = Path(__file__).parent / "fixtures" / f"{base_name}.yaml"
fixture_path = FIXTURES_DIR / f"{base_name}.yaml"
if not fixture_path.exists():
raise FileNotFoundError(f"Fixture file not found: {fixture_path}")
loop = asyncio.get_running_loop()
content = await loop.run_in_executor(None, fixture_path.read_text)
content = await loop.run_in_executor(None, read_file, fixture_path)
# Replace the port in the config if it contains api section
if "api:" in content:
@@ -226,11 +254,13 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s
# Replace external component path placeholder if present
if "EXTERNAL_COMPONENT_PATH" in content:
external_components_path = str(
Path(__file__).parent / "fixtures" / "external_components"
)
external_components_path = str(FIXTURES_DIR / "external_components")
content = content.replace("EXTERNAL_COMPONENT_PATH", external_components_path)
if shared_name is not None:
# _compile verifies the marked test compiles this content unmodified
request.node._shared_yaml_content = content
return content
@@ -240,24 +270,218 @@ async def write_yaml_config(
) -> AsyncGenerator[ConfigWriter]:
"""Write YAML configuration to a file."""
# Get the test name for default filename
test_name = request.node.name
base_name = test_name.replace("test_", "").split("[")[0]
base_name = request.node.name.replace("test_", "").partition("[")[0]
async def _write_config(content: str, filename: str | None = None) -> Path:
if filename is None:
filename = f"{base_name}.yaml"
config_path = integration_test_dir / filename
loop = asyncio.get_running_loop()
await loop.run_in_executor(None, config_path.write_text, content)
await loop.run_in_executor(None, write_file, config_path, content)
return config_path
yield _write_config
# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale
# dirs for a fixture are pruned when its content hash changes.
SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds"
# In the dir name (not just the hash) so pruning stays inside this checkout
_REPO_KEY = hashlib.sha256(str(REPO_ROOT).encode()).hexdigest()[:8]
# Give a contended shared build lock time for a full cold compile ahead of us
_SHARED_LOCK_TIMEOUT_S = 900
_SHARED_LOCK_POLL_S = 0.1
_SHARED_LOCK_REPORT_S = 30
# Reclaims dirs orphaned by fixture renames or deleted checkouts
_STALE_BUILD_MAX_AGE_S = 30 * 24 * 3600
# ELF path per shared build dir; constant once compiled, so resolve it only once
_shared_elf_paths: dict[Path, Path] = {}
# Dirs this process already swept; pruning is session-scoped work
_pruned_dirs: set[Path] = set()
def _shared_yaml_name(request: pytest.FixtureRequest) -> str | None:
"""Name passed to the shared_yaml marker, or None when unmarked."""
marker = request.node.get_closest_marker("shared_yaml")
if marker is None:
return None
# Exactly one \w+ positional arg: the name doubles as a build dir
# component, and CI test selection (script/helpers.py) parses the same shape
if (
len(marker.args) != 1
or marker.kwargs
or not re.fullmatch(r"\w+", str(marker.args[0]))
):
raise ValueError(
"shared_yaml marker requires exactly one \\w+ fixture name literal"
)
return marker.args[0]
def _shared_build_prefix(name: str) -> str:
return f"{name}-{_REPO_KEY}-"
@cache
def _shared_build_dir(name: str) -> Path:
"""Dir keyed by checkout and fixture source, before per-test injections."""
key = hashlib.sha256((FIXTURES_DIR / f"{name}.yaml").read_bytes()).hexdigest()[:16]
return SHARED_BUILDS_ROOT / (_shared_build_prefix(name) + key)
def _read_stamp(stamp: Path, shared_dir: Path) -> Path | None:
"""ELF path recorded by the last completed compile, or None."""
try:
text = stamp.read_text(encoding="utf-8").strip()
except FileNotFoundError:
return None
except OSError as err:
print(f"Cannot read {stamp}: {err}")
return None
if not text:
print(f"Ignoring empty stamp {stamp}")
return None
built = Path(text)
# Never trust a stamp pointing outside its own build dir as an unlink target
if shared_dir.resolve() in built.resolve().parents:
return built
print(f"Ignoring stamp {stamp} pointing outside {shared_dir}")
return None
def _unused_since(stale: Path, cutoff: float) -> bool:
"""Whether a build dir looks untouched since cutoff; unknown counts as used."""
# Newest of the .built stamp (rewritten by every completed compile) and the
# dir itself (freshened by a worker claiming the dir before locking)
newest: float | None = None
for probe in (stale / ".built", stale):
try:
mtime = probe.stat().st_mtime
except FileNotFoundError:
continue
except NotADirectoryError:
return True # a stray file where a dir should be; reclaimable
except OSError as err:
print(f"Cannot age-probe {stale}: {err}")
return False # unknown never authorizes deletion
newest = mtime if newest is None else max(newest, mtime)
return newest is not None and newest < cutoff
def _prune_stale_builds(name: str, keep: Path) -> None:
"""Remove outdated build dirs (blocking, run in executor): this checkout's
other dirs for the fixture, plus anything untouched for 30 days. Tolerates
other workers pruning the same dirs concurrently."""
cutoff = time.time() - _STALE_BUILD_MAX_AGE_S
prefix = _shared_build_prefix(name)
for stale in SHARED_BUILDS_ROOT.iterdir():
if stale == keep:
continue
same_fixture = stale.name.startswith(prefix)
if not same_fixture and not _unused_since(stale, cutoff):
continue
# Creating .lock bumps the dir mtime, so remember whether the re-probe
# under the lock can trust it
lock_preexisting = (stale / ".lock").exists()
try:
lock_file = (stale / ".lock").open("w")
except FileNotFoundError:
continue # pruned by another worker meanwhile
except NotADirectoryError:
print(f"Removing stray file {stale}")
stale.unlink(missing_ok=True)
continue
except OSError as err:
print(f"Cannot prune {stale}: {err}")
continue
with lock_file:
try:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
continue # still in use by another run
# Re-probe under the lock: a worker freshens its dir before
# locking, so a just-claimed dir no longer looks unused. A dir
# whose .lock we just created cannot be held by anyone, and our
# own open bumped its mtime, so its pre-open probe stands
if (
lock_preexisting
and not same_fixture
and not _unused_since(stale, cutoff)
):
continue
# rmtree tolerates races; a leftover partial tree only costs a
# rebuild, since the ELF is deleted before every compile
try:
rmtree(stale)
except OSError as err:
print(f"Failed to prune {stale}: {err}")
async def _run_esphome_compile(
config_path: Path, cwd: Path, env: dict[str, str]
) -> None:
"""Run `esphome compile`, retrying up to 3 times on a segfault."""
max_retries = 3
for attempt in range(max_retries):
# Compile using subprocess, inheriting stdout/stderr to show progress
proc = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"esphome",
"compile",
str(config_path),
cwd=cwd,
stdout=None, # Inherit stdout
stderr=None, # Inherit stderr
stdin=asyncio.subprocess.DEVNULL,
# Start in a new process group to isolate signal handling
start_new_session=True,
env=env,
close_fds=False,
)
await proc.wait()
if proc.returncode == 0:
break
if proc.returncode == -11 and attempt < max_retries - 1:
# Segfault (-11 = SIGSEGV), retry
print(
f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..."
)
await asyncio.sleep(1) # Brief pause before retry
continue
raise RuntimeError(
f"Failed to compile {config_path}, return code: {proc.returncode}. "
f"Run with 'pytest -s' to see compilation output."
)
def _resolve_compiled_binary(config_path: Path) -> Path:
"""Load the config to learn the compiled ELF path (blocking, run in executor)."""
CORE.reset() # Reset CORE state between test runs
CORE.config_path = config_path
config = esphome.config.read_config(
{"command": "compile", "config": str(config_path)}
)
if config is None:
raise RuntimeError(f"Failed to read config from {config_path}")
idedata = get_idedata(config)
binary_path = Path(idedata.firmware_elf_path)
if not binary_path.exists():
raise RuntimeError(f"Compiled binary not found at {binary_path}")
return binary_path
@pytest_asyncio.fixture
async def compile_esphome(
integration_test_dir: Path,
shared_platformio_cache: Path,
request: pytest.FixtureRequest,
) -> AsyncGenerator[CompileFunction]:
"""Compile an ESPHome configuration and return the binary path."""
@@ -265,66 +489,96 @@ async def compile_esphome(
# Use the shared PlatformIO cache for faster compilation
# This avoids re-downloading dependencies for each test
env = _get_platformio_env(shared_platformio_cache)
# Retry compilation up to 3 times if we get a segfault
max_retries = 3
for attempt in range(max_retries):
# Compile using subprocess, inheriting stdout/stderr to show progress
proc = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"esphome",
"compile",
str(config_path),
cwd=integration_test_dir,
stdout=None, # Inherit stdout
stderr=None, # Inherit stderr
stdin=asyncio.subprocess.DEVNULL,
# Start in a new process group to isolate signal handling
start_new_session=True,
env=env,
close_fds=False,
)
await proc.wait()
if proc.returncode == 0:
# Success!
break
if proc.returncode == -11 and attempt < max_retries - 1:
# Segfault (-11 = SIGSEGV), retry
print(
f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..."
)
await asyncio.sleep(1) # Brief pause before retry
continue
# Other error or final retry
raise RuntimeError(
f"Failed to compile {config_path}, return code: {proc.returncode}. "
f"Run with 'pytest -s' to see compilation output."
)
# Load the config to get idedata (blocking call, must use executor)
loop = asyncio.get_running_loop()
def _read_config_and_get_binary():
CORE.reset() # Reset CORE state between test runs
CORE.config_path = config_path
config = esphome.config.read_config(
{"command": "compile", "config": str(config_path)}
name = _shared_yaml_name(request)
if name is None:
await _run_esphome_compile(config_path, integration_test_dir, env)
return await loop.run_in_executor(
None, _resolve_compiled_binary, config_path
)
if config is None:
raise RuntimeError(f"Failed to read config from {config_path}")
# Get the compiled binary path
idedata = get_idedata(config)
return Path(idedata.firmware_elf_path)
binary_path = await loop.run_in_executor(None, _read_config_and_get_binary)
if not binary_path.exists():
raise RuntimeError(f"Compiled binary not found at {binary_path}")
return binary_path
# Shared fixture: build in a hash-keyed dir so tests sharing a config
# pay one full compile and later only a main.cpp (port) rebuild + relink
shared_dir = _shared_build_dir(name)
shared_dir.mkdir(parents=True, exist_ok=True)
# Freshen the dir before locking so a concurrent age sweep, which
# re-probes under the lock, never reaps a dir a worker just claimed;
# if a peer reaped it already, the guarded lock open recreates it
with suppress(FileNotFoundError):
os.utime(shared_dir)
if shared_dir not in _pruned_dirs:
_pruned_dirs.add(shared_dir)
await loop.run_in_executor(None, _prune_stale_builds, name, shared_dir)
shared_config = shared_dir / f"{name}.yaml"
private_binary = integration_test_dir / f"{name}.elf"
content = await loop.run_in_executor(None, read_file, config_path)
if content != getattr(request.node, "_shared_yaml_content", None):
# The dir is keyed by the fixture source; a mutated config would be
# cached under a hash that does not describe it
raise RuntimeError(
"shared_yaml tests must compile the yaml_config content unmodified"
)
# flock serializes concurrent xdist workers; closing the fd releases it.
# Hand-rolled rather than filelock.FileLock: non-blocking retries keep
# the wait cancellable, while a blocking acquire in an executor thread
# would survive test cancellation holding the fd
try:
lock_file = (shared_dir / ".lock").open("w")
except FileNotFoundError:
# A peer run pruning divergent hashes reaped the dir between our
# mkdir and this open; recreate it and pay a full rebuild
shared_dir.mkdir(parents=True, exist_ok=True)
lock_file = (shared_dir / ".lock").open("w")
with lock_file:
start = time.monotonic()
last_report = start
while True:
try:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except BlockingIOError:
now = time.monotonic()
if now - start > _SHARED_LOCK_TIMEOUT_S:
raise RuntimeError(
f"Timed out waiting for the {shared_dir} lock"
) from None
if now - last_report >= _SHARED_LOCK_REPORT_S:
last_report = now
print(
f"Waited {now - start:.0f}s for another worker's "
f"build of {shared_dir.name}"
)
await asyncio.sleep(_SHARED_LOCK_POLL_S)
# .built carries the ELF path of the last completed compile, so
# later workers skip the config re-read in _resolve_compiled_binary
stamp = shared_dir / ".built"
if (built := _shared_elf_paths.get(shared_dir)) is None:
built = await loop.run_in_executor(None, _read_stamp, stamp, shared_dir)
# Delete the ELF before compiling: whatever exists afterwards is
# this compile's output, so no staleness check is ever needed.
# With no usable stamp, sweep any leftover at the known layout
if built is not None:
built.unlink(missing_ok=True)
else:
# Layout-agnostic: ESPHOME_BUILD_PATH can move the build tree
for leftover in shared_dir.rglob("program"):
if leftover.is_file():
leftover.unlink()
await loop.run_in_executor(
None, write_file_if_changed, shared_config, content
)
await _run_esphome_compile(shared_config, shared_dir, env)
if built is None or not built.exists():
built = await loop.run_in_executor(
None, _resolve_compiled_binary, shared_config
)
_shared_elf_paths[shared_dir] = built
await loop.run_in_executor(None, write_file, stamp, str(built))
# Copy out before unlocking: another worker may relink firmware.elf
# while this test is still running its private copy
await loop.run_in_executor(None, shutil.copy2, built, private_binary)
return private_binary
yield _compile
@@ -1,58 +0,0 @@
esphome:
name: test-batch-window-filters
host:
api:
batch_delay: 0ms # Disable batching to receive all state updates
logger:
level: DEBUG
# Template sensor that we'll use to publish values
sensor:
- platform: template
name: "Source Sensor"
id: source_sensor
accuracy_decimals: 2
# Batch window filters (window_size == send_every) - use streaming filters
- platform: copy
source_id: source_sensor
name: "Min Sensor"
id: min_sensor
filters:
- min:
window_size: 5
send_every: 5
send_first_at: 1
- platform: copy
source_id: source_sensor
name: "Max Sensor"
id: max_sensor
filters:
- max:
window_size: 5
send_every: 5
send_first_at: 1
- platform: copy
source_id: source_sensor
name: "Moving Avg Sensor"
id: moving_avg_sensor
filters:
- sliding_window_moving_average:
window_size: 5
send_every: 5
send_first_at: 1
# Button to trigger publishing test values
button:
- platform: template
name: "Publish Values Button"
id: publish_button
on_press:
- lambda: |-
// Publish 10 values: 1.0, 2.0, ..., 10.0
for (int i = 1; i <= 10; i++) {
id(source_sensor).publish_state(float(i));
}
@@ -1,111 +0,0 @@
esphome:
name: uart-mock-modbus-cli-rw
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
# Two virtual buses looped back to each other: the client's transmissions reach the server and the
# server's replies reach the client. auto_start so forwarding is active before the button fires.
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_client
data: !lambda return data;
- id: virtual_uart_client
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: stored_1
type: uint16_t
initial_value: "0"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_client
id: virtual_modbus_client
role: client
turnaround_time: 10ms
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
registers:
# Writable + readable register: the read publishes what it returns, so the test can confirm the
# write half of the 0x17 ran before the read half (Modbus 6.17).
- address: 0x01
value_type: U_WORD
read_lambda: |-
id(srv_read_1).publish_state(id(stored_1));
return id(stored_1);
write_lambda: |-
id(stored_1) = x;
id(srv_write_1).publish_state(x);
return true;
# Read-only register, returned together with 0x01 by the 2-register read half.
- address: 0x02
value_type: U_WORD
read_lambda: return 0x00AA;
sensor:
# Server-side observations.
- platform: template
name: "srv_write_1"
id: srv_write_1
- platform: template
name: "srv_read_1"
id: srv_read_1
# Client-side read-back: the values the client's on_response received.
- platform: template
name: "client_read_0"
id: client_read_0
- platform: template
name: "client_read_1"
id: client_read_1
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
on_press:
# FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction.
- modbus_client.read_write_multiple_registers:
address: 0x01
read_address: 0x0001
read_count: 2
write_address: 0x0001
values: [0x1234]
on_response:
then:
- lambda: |-
// values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002.
if (values.size() >= 2) {
id(client_read_0).publish_state(values[0]);
id(client_read_1).publish_state(values[1]);
}
@@ -1,88 +0,0 @@
esphome:
name: uart-mock-modbus-custom-pdu
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 259;
sensor:
# Plain read to confirm the controller <-> server link is up.
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "plain_read"
address: 0x01
register_type: holding
value_type: U_WORD
# Custom PDU: read holding register 0x0001, count 1. The PDU is
# {function code, address hi, address lo, count hi, count lo}; the device
# address and CRC are added by the hub. The lambda parses the response payload
# (the register value, big-endian).
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "custom_read"
custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01]
lambda: |-
if (data.size() < 2) return {};
return (float) ((data[0] << 8) | data[1]);
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
@@ -1,106 +0,0 @@
esphome:
name: uart-mock-modbus-dep-buffer
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: reg10
type: uint16_t
initial_value: "0"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x10
value_type: U_WORD
read_lambda: return id(reg10);
write_lambda: |-
id(reg10) = x;
return true;
# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw
# frame as words: device address + function code + data) instead of the new item->write_* API. The write
# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per
# entity no matter how many writes happen.
number:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "buf_number"
id: buf_number
address: 0x10
register_type: holding
value_type: U_WORD
min_value: 0
max_value: 1000
step: 1
write_lambda: |-
// Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value.
payload.push_back(0x0106);
payload.push_back(0x0010);
payload.push_back((uint16_t) x);
return {};
# Reports the server-side register so the test can observe that the deprecated buffer write landed.
sensor:
- platform: template
name: "written_value"
id: written_value
update_interval: 0.5s
lambda: "return id(reg10);"
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# The test drives the writes via number_command; the mock is autostart.
@@ -1,95 +0,0 @@
esphome:
name: uart-mock-modbus-lambda-invert
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: reg40
type: uint16_t
initial_value: "5"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x40
value_type: U_WORD
read_lambda: return id(reg40);
write_lambda: id(reg40) = x; return true;
# An active-low holding switch: the write_lambda inverts the wire value, but the entity must still
# report the REQUESTED state. assumed_state keeps the register unpolled, so the published state comes
# only from write_state() - turning ON writes 0x0000 yet the switch shows ON.
switch:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "invert_switch"
register_type: holding
address: 0x40
assumed_state: true
write_lambda: |-
return !x;
sensor:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_40"
address: 0x40
register_type: holding
value_type: U_WORD
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
@@ -1,97 +0,0 @@
esphome:
name: uart-mock-modbus-lambda-write
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: reg30
type: uint16_t
initial_value: "0"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x30
value_type: U_WORD
read_lambda: return id(reg30);
write_lambda: id(reg30) = x; return true;
# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead
# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so
# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing
# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write.
switch:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "cross_switch"
register_type: coil
address: 0x00
assumed_state: true
write_lambda: |-
item->write_single_register(0x30, x ? 1234 : 0);
return {};
sensor:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_30"
address: 0x30
register_type: holding
value_type: U_WORD
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
@@ -0,0 +1,233 @@
esphome:
name: uart-mock-modbus-loopback
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
# Shared loopback fixture (see the shared_yaml markers in the test file);
# register spaces are disjoint so each test only observes its own entities.
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: reg10
type: uint16_t
initial_value: "100"
- id: reg11
type: uint16_t
initial_value: "200"
- id: reg12
type: uint16_t
initial_value: "300"
- id: reg13
type: uint16_t
initial_value: "0xABCD"
- id: reg30
type: uint16_t
initial_value: "0"
- id: reg40
type: uint16_t
initial_value: "5"
- id: reg50
type: uint16_t
initial_value: "0"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 259;
- address: 0x10
value_type: U_WORD
read_lambda: return id(reg10);
write_lambda: id(reg10) = x; return true;
- address: 0x11
value_type: U_WORD
read_lambda: return id(reg11);
write_lambda: id(reg11) = x; return true;
- address: 0x12
value_type: U_WORD
read_lambda: return id(reg12);
write_lambda: id(reg12) = x; return true;
- address: 0x13
value_type: U_WORD
read_lambda: return id(reg13);
- address: 0x30
value_type: U_WORD
read_lambda: return id(reg30);
write_lambda: id(reg30) = x; return true;
- address: 0x40
value_type: U_WORD
read_lambda: return id(reg40);
write_lambda: id(reg40) = x; return true;
- address: 0x50
value_type: U_WORD
read_lambda: return id(reg50);
write_lambda: id(reg50) = x; return true;
# Byte-based offset: 2 bytes -> register 0x11 (the old code folded it in as a
# register count, hitting 0x12). assumed_state keeps the switch write-only.
switch:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "offset_switch"
register_type: holding
address: 0x10
offset: 2
assumed_state: true
# Reading switch, byte offset 6 -> register 0x13; the pre-fix resolution (0x16)
# would draw ILLEGAL_DATA_ADDRESS and never publish.
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "read_offset_switch"
register_type: holding
address: 0x10
offset: 6
bitmask: 0x1
# Coil switch whose write_lambda dispatches a holding-register write via `item`;
# returning an empty optional suppresses the default coil write.
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "cross_switch"
register_type: coil
address: 0x00
assumed_state: true
write_lambda: |-
item->write_single_register(0x30, x ? 1234 : 0);
return {};
# Active-low: the write_lambda inverts the wire value but the entity must still
# report the requested state (assumed_state keeps the register unpolled).
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "invert_switch"
register_type: holding
address: 0x40
assumed_state: true
write_lambda: |-
return !x;
# Uses the deprecated buffer parameter (legacy raw frame as words); the write
# must land and the deprecation warning must fire only once per entity.
number:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "buf_number"
id: buf_number
address: 0x50
register_type: holding
value_type: U_WORD
min_value: 0
max_value: 1000
step: 1
write_lambda: |-
// Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0050, value.
payload.push_back(0x0106);
payload.push_back(0x0050);
payload.push_back((uint16_t) x);
return {};
sensor:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "plain_read"
address: 0x01
register_type: holding
value_type: U_WORD
# Custom PDU: read holding register 0x0001; device address and CRC are added
# by the hub. The lambda parses the big-endian register value.
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "custom_read"
custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01]
lambda: |-
if (data.size() < 2) return {};
return (float) ((data[0] << 8) | data[1]);
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_10"
address: 0x10
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_11"
address: 0x11
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_12"
address: 0x12
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_30"
address: 0x30
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_40"
address: 0x40
register_type: holding
value_type: U_WORD
# Reports the server-side register so the test can observe that the deprecated buffer write landed.
- platform: template
name: "written_value"
id: written_value
update_interval: 0.5s
lambda: "return id(reg50);"
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# Nothing to start (mock is autostart); tests drive entities directly
@@ -1,5 +1,5 @@
esphome:
name: uart-mock-modbus-server-contro
name: uart-mock-modbus-mesh
host:
api:
@@ -17,13 +17,14 @@ uart:
baud_rate: 115200
port: /dev/null
# Shared 3-bus mesh (see the shared_yaml markers): addr 1 = typed read-only
# registers, addr 5 = the read/write 0x17 target, addr 2/3 on the second
# server hub. auto_start everywhere: the controller polls at boot, so the
# forwarding must already be live or early requests generate warnings.
# Every test presses Start Scenario, so all merged actions fire in every test.
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
# auto_start must be true for loopback fixtures: the modbus controller
# polls on its update_interval immediately at boot, so the uart_mock
# forwarding must already be active or early requests are lost and
# generate modbus warnings.
auto_start: true
debug:
on_tx:
@@ -31,35 +32,68 @@ uart_mock:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
- uart_mock.inject_rx:
id: virtual_uart_server_2
data: !lambda return data;
- id: virtual_uart_server_2
baud_rate: 9600
auto_start: true # See comment on virtual_uart_server above
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
- uart_mock.inject_rx:
id: virtual_uart_server_2
data: !lambda return data;
globals:
- id: stored_1
type: uint16_t
initial_value: "0"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_server_2
id: virtual_modbus_server_2
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
id: virtual_modbus_client
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
modbus_id: virtual_modbus_client
id: modbus_controller_1
update_interval: 1s
- address: 2
modbus_id: virtual_modbus_client
id: modbus_controller_2
update_interval: 1s
- address: 3
modbus_id: virtual_modbus_client
id: modbus_controller_3
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x01
value_type: U_WORD
@@ -103,6 +137,34 @@ modbus_server:
- address: 0x28
value_type: FP32_R
read_lambda: return 3.14;
- address: 5
modbus_id: virtual_modbus_server
registers:
# Writable + readable register: srv_write_1 plus the client's read-back
# confirm the write half of the 0x17 ran before the read half (Modbus 6.17).
- address: 0x01
value_type: U_WORD
read_lambda: return id(stored_1);
write_lambda: |-
id(stored_1) = x;
id(srv_write_1).publish_state(x);
return true;
# Read-only register, returned together with 0x01 by the 2-register read half.
- address: 0x02
value_type: U_WORD
read_lambda: return 0x00AA;
- address: 2
modbus_id: virtual_modbus_server_2
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 919;
- address: 3
modbus_id: virtual_modbus_server_2
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 929;
sensor:
- platform: modbus_controller
@@ -195,9 +257,46 @@ sensor:
address: 0x28
register_type: holding
value_type: FP32_R
- platform: modbus_controller
modbus_controller_id: modbus_controller_2
name: "multi_reg_a"
address: 0x01
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_3
name: "multi_reg_b"
address: 0x01
register_type: holding
value_type: U_WORD
# client_read_write observations, server- and client-side.
- platform: template
name: "srv_write_1"
id: srv_write_1
- platform: template
name: "client_read_0"
id: client_read_0
- platform: template
name: "client_read_1"
id: client_read_1
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
on_press:
# FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction.
- modbus_client.read_write_multiple_registers:
address: 5
read_address: 0x0001
read_count: 2
write_address: 0x0001
values: [0x1234]
on_response:
then:
- lambda: |-
// values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002.
if (values.size() >= 2) {
id(client_read_0).publish_state(values[0]);
id(client_read_1).publish_state(values[1]);
}
@@ -1,138 +0,0 @@
esphome:
name: uart-mock-modbus-reg-offset
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
globals:
- id: reg10
type: uint16_t
initial_value: "100"
- id: reg11
type: uint16_t
initial_value: "200"
- id: reg12
type: uint16_t
initial_value: "300"
- id: reg13
type: uint16_t
initial_value: "0xABCD"
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_controller
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_controller
id: modbus_controller_1
update_interval: 1s
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
id: modbus_server_1
registers:
- address: 0x10
value_type: U_WORD
read_lambda: return id(reg10);
write_lambda: id(reg10) = x; return true;
- address: 0x11
value_type: U_WORD
read_lambda: return id(reg11);
write_lambda: id(reg11) = x; return true;
- address: 0x12
value_type: U_WORD
read_lambda: return id(reg12);
write_lambda: id(reg12) = x; return true;
- address: 0x13
value_type: U_WORD
read_lambda: return id(reg13);
write_lambda: id(reg13) = x; return true;
# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target
# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register
# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register.
switch:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "offset_switch"
register_type: holding
address: 0x10
offset: 2
assumed_state: true
# A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix
# the switch itself resolves to 0x13 (the even byte offset folds into the address as whole registers) and
# joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds
# into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes.
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "read_offset_switch"
register_type: holding
address: 0x10
offset: 6
bitmask: 0x1
sensor:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_10"
address: 0x10
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_11"
address: 0x11
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_12"
address: 0x12
register_type: holding
value_type: U_WORD
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
@@ -1,124 +0,0 @@
esphome:
name: uart-mock-modbus-server-test
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_dev
baud_rate: 9600
rx_full_threshold: 120
rx_timeout: 2
auto_start: false
debug:
injections:
- delay: 100ms
inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read)
- delay: 100ms
# Read holding register 7 on device 2
# Reply from device 2
# Read holding register 5 on device 1 (read_after_peer_response)
inject_rx:
[
0x02,
0x03,
0x00,
0x07,
0x00,
0x01,
0x35,
0xF8,
0x02,
0x03,
0x02,
0x00,
0xF0,
0xFC,
0x00,
0x01,
0x03,
0x00,
0x05,
0x00,
0x01,
0x94,
0x0B,
]
- delay: 100ms
inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response
- delay: 100ms
# Read holding register 7 on device 2, with no response
# Read holding register A on device 1 (read_after_peer_timeout)
inject_rx:
[
0x02,
0x03,
0x00,
0x07,
0x00,
0x01,
0x35,
0xF8,
0x01,
0x03,
0x00,
0x0A,
0x00,
0x01,
0xA4,
0x08,
]
modbus:
uart_id: virtual_uart_dev
role: server
modbus_server:
- address: 1
registers:
- address: 0x03
value_type: U_WORD
read_lambda: |-
id(basic_read).publish_state(1);
return 1;
- address: 0x05
value_type: U_WORD
read_lambda: |-
id(read_after_peer_response).publish_state(1);
return 1;
- address: 0x0A
value_type: U_WORD
read_lambda: |-
id(read_after_peer_timeout).publish_state(1);
return 1;
sensor:
- platform: template
name: "basic_read"
id: basic_read
- platform: template
name: "read_after_peer_response"
id: read_after_peer_response
- platform: template
name: "read_after_peer_timeout"
id: read_after_peer_timeout
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
on_press:
- lambda: "id(virtual_uart_dev).start_scenario();"
@@ -1,116 +0,0 @@
esphome:
name: uart-mock-modbus-server-mult
host:
api:
logger:
level: VERBOSE
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"]
# The actual UART bus used is the uart_mock component below
uart:
baud_rate: 115200
port: /dev/null
uart_mock:
- id: virtual_uart_server
baud_rate: 9600
# auto_start must be true for loopback fixtures: the modbus controller
# polls on its update_interval immediately at boot, so the uart_mock
# forwarding must already be active or early requests are lost and
# generate modbus warnings.
auto_start: true
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- uart_mock.inject_rx:
id: virtual_uart_server_2
data: !lambda return data;
- id: virtual_uart_server_2
baud_rate: 9600
auto_start: true # See comment on virtual_uart_server above
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
- uart_mock.inject_rx:
id: virtual_uart_controller
data: !lambda return data;
- id: virtual_uart_controller
baud_rate: 9600
auto_start: true # See comment on virtual_uart_server above
debug:
on_tx:
- then:
- uart_mock.inject_rx:
id: virtual_uart_server
data: !lambda return data;
- uart_mock.inject_rx:
id: virtual_uart_server_2
data: !lambda return data;
modbus:
- uart_id: virtual_uart_server
id: virtual_modbus_server
role: server
- uart_id: virtual_uart_server_2
id: virtual_modbus_server_2
role: server
- uart_id: virtual_uart_controller
id: virtual_modbus_client
role: client
turnaround_time: 10ms
modbus_controller:
- address: 1
modbus_id: virtual_modbus_client
update_interval: 1s
id: modbus_controller_1
- address: 2
modbus_id: virtual_modbus_client
update_interval: 1s
id: modbus_controller_2
modbus_server:
- address: 1
modbus_id: virtual_modbus_server
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 919;
- address: 2
modbus_id: virtual_modbus_server_2
registers:
- address: 0x01
value_type: U_WORD
read_lambda: return 929;
sensor:
- platform: modbus_controller
modbus_controller_id: modbus_controller_1
name: "reg_u_word"
address: 0x01
register_type: holding
value_type: U_WORD
- platform: modbus_controller
modbus_controller_id: modbus_controller_2
name: "reg_u_word_2"
address: 0x01
register_type: holding
value_type: U_WORD
button:
- platform: template
name: "Start Scenario"
id: start_scenario_btn
# This test does not have anything to start (mock is autostart)
@@ -1,5 +1,5 @@
esphome:
name: uart-mock-modbus-srv-rw
name: uart-mock-modbus-srv-injected
host:
api:
@@ -17,6 +17,8 @@ uart:
baud_rate: 115200
port: /dev/null
# Shared server-role fixture (see the shared_yaml markers in the test file);
# the injections concatenate and each test waits only on its own sensors.
uart_mock:
- id: virtual_uart_dev
baud_rate: 9600
@@ -25,18 +27,31 @@ uart_mock:
auto_start: false
debug:
injections:
# FC 0x17 Read/Write Multiple Registers on device 1:
# write reg 0x0001 = 0x1234 (qty 1), then read regs 0x0001..0x0002 (qty 2).
# Per Modbus 6.17 the write is performed before the read, so reg 0x0001 must
# read back the just-written 0x1234 in the same request.
- delay: 100ms
inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read)
- delay: 100ms
# Read holding register 7 on device 2, its reply, then read holding
# register 5 on device 1 (read_after_peer_response)
inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8,
0x02, 0x03, 0x02, 0x00, 0xF0, 0xFC,
0x00, 0x01, 0x03, 0x00, 0x05, 0x00, 0x01, 0x94, 0x0B]
- delay: 100ms
inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response
- delay: 100ms
# Read holding register 7 on device 2 with no response, then read
# holding register A on device 1 (read_after_peer_timeout)
inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8,
0x01, 0x03, 0x00, 0x0A, 0x00, 0x01, 0xA4, 0x08]
# FC 0x17 on device 1: write reg 0x0001 = 0x1234 then read 0x0001..0x0002;
# per Modbus 6.17 the write runs first, so 0x0001 must read back 0x1234.
- delay: 100ms
inject_rx:
[0x01, 0x17, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x02, 0x12, 0x34, 0x49, 0xD8]
# FC 0x17: write reg 0x0003 = 0x5678 (qty 1), then read reg 0x0003 (qty 1) -
# FC 0x17: write reg 0x0006 = 0x5678 (qty 1), then read reg 0x0006 (qty 1) -
# a write and read targeting a different register block.
- delay: 100ms
inject_rx:
[0x01, 0x17, 0x00, 0x03, 0x00, 0x01, 0x00, 0x03, 0x00, 0x01, 0x02, 0x56, 0x78, 0x9B, 0x10]
[0x01, 0x17, 0x00, 0x06, 0x00, 0x01, 0x00, 0x06, 0x00, 0x01, 0x02, 0x56, 0x78, 0x8B, 0x55]
globals:
- id: stored_1
@@ -70,8 +85,18 @@ modbus_server:
read_lambda: |-
id(rw_read_2).publish_state(0x00AA);
return 0x00AA;
# Second writable + readable register, targeted by the second request.
- address: 0x03
value_type: U_WORD
read_lambda: |-
id(basic_read).publish_state(1);
return 1;
- address: 0x05
value_type: U_WORD
read_lambda: |-
id(read_after_peer_response).publish_state(1);
return 1;
# Second writable + readable register, targeted by the second FC 0x17 request.
- address: 0x06
value_type: U_WORD
read_lambda: |-
id(rw_read_3).publish_state(id(stored_3));
@@ -80,8 +105,22 @@ modbus_server:
id(stored_3) = x;
id(rw_write_3).publish_state(x);
return true;
- address: 0x0A
value_type: U_WORD
read_lambda: |-
id(read_after_peer_timeout).publish_state(1);
return 1;
sensor:
- platform: template
name: "basic_read"
id: basic_read
- platform: template
name: "read_after_peer_response"
id: read_after_peer_response
- platform: template
name: "read_after_peer_timeout"
id: read_after_peer_timeout
- platform: template
name: "rw_write_1"
id: rw_write_1
+11 -3
View File
@@ -1,7 +1,7 @@
"""Helpers for manipulating the host platform's preferences file.
ESPHome's host platform stores preferences in
``~/.esphome/prefs/<app_name>.prefs`` using a simple binary layout that
``$ESPHOME_PREFDIR/<app_name>.prefs`` using a simple binary layout that
mirrors ``HostPreferences::sync()``:
``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry.
@@ -11,13 +11,21 @@ boot (e.g. forcing safe mode) or to clear stale state between runs.
from __future__ import annotations
import os
from pathlib import Path
import struct
def host_prefs_path(device_name: str) -> Path:
"""Return the on-disk prefs file path for a host-platform device."""
return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs"
"""Return the on-disk prefs file path for a host-platform device.
Requires ESPHOME_PREFDIR, which the autouse isolated_preferences fixture
sets; refusing the ~/.esphome/prefs fallback keeps tests off real user
data if the fixture is ever bypassed."""
prefdir = os.environ.get("ESPHOME_PREFDIR")
if not prefdir:
raise RuntimeError("ESPHOME_PREFDIR is not set; refusing the real prefs dir")
return Path(prefdir) / f"{device_name}.prefs"
def clear_host_prefs(device_name: str) -> None:
+152 -141
View File
@@ -1,143 +1,154 @@
{
"tests/integration/test_action_concurrent_reentry.py": 57.91,
"tests/integration/test_addressable_light_transition.py": 21.25,
"tests/integration/test_alarm_control_panel_state_transitions.py": 70.71,
"tests/integration/test_api_action_metadata.py": 66.6,
"tests/integration/test_api_action_responses.py": 36.1,
"tests/integration/test_api_action_timeout.py": 68.86,
"tests/integration/test_api_conditional_memory.py": 15.48,
"tests/integration/test_api_custom_services.py": 18.77,
"tests/integration/test_api_get_time_response_timezone.py": 21.08,
"tests/integration/test_api_homeassistant.py": 65.59,
"tests/integration/test_api_homeassistant_action_no_subscriber.py": 18.44,
"tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 15.05,
"tests/integration/test_api_list_entities_backpressure.py": 13.88,
"tests/integration/test_api_message_size_batching.py": 29.98,
"tests/integration/test_api_reboot_timeout.py": 16.05,
"tests/integration/test_api_string_lambda.py": 15.31,
"tests/integration/test_api_vv_logging.py": 19.28,
"tests/integration/test_api_zero_psk_provisioning.py": 31.5,
"tests/integration/test_areas_and_devices.py": 24.95,
"tests/integration/test_automation_wait_actions.py": 20.92,
"tests/integration/test_automations.py": 35.19,
"tests/integration/test_batch_delay_zero_rapid_transitions.py": 17.99,
"tests/integration/test_binary_sensor_autorepeat_filter.py": 20.39,
"tests/integration/test_binary_sensor_invalidate_state.py": 18.41,
"tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 24.69,
"tests/integration/test_build_info.py": 18.7,
"tests/integration/test_camera_mock.py": 16.23,
"tests/integration/test_climate_control_action.py": 21.14,
"tests/integration/test_climate_custom_modes.py": 20.74,
"tests/integration/test_continuation_actions.py": 16.81,
"tests/integration/test_cover_control_action.py": 20.34,
"tests/integration/test_crc8_helper.py": 9.36,
"tests/integration/test_device_id_in_state.py": 44.67,
"tests/integration/test_duplicate_entities.py": 23.58,
"tests/integration/test_entity_icon.py": 34.35,
"tests/integration/test_fan_turn_on_action.py": 24.23,
"tests/integration/test_fnv1_hash_object_id.py": 16.21,
"tests/integration/test_fnv1a_hash.py": 13.38,
"tests/integration/test_gpio_expander_cache.py": 13.06,
"tests/integration/test_host_logger_thread_safety.py": 23.66,
"tests/integration/test_host_mode_basic.py": 8.01,
"tests/integration/test_host_mode_batch_delay.py": 21.0,
"tests/integration/test_host_mode_climate_basic_state.py": 22.14,
"tests/integration/test_host_mode_climate_control.py": 19.39,
"tests/integration/test_host_mode_empty_string_options.py": 21.76,
"tests/integration/test_host_mode_entity_fields.py": 29.61,
"tests/integration/test_host_mode_fan_preset.py": 20.01,
"tests/integration/test_host_mode_many_entities.py": 39.08,
"tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.92,
"tests/integration/test_host_mode_noise_encryption.py": 42.42,
"tests/integration/test_host_mode_reconnect.py": 3.41,
"tests/integration/test_host_mode_sensor.py": 22.96,
"tests/integration/test_host_ota.py": 29.5,
"tests/integration/test_host_preferences.py": 16.06,
"tests/integration/test_host_preferences_suspend_resume.py": 18.71,
"tests/integration/test_improv_serial_uart.py": 20.22,
"tests/integration/test_large_message_batching.py": 26.56,
"tests/integration/test_legacy_area.py": 22.72,
"tests/integration/test_legacy_climate_compat.py": 14.13,
"tests/integration/test_legacy_fan_compat.py": 14.33,
"tests/integration/test_light_automations.py": 18.81,
"tests/integration/test_light_binary_effect_off_phase.py": 8.38,
"tests/integration/test_light_calls.py": 21.88,
"tests/integration/test_light_constant_brightness.py": 59.45,
"tests/integration/test_light_control_action.py": 31.91,
"tests/integration/test_light_dim_relative_action.py": 14.43,
"tests/integration/test_light_effect_zero_brightness.py": 25.05,
"tests/integration/test_light_initial_state.py": 18.97,
"tests/integration/test_light_toggle_action.py": 17.44,
"tests/integration/test_lock_automations.py": 18.9,
"tests/integration/test_logger_buffered_recursion_guard.py": 18.2,
"tests/integration/test_loop_disable_enable.py": 63.35,
"tests/integration/test_loop_interval_decoupling.py": 17.7,
"tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.56,
"tests/integration/test_micros_to_millis.py": 15.89,
"tests/integration/test_multi_click_trigger.py": 17.23,
"tests/integration/test_multi_device_preferences.py": 19.4,
"tests/integration/test_noise_encryption_key_protection.py": 72.59,
"tests/integration/test_object_id_api_verification.py": 19.22,
"tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.77,
"tests/integration/test_object_id_no_friendly_name.py": 45.8,
"tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 86.73,
"tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.4,
"tests/integration/test_online_image_bmp.py": 37.24,
"tests/integration/test_oversized_payloads.py": 55.75,
"tests/integration/test_preference_key_stability.py": 25.49,
"tests/integration/test_runtime_stats.py": 29.81,
"tests/integration/test_safe_mode_loop_runs.py": 6.26,
"tests/integration/test_scheduler_blocking_warning.py": 37.98,
"tests/integration/test_scheduler_bulk_cleanup.py": 18.67,
"tests/integration/test_scheduler_defer_cancel.py": 18.46,
"tests/integration/test_scheduler_defer_cancel_regular.py": 16.34,
"tests/integration/test_scheduler_defer_fifo_simple.py": 18.26,
"tests/integration/test_scheduler_defer_stress.py": 17.74,
"tests/integration/test_scheduler_heap_stress.py": 3.89,
"tests/integration/test_scheduler_internal_id_no_collision.py": 20.01,
"tests/integration/test_scheduler_interval_reschedule.py": 16.29,
"tests/integration/test_scheduler_interval_zero_coerced.py": 16.09,
"tests/integration/test_scheduler_null_name.py": 14.69,
"tests/integration/test_scheduler_numeric_id_test.py": 17.08,
"tests/integration/test_scheduler_pool.py": 19.88,
"tests/integration/test_scheduler_rapid_cancellation.py": 4.42,
"tests/integration/test_scheduler_recursive_timeout.py": 4.3,
"tests/integration/test_scheduler_removed_item_race.py": 15.49,
"tests/integration/test_scheduler_self_keyed.py": 25.77,
"tests/integration/test_scheduler_simultaneous_callbacks.py": 14.84,
"tests/integration/test_scheduler_string_test.py": 15.42,
"tests/integration/test_script_array_params.py": 12.73,
"tests/integration/test_script_delay_params.py": 12.69,
"tests/integration/test_script_queued.py": 20.38,
"tests/integration/test_script_queued_idle_loop.py": 25.06,
"tests/integration/test_script_wait_on_boot.py": 15.67,
"tests/integration/test_select_stringref_trigger.py": 19.48,
"tests/integration/test_sensor_filters_delta.py": 27.62,
"tests/integration/test_sensor_filters_ring_buffer.py": 20.27,
"tests/integration/test_sensor_filters_sliding_window.py": 56.28,
"tests/integration/test_sensor_filters_value_list.py": 20.6,
"tests/integration/test_sensor_timeout_filter.py": 22.21,
"tests/integration/test_socket_wake_gate_tcp.py": 16.37,
"tests/integration/test_status_flags.py": 29.68,
"tests/integration/test_strftime_to.py": 17.42,
"tests/integration/test_syslog.py": 18.39,
"tests/integration/test_template_alarm_control_panel_many_sensors.py": 25.61,
"tests/integration/test_template_text_save.py": 19.16,
"tests/integration/test_text_command.py": 16.43,
"tests/integration/test_text_sensor_raw_state.py": 17.19,
"tests/integration/test_uart_mock_ld2410.py": 37.0,
"tests/integration/test_uart_mock_ld2412.py": 40.82,
"tests/integration/test_uart_mock_ld2420.py": 32.7,
"tests/integration/test_uart_mock_ld2450.py": 32.84,
"tests/integration/test_uart_mock_modbus.py": 548.87,
"tests/integration/test_udp.py": 16.67,
"tests/integration/test_use_address_runtime.py": 27.26,
"tests/integration/test_valve_control_action.py": 24.58,
"tests/integration/test_varint_five_byte_device_id.py": 22.5,
"tests/integration/test_wait_until_mid_loop_timing.py": 22.05,
"tests/integration/test_wait_until_on_boot.py": 10.37,
"tests/integration/test_wait_until_ordering.py": 18.23,
"tests/integration/test_wait_until_reentrant_restart.py": 19.35,
"tests/integration/test_wake_loop_forces_phase_b.py": 17.83,
"tests/integration/test_water_heater_template.py": 25.7
"tests/integration/test_action_concurrent_reentry.py": 30.48,
"tests/integration/test_addressable_light_transition.py": 42.1,
"tests/integration/test_alarm_control_panel_state_transitions.py": 35.76,
"tests/integration/test_api_action_metadata.py": 22.35,
"tests/integration/test_api_action_responses.py": 30.31,
"tests/integration/test_api_action_timeout.py": 34.73,
"tests/integration/test_api_conditional_memory.py": 18.35,
"tests/integration/test_api_custom_services.py": 15.99,
"tests/integration/test_api_get_time_response_timezone.py": 24.21,
"tests/integration/test_api_homeassistant.py": 33.77,
"tests/integration/test_api_homeassistant_action_no_subscriber.py": 20.8,
"tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 23.55,
"tests/integration/test_api_list_entities_backpressure.py": 23.04,
"tests/integration/test_api_message_size_batching.py": 27.31,
"tests/integration/test_api_reboot_timeout.py": 29.32,
"tests/integration/test_api_string_lambda.py": 14.9,
"tests/integration/test_api_vv_logging.py": 26.25,
"tests/integration/test_api_zero_psk_provisioning.py": 38.19,
"tests/integration/test_areas_and_devices.py": 27.52,
"tests/integration/test_automation_wait_actions.py": 24.25,
"tests/integration/test_automations.py": 36.02,
"tests/integration/test_batch_delay_zero_rapid_transitions.py": 18.46,
"tests/integration/test_binary_sensor_autorepeat_filter.py": 17.47,
"tests/integration/test_binary_sensor_invalidate_state.py": 14.79,
"tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 21.52,
"tests/integration/test_build_info.py": 21.42,
"tests/integration/test_camera_mock.py": 17.02,
"tests/integration/test_climate_control_action.py": 26.56,
"tests/integration/test_climate_custom_modes.py": 18.82,
"tests/integration/test_continuation_actions.py": 20.39,
"tests/integration/test_cover_control_action.py": 19.91,
"tests/integration/test_crc8_helper.py": 16.73,
"tests/integration/test_device_id_in_state.py": 58.41,
"tests/integration/test_duplicate_entities.py": 30.76,
"tests/integration/test_entity_icon.py": 25.34,
"tests/integration/test_fan_turn_on_action.py": 23.64,
"tests/integration/test_fnv1_hash_object_id.py": 25.44,
"tests/integration/test_fnv1a_hash.py": 20.85,
"tests/integration/test_gpio_expander_cache.py": 14.42,
"tests/integration/test_host_logger_thread_safety.py": 21.31,
"tests/integration/test_host_mode_basic.py": 2.65,
"tests/integration/test_host_mode_batch_delay.py": 22.21,
"tests/integration/test_host_mode_climate_basic_state.py": 27.12,
"tests/integration/test_host_mode_climate_control.py": 21.57,
"tests/integration/test_host_mode_empty_string_options.py": 27.17,
"tests/integration/test_host_mode_entity_fields.py": 30.1,
"tests/integration/test_host_mode_fan_preset.py": 17.55,
"tests/integration/test_host_mode_many_entities.py": 38.98,
"tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.82,
"tests/integration/test_host_mode_noise_encryption.py": 39.84,
"tests/integration/test_host_mode_reconnect.py": 13.1,
"tests/integration/test_host_mode_sensor.py": 22.17,
"tests/integration/test_host_ota.py": 92.05,
"tests/integration/test_host_preferences.py": 20.29,
"tests/integration/test_host_preferences_suspend_resume.py": 15.02,
"tests/integration/test_improv_serial_uart.py": 30.15,
"tests/integration/test_large_message_batching.py": 25.84,
"tests/integration/test_legacy_area.py": 21.24,
"tests/integration/test_legacy_climate_compat.py": 17.34,
"tests/integration/test_legacy_fan_compat.py": 22.6,
"tests/integration/test_light_automations.py": 29.13,
"tests/integration/test_light_binary_effect_off_phase.py": 33.99,
"tests/integration/test_light_calls.py": 26.81,
"tests/integration/test_light_constant_brightness.py": 25.0,
"tests/integration/test_light_control_action.py": 25.57,
"tests/integration/test_light_dim_relative_action.py": 21.4,
"tests/integration/test_light_effect_zero_brightness.py": 19.65,
"tests/integration/test_light_initial_state.py": 17.58,
"tests/integration/test_light_toggle_action.py": 28.28,
"tests/integration/test_lock_automations.py": 23.3,
"tests/integration/test_logger_buffered_recursion_guard.py": 22.96,
"tests/integration/test_loop_disable_enable.py": 16.18,
"tests/integration/test_loop_interval_decoupling.py": 25.19,
"tests/integration/test_loop_interval_default_not_pulled_forward.py": 20.59,
"tests/integration/test_lvgl_headless_render.py": 87.78,
"tests/integration/test_micros_to_millis.py": 18.73,
"tests/integration/test_multi_click_trigger.py": 24.2,
"tests/integration/test_multi_device_preferences.py": 20.52,
"tests/integration/test_noise_encryption_key_protection.py": 19.1,
"tests/integration/test_object_id_api_verification.py": 26.24,
"tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 14.88,
"tests/integration/test_object_id_no_friendly_name.py": 61.27,
"tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 82.32,
"tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 46.03,
"tests/integration/test_online_image_bmp.py": 34.21,
"tests/integration/test_oversized_payloads.py": 62.75,
"tests/integration/test_preference_key_stability.py": 26.8,
"tests/integration/test_runtime_stats.py": 28.26,
"tests/integration/test_safe_mode_loop_runs.py": 18.14,
"tests/integration/test_scheduler_blocking_warning.py": 28.7,
"tests/integration/test_scheduler_bulk_cleanup.py": 20.73,
"tests/integration/test_scheduler_defer_cancel.py": 22.99,
"tests/integration/test_scheduler_defer_cancel_regular.py": 21.97,
"tests/integration/test_scheduler_defer_fifo_simple.py": 24.15,
"tests/integration/test_scheduler_defer_stress.py": 23.11,
"tests/integration/test_scheduler_heap_stress.py": 20.2,
"tests/integration/test_scheduler_internal_id_no_collision.py": 23.75,
"tests/integration/test_scheduler_interval_reschedule.py": 15.32,
"tests/integration/test_scheduler_interval_zero_coerced.py": 20.1,
"tests/integration/test_scheduler_null_name.py": 17.43,
"tests/integration/test_scheduler_numeric_id_test.py": 25.51,
"tests/integration/test_scheduler_pool.py": 24.22,
"tests/integration/test_scheduler_rapid_cancellation.py": 24.01,
"tests/integration/test_scheduler_recursive_timeout.py": 22.94,
"tests/integration/test_scheduler_removed_item_race.py": 23.07,
"tests/integration/test_scheduler_self_keyed.py": 18.43,
"tests/integration/test_scheduler_simultaneous_callbacks.py": 21.99,
"tests/integration/test_scheduler_string_test.py": 17.27,
"tests/integration/test_script_array_params.py": 4.59,
"tests/integration/test_script_delay_params.py": 22.46,
"tests/integration/test_script_queued.py": 25.24,
"tests/integration/test_script_queued_idle_loop.py": 5.04,
"tests/integration/test_script_wait_on_boot.py": 21.77,
"tests/integration/test_sdl_headless_screenshot.py": 19.23,
"tests/integration/test_select_stringref_trigger.py": 19.31,
"tests/integration/test_sensor_filters_delta.py": 25.92,
"tests/integration/test_sensor_filters_ring_buffer.py": 22.39,
"tests/integration/test_sensor_filters_sliding_window.py": 57.93,
"tests/integration/test_sensor_filters_value_list.py": 20.32,
"tests/integration/test_sensor_timeout_filter.py": 25.35,
"tests/integration/test_snapshot_display.py": 19.7,
"tests/integration/test_socket_wake_gate_tcp.py": 14.5,
"tests/integration/test_status_flags.py": 33.83,
"tests/integration/test_strftime_to.py": 17.64,
"tests/integration/test_syslog.py": 24.49,
"tests/integration/test_template_alarm_control_panel_many_sensors.py": 24.81,
"tests/integration/test_template_climate_basic.py": 15.28,
"tests/integration/test_template_climate_custom_modes.py": 25.07,
"tests/integration/test_template_climate_nonoptimistic.py": 24.25,
"tests/integration/test_template_climate_on_control_ordering.py": 24.09,
"tests/integration/test_template_climate_publish_all_fields.py": 17.78,
"tests/integration/test_template_climate_sensor_push.py": 17.42,
"tests/integration/test_template_climate_set_actions.py": 23.63,
"tests/integration/test_template_climate_two_point_temperature.py": 25.19,
"tests/integration/test_template_text_save.py": 17.88,
"tests/integration/test_text_command.py": 22.71,
"tests/integration/test_text_sensor_raw_state.py": 25.17,
"tests/integration/test_uart_mock_ld2410.py": 58.15,
"tests/integration/test_uart_mock_ld2412.py": 61.14,
"tests/integration/test_uart_mock_ld2420.py": 33.87,
"tests/integration/test_uart_mock_ld2450.py": 26.06,
"tests/integration/test_uart_mock_modbus.py": 391.79,
"tests/integration/test_udp.py": 7.38,
"tests/integration/test_use_address_runtime.py": 24.09,
"tests/integration/test_valve_control_action.py": 23.22,
"tests/integration/test_varint_five_byte_device_id.py": 17.93,
"tests/integration/test_wait_until_mid_loop_timing.py": 22.26,
"tests/integration/test_wait_until_on_boot.py": 17.46,
"tests/integration/test_wait_until_ordering.py": 11.89,
"tests/integration/test_wait_until_reentrant_restart.py": 22.88,
"tests/integration/test_wake_loop_forces_phase_b.py": 16.6,
"tests/integration/test_water_heater_template.py": 19.66
}
@@ -24,7 +24,6 @@ from .types import (
RunCompiledFunction,
)
pytestmark = pytest.mark.usefixtures("isolated_preferences")
NEW_KEY = PROVISIONING_PSK
@@ -41,15 +41,6 @@ async def _poll_until_exists(path: Path) -> None:
await asyncio.sleep(0.05)
@pytest.fixture(autouse=True)
def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> Path:
"""Keep host preferences per-test so this test never touches the real
~/.esphome/prefs and never races other tests over ESPHOME_PREFDIR."""
prefdir = tmp_path / "prefs"
monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir))
return prefdir / f"{DEVICE_NAME}.prefs"
@pytest.mark.asyncio
async def test_host_preferences_suspend_resume(
yaml_config: str,
@@ -58,7 +49,7 @@ async def test_host_preferences_suspend_resume(
isolated_preferences: Path,
) -> None:
"""Test that a running syncer flushes, a suspended one doesn't, and resume restores flushing."""
pref_file = isolated_preferences
pref_file = isolated_preferences / f"{DEVICE_NAME}.prefs"
loop = asyncio.get_running_loop()
saved_in_memory = loop.create_future()
@@ -11,14 +11,6 @@ from .state_utils import InitialStateHelper, require_entity
from .types import APIClientConnectedFactory, RunCompiledFunction
@pytest.fixture(autouse=True)
def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
"""Keep host preferences per-test so RESTORE_AND_ON never loads a stale value left
behind by a previous run (host preferences otherwise persist to ~/.esphome/prefs,
keyed only by device name)."""
monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs"))
@pytest.mark.asyncio
async def test_light_initial_state(
yaml_config: str,
+16 -7
View File
@@ -173,6 +173,7 @@ async def test_uart_mock_modbus_no_threshold(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_server_injected")
@pytest.mark.asyncio
async def test_uart_mock_modbus_server(
yaml_config: str,
@@ -203,6 +204,7 @@ async def test_uart_mock_modbus_server(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_server_injected")
@pytest.mark.asyncio
async def test_uart_mock_modbus_server_read_write(
yaml_config: str,
@@ -231,8 +233,8 @@ async def test_uart_mock_modbus_server_read_write(
"rw_write_1": 4660, # 0x1234 written to reg 0x0001
"rw_read_1": 4660, # reg 0x0001 reads back the just-written value
"rw_read_2": 170, # 0x00AA read from reg 0x0002 in the same request
"rw_write_3": 22136, # 0x5678 written to reg 0x0003
"rw_read_3": 22136, # reg 0x0003 reads back the just-written value
"rw_write_3": 22136, # 0x5678 written to reg 0x0006
"rw_read_3": 22136, # reg 0x0006 reads back the just-written value
}
)
@@ -241,7 +243,8 @@ async def test_uart_mock_modbus_server_read_write(
api_client_connected() as client,
):
await tracker.setup_and_start_scenario(client)
await tracker.await_all(futures)
# The FC 0x17 injections fire last, behind four earlier 100ms delays
await tracker.await_all(futures, timeout=4.0)
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@@ -296,6 +299,7 @@ async def test_uart_mock_modbus_server_read_write_invalid(
)
@pytest.mark.shared_yaml("uart_mock_modbus_mesh")
@pytest.mark.asyncio
async def test_uart_mock_modbus_server_controller(
yaml_config: str,
@@ -485,6 +489,7 @@ async def test_uart_mock_modbus_server_controller_bits(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_mesh")
@pytest.mark.asyncio
async def test_uart_mock_modbus_server_controller_multiple(
yaml_config: str,
@@ -495,7 +500,7 @@ async def test_uart_mock_modbus_server_controller_multiple(
line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback()
expected_values = {"reg_u_word": 919, "reg_u_word_2": 929}
expected_values = {"multi_reg_a": 919, "multi_reg_b": 929}
tracker = SensorTracker(list(expected_values.keys()))
futures = tracker.expect_all(expected_values)
@@ -706,6 +711,7 @@ async def test_uart_mock_modbus_shared_address(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_loopback")
@pytest.mark.asyncio
async def test_uart_mock_modbus_custom_pdu(
yaml_config: str,
@@ -932,6 +938,7 @@ async def test_uart_mock_modbus_broadcast_write(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_mesh")
@pytest.mark.asyncio
async def test_uart_mock_modbus_client_read_write(
yaml_config: str,
@@ -947,9 +954,7 @@ async def test_uart_mock_modbus_client_read_write(
"""
line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback()
tracker = SensorTracker(
["srv_write_1", "srv_read_1", "client_read_0", "client_read_1"]
)
tracker = SensorTracker(["srv_write_1", "client_read_0", "client_read_1"])
futures = tracker.expect_all(
{
"srv_write_1": 4660, # server wrote 0x1234 to reg 0x0001
@@ -967,6 +972,7 @@ async def test_uart_mock_modbus_client_read_write(
_assert_no_modbus_errors(error_log_lines, warning_log_lines)
@pytest.mark.shared_yaml("uart_mock_modbus_loopback")
@pytest.mark.asyncio
async def test_uart_mock_modbus_register_offset(
yaml_config: str,
@@ -1022,6 +1028,7 @@ async def test_uart_mock_modbus_register_offset(
)
@pytest.mark.shared_yaml("uart_mock_modbus_loopback")
@pytest.mark.asyncio
async def test_uart_mock_modbus_lambda_write(
yaml_config: str,
@@ -1058,6 +1065,7 @@ async def test_uart_mock_modbus_lambda_write(
await tracker.await_change(wrote_30, "reg_30", timeout=4.0)
@pytest.mark.shared_yaml("uart_mock_modbus_loopback")
@pytest.mark.asyncio
async def test_uart_mock_modbus_lambda_invert(
yaml_config: str,
@@ -1113,6 +1121,7 @@ async def test_uart_mock_modbus_lambda_invert(
)
@pytest.mark.shared_yaml("uart_mock_modbus_loopback")
@pytest.mark.asyncio
async def test_uart_mock_modbus_deprecated_write_buffer(
yaml_config: str,
-92
View File
@@ -78,17 +78,6 @@ def mock_esp32_platformio_components_to_test() -> Generator[Mock, None, None]:
yield mock
@pytest.fixture
def mock_esp8266_native_components_to_test() -> Generator[Mock, None, None]:
"""Mock esp8266_native_components_to_test from determine_jobs.
main() drives both the ``esp8266_native`` boolean output and the
``esp8266_native_components`` CSV from this one function.
"""
with patch.object(determine_jobs, "esp8266_native_components_to_test") as mock:
yield mock
@pytest.fixture
def mock_determine_cpp_unit_tests() -> Generator[Mock, None, None]:
"""Mock determine_cpp_unit_tests from helpers."""
@@ -117,7 +106,6 @@ def clear_determine_jobs_caches() -> None:
"""Clear all cached functions before each test."""
determine_jobs._is_clang_tidy_full_scan.cache_clear()
determine_jobs._component_has_tests.cache_clear()
determine_jobs._cached_components_closure.cache_clear()
def test_main_all_tests_should_run(
@@ -128,7 +116,6 @@ def test_main_all_tests_should_run(
mock_should_run_import_time: Mock,
mock_should_run_device_builder: Mock,
mock_esp32_platformio_components_to_test: Mock,
mock_esp8266_native_components_to_test: Mock,
mock_changed_files: Mock,
mock_determine_cpp_unit_tests: Mock,
capsys: pytest.CaptureFixture[str],
@@ -145,7 +132,6 @@ def test_main_all_tests_should_run(
mock_should_run_import_time.return_value = True
mock_should_run_device_builder.return_value = True
mock_esp32_platformio_components_to_test.return_value = ["api", "esp32"]
mock_esp8266_native_components_to_test.return_value = ["api", "logger"]
mock_determine_cpp_unit_tests.return_value = (False, ["wifi", "api", "sensor"])
# Mock changed_files to return non-component files (to avoid memory impact)
@@ -222,8 +208,6 @@ def test_main_all_tests_should_run(
assert output["device_builder"] is True
assert output["esp32_platformio"] is True
assert output["esp32_platformio_components"] == "api,esp32"
assert output["esp8266_native"] is True
assert output["esp8266_native_components"] == "api,logger"
assert output["changed_components"] == ["wifi", "api", "sensor"]
# changed_components_with_tests will only include components that actually have test files
assert "changed_components_with_tests" in output
@@ -260,7 +244,6 @@ def test_main_no_tests_should_run(
mock_should_run_import_time: Mock,
mock_should_run_device_builder: Mock,
mock_esp32_platformio_components_to_test: Mock,
mock_esp8266_native_components_to_test: Mock,
mock_changed_files: Mock,
mock_determine_cpp_unit_tests: Mock,
capsys: pytest.CaptureFixture[str],
@@ -277,7 +260,6 @@ def test_main_no_tests_should_run(
mock_should_run_import_time.return_value = False
mock_should_run_device_builder.return_value = False
mock_esp32_platformio_components_to_test.return_value = []
mock_esp8266_native_components_to_test.return_value = []
mock_determine_cpp_unit_tests.return_value = (False, [])
# Mock changed_files to return no component files
@@ -320,8 +302,6 @@ def test_main_no_tests_should_run(
assert output["device_builder"] is False
assert output["esp32_platformio"] is False
assert output["esp32_platformio_components"] == ""
assert output["esp8266_native"] is False
assert output["esp8266_native_components"] == ""
assert output["changed_components"] == []
assert output["changed_components_with_tests"] == []
assert output["component_test_count"] == 0
@@ -3171,78 +3151,6 @@ def test_memory_impact_elf_layouts_are_found(tmp_path: Path) -> None:
assert find_elf_path(build_path) == elf, f"{platform} ELF not found"
@pytest.mark.parametrize(
"changed",
[
"esphome/arduino8266/framework.py",
"esphome/build_gen/arduino8266.py",
"esphome/components/esp8266/build_surgery.py",
# Shared modules the native build depends on
"esphome/build_helpers/idedata.py",
"esphome/platformio/library.py",
# Top-level esphome/*.py modules the backend imports directly
"esphome/framework_helpers.py",
"esphome/writer.py",
# esp8266/__init__.py imports copy_ccache_script from it
"esphome/platformio/toolchain.py",
# The composite cache action must not ship unexercised
".github/actions/cache-arduino8266/action.yml",
],
)
def test_esp8266_native_components_full_list_on_infra_change(changed: str) -> None:
"""Native-ESP8266 infrastructure changes run the full test list."""
with (
patch.object(determine_jobs, "changed_files", return_value=[changed]),
patch.object(
determine_jobs,
"get_components_with_dependencies",
return_value=["wifi"],
),
):
result = determine_jobs.esp8266_native_components_to_test()
assert result == sorted(determine_jobs.ESP8266_NATIVE_TEST_COMPONENTS)
@pytest.mark.parametrize(
("changed_files", "dependency_closure", "expected"),
[
# Tested component changed -- narrow to the intersection.
(
["esphome/components/mqtt/mqtt_client.cpp"],
["mqtt", "json"],
["mqtt"],
),
# Components outside the test set return an empty list (job skipped).
(
["esphome/components/wifi/wifi_component.cpp"],
["wifi", "network"],
[],
),
# espidf infrastructure is not an esp8266-native trigger; the
# native backend depends on esphome/build_helpers/ instead.
(["esphome/build_gen/espidf.py"], [], []),
(["esphome/espidf/toolchain.py"], [], []),
(["README.md"], [], []),
],
)
def test_esp8266_native_components_to_test_narrowing(
changed_files: list[str],
dependency_closure: list[str],
expected: list[str],
) -> None:
"""Component changes narrow the native-ESP8266 test list."""
with (
patch.object(determine_jobs, "changed_files", return_value=changed_files),
patch.object(
determine_jobs,
"get_components_with_dependencies",
return_value=dependency_closure,
),
):
result = determine_jobs.esp8266_native_components_to_test()
assert result == expected
def test_compute_integration_test_buckets_no_durations_full_fanout() -> None:
"""Without recorded durations the fan-out stays at the maximum."""
files = [f"tests/integration/test_{i:03d}.py" for i in range(15)]
+28
View File
@@ -2122,6 +2122,34 @@ def test_get_cpp_changed_components_independent_of_cwd(
) == ["time"]
def test_fixture_map_includes_shared_yaml_markers() -> None:
"""Fixtures named only by shared_yaml markers must map to their test file."""
helpers.get_fixture_to_test_files.cache_clear()
mapping = helpers.get_fixture_to_test_files()
for fixture in (
"uart_mock_modbus_loopback",
"uart_mock_modbus_mesh",
"uart_mock_modbus_server_injected",
):
assert mapping[fixture] == frozenset(
{"tests/integration/test_uart_mock_modbus.py"}
)
def test_no_orphan_integration_fixtures() -> None:
"""Every fixture must reach CI test selection; an orphan selects nothing."""
helpers.get_fixture_to_test_files.cache_clear()
mapping = helpers.get_fixture_to_test_files()
fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve()
fixtures = list(fixtures_dir.glob("*.yaml"))
assert fixtures, f"no fixtures found under {fixtures_dir}"
# cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead
orphans = [
f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping
]
assert not orphans, f"fixtures invisible to CI test selection: {orphans}"
def test_lpt_partition_balances_skewed_weights() -> None:
"""Heavy items spread across groups instead of clustering."""
items = [f"i{n}" for n in range(6)]
@@ -236,98 +236,3 @@ def test_run_grouped_test_closes_group_when_subprocess_raises(
)
assert "::endgroup::" in capsys.readouterr().out
def test_components_empty_match_fails_with_flag(
capsys: pytest.CaptureFixture[str],
) -> None:
"""Under --fail-on-no-tests, a real component filtered to a platform
with no matching test file must not pass CI as a green zero-component
compile."""
rc = tbc.test_components(
["logger"],
"zz-none",
"compile",
False,
enable_grouping=False,
fail_on_no_tests=True,
)
assert rc == 1
assert "No tests ran for requested pattern(s): logger" in (capsys.readouterr().out)
def test_components_component_with_no_base_file_fails_with_flag(
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A component whose fixture matches the platform but whose platform has
no base file builds nothing; under the flag that silent zero fails by
component name instead of hiding behind other components."""
monkeypatch.setattr(tbc, "get_platform_base_files", lambda base_dir: {})
rc = tbc.test_components(
["logger"],
"esp8266-ard",
"compile",
False,
enable_grouping=False,
fail_on_no_tests=True,
)
assert rc == 1
assert "No tests ran for requested pattern(s): logger" in (capsys.readouterr().out)
def test_components_blank_list_fails_with_flag(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A fully blank component list must not slide into the baseline
fallback and exit green under the flag."""
rc = tbc.test_components(
[""], "esp8266-ard", "compile", False, fail_on_no_tests=True
)
assert rc == 1
assert "blank component list" in capsys.readouterr().out
def test_components_wildcard_no_match_fails_with_flag(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A wildcard matching nothing must not degrade to the synthetic
baseline build and exit green under the flag."""
rc = tbc.test_components(
["zz_no_such*"],
"esp8266-ard",
"compile",
False,
enable_grouping=False,
fail_on_no_tests=True,
)
assert rc == 1
assert "No components found matching" in capsys.readouterr().out
def test_components_empty_match_tolerated_without_flag() -> None:
"""The esp32-ard smoke leg deliberately builds only the subset with a
matching fixture; without the flag an empty match stays green."""
assert (
tbc.test_components(
["logger"], "zz-none", "compile", False, enable_grouping=False
)
== 0
)
def test_components_unknown_component_fails_with_flag(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A renamed smoke-test component must shrink coverage loudly, not fall
into the reference-baseline build."""
rc = tbc.test_components(
["no_such_component_xyz"],
"esp8266-ard",
"compile",
False,
enable_grouping=False,
fail_on_no_tests=True,
)
assert rc == 1
assert "No components found matching" in capsys.readouterr().out
File diff suppressed because it is too large Load Diff
@@ -10,7 +10,6 @@ from unittest.mock import patch
import pytest
from esphome.build_helpers import ccache
from esphome.core import CORE
def test_resolve_opt_out() -> None:
@@ -121,17 +120,3 @@ def test_parse_enable_env_spelling_tables(
"""cv.boolean's spelling tables plus the 1/0 env convention."""
monkeypatch.setenv("ESPHOME_CCACHE_ENABLE", raw)
assert ccache.parse_enable_env("ESPHOME_CCACHE_ENABLE") is expected
def test_effective_ccache_basedir_prefers_user_value(tmp_path: Path) -> None:
CORE.build_path = tmp_path
# Drive-qualified on Windows: "/custom/base" is not absolute there
base = "C:\\custom\\base" if os.name == "nt" else "/custom/base"
with patch.dict(os.environ, {"CCACHE_BASEDIR": base}, clear=True):
assert ccache.effective_ccache_basedir() == base
with patch.dict(os.environ, {}, clear=True):
assert ccache.effective_ccache_basedir() == str(tmp_path.resolve())
# Degenerate values would strip substrings ccache never rewrites
for bad in ("", "/", "a/b"):
with patch.dict(os.environ, {"CCACHE_BASEDIR": bad}, clear=True):
assert ccache.effective_ccache_basedir() == str(tmp_path.resolve())
@@ -85,87 +85,6 @@ def test_parse_entry_resolves_relative_includes() -> None:
assert all(Path(inc).is_absolute() for inc in includes)
def test_parse_entry_resolves_force_include_path(tmp_path: Path) -> None:
"""The pch -include is emitted relative to the build dir; idedata must
resolve it so cached flags work from any cwd."""
(tmp_path / "esphome_pch.h").write_text("")
entry = _entry(
str(tmp_path),
f"{tmp_path}/src/esphome/x.cpp",
"g++ -include esphome_pch.h -c x.cpp",
)
_, _, _, cxx_flags = idedata.parse_entry(entry)
idx = cxx_flags.index("-include")
resolved = cxx_flags[idx + 1]
assert Path(resolved).is_absolute()
assert resolved == str(tmp_path / "esphome_pch.h").replace("\\", "/")
def test_parse_entry_resolves_joined_force_include(tmp_path: Path) -> None:
"""The joined -includefoo.h spelling takes the same resolve path."""
(tmp_path / "esphome_pch.h").write_text("")
entry = _entry(
str(tmp_path),
f"{tmp_path}/src/esphome/x.cpp",
"g++ -includeesphome_pch.h -c x.cpp",
)
_, _, _, cxx_flags = idedata.parse_entry(entry)
resolved = cxx_flags[cxx_flags.index("-include") + 1]
assert resolved == str(tmp_path / "esphome_pch.h").replace("\\", "/")
def test_parse_entry_keeps_search_chain_force_include(tmp_path: Path) -> None:
"""-include names resolved via the -I chain (libretiny's Arduino.h) must
not be re-anchored to a nonexistent build-dir path."""
entry = _entry(
str(tmp_path),
f"{tmp_path}/src/esphome/x.cpp",
"g++ -include Arduino.h -c x.cpp",
)
_, _, _, cxx_flags = idedata.parse_entry(entry)
assert cxx_flags[cxx_flags.index("-include") + 1] == "Arduino.h"
def test_parse_entry_warns_on_vanished_force_include(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A build-dir force-include deleted by clean_build must leave a trail;
a name resolvable via the -I chain must not warn."""
inc = tmp_path / "inc"
inc.mkdir()
(inc / "Arduino.h").write_text("")
entry = _entry(
str(tmp_path),
f"{tmp_path}/src/esphome/x.cpp",
f"g++ -I{inc} -include Arduino.h -include esphome_pch.h -c x.cpp",
)
_, _, _, cxx_flags = idedata.parse_entry(entry)
assert "Arduino.h" in cxx_flags
assert "esphome_pch.h" in caplog.text
assert "Arduino.h" not in caplog.text
def test_parse_entry_drops_trailing_force_include(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
entry = _entry(
str(tmp_path), f"{tmp_path}/src/esphome/x.cpp", "g++ -c x.cpp -include"
)
_, _, _, cxx_flags = idedata.parse_entry(entry)
assert "-include" not in cxx_flags
assert "no argument" in caplog.text
def test_parse_entry_skips_dependency_flags() -> None:
"""Dependency-generation flags (and their args) are dropped."""
entry = _entry(
-218
View File
@@ -1,218 +0,0 @@
"""Tests for esphome.build_helpers.pch."""
from __future__ import annotations
import os
from pathlib import Path
from unittest.mock import patch
import pytest
from esphome.build_helpers import pch
def _write(src_dir: Path, name: str, content: str) -> None:
path = src_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
@pytest.mark.parametrize(
("value", "expected"),
[
(None, True),
("1", True),
("0", False),
("false", False),
("", False),
],
)
def test_pch_enabled(value: str | None, expected: bool) -> None:
env = {} if value is None else {"ESPHOME_PCH_ENABLE": value}
with patch.dict(os.environ, env, clear=True):
assert pch.pch_enabled() is expected
def test_ccache_pch_env_empty_until_emitted() -> None:
"""No sloppiness relaxation for a build that skipped the pch."""
with patch.dict(os.environ, {}, clear=True):
assert pch.ccache_pch_env() == {}
def test_ccache_pch_env_enabled() -> None:
pch.mark_pch_emitted()
with patch.dict(os.environ, {}, clear=True):
env = pch.ccache_pch_env()
assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
assert env["CCACHE_PCH_EXTSUM"] == "true"
def test_ccache_pch_env_disabled() -> None:
with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True):
assert pch.ccache_pch_env() == {}
def test_ccache_pch_env_token_check_is_membership_not_substring(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A token merely containing ours must not suppress the union."""
pch.mark_pch_emitted()
with patch.dict(os.environ, {"CCACHE_SLOPPINESS": "pch_defines_extra"}, clear=True):
env = pch.ccache_pch_env()
assert env["CCACHE_SLOPPINESS"] == "pch_defines_extra,pch_defines,time_macros"
def test_ccache_pch_env_unions_user_sloppiness(
caplog: pytest.LogCaptureFixture,
) -> None:
"""Without pch_defines/time_macros ccache declines every pch-consuming
compile, so missing tokens are unioned onto the user's value."""
pch.mark_pch_emitted()
with patch.dict(os.environ, {"CCACHE_SLOPPINESS": "locale"}, clear=True):
env = pch.ccache_pch_env()
assert env["CCACHE_SLOPPINESS"] == "locale,pch_defines,time_macros"
assert env["CCACHE_PCH_EXTSUM"] == "true"
assert "Adding pch_defines,time_macros" in caplog.text
caplog.clear()
with patch.dict(
os.environ, {"CCACHE_SLOPPINESS": "pch_defines,time_macros"}, clear=True
):
env = pch.ccache_pch_env()
assert "CCACHE_SLOPPINESS" not in env
assert not caplog.records
def test_pch_header_text_preserves_order() -> None:
text = pch.pch_header_text(["b.h", "a.h"])
assert text == '#include "b.h"\n#include "a.h"\n'
def test_include_closure_resolves_relative_and_root(tmp_path: Path) -> None:
"""Sibling includes resolve against the includer's directory first,
full paths against the src root; unresolvable names end the walk."""
_write(tmp_path, "esphome/components/x/a.h", '#include "b.h"\n')
_write(
tmp_path,
"esphome/components/x/b.h",
'#include "esphome/core/deep.h"\n#include <system.h>\n#include "missing.h"\n',
)
_write(tmp_path, "esphome/core/deep.h", "")
closure = pch._include_closure(tmp_path, ["esphome/components/x/a.h"])
assert sorted(closure) == [
"esphome/components/x/a.h",
"esphome/components/x/b.h",
"esphome/core/deep.h",
]
def test_include_closure_handles_cycles(tmp_path: Path) -> None:
_write(tmp_path, "a.h", '#include "b.h"\n')
_write(tmp_path, "b.h", '#include "a.h"\n')
assert sorted(pch._include_closure(tmp_path, ["a.h"])) == ["a.h", "b.h"]
def test_include_closure_blocks_parent_escape(tmp_path: Path) -> None:
_write(tmp_path / "src", "a.h", '#include "../outside.h"\n')
(tmp_path / "outside.h").write_text("")
assert sorted(pch._include_closure(tmp_path / "src", ["a.h"])) == ["a.h"]
def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None:
"""A transitive header edit or an extra-identity change must change the
digest; unrelated files must not."""
_write(tmp_path, "root.h", '#include "nested.h"\n')
_write(tmp_path, "nested.h", "int a;\n")
_write(tmp_path, "unrelated.h", "int u;\n")
base = pch.pch_checksum(tmp_path, ["root.h"], ["id"])
assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
assert base != pch.pch_checksum(tmp_path, ["root.h"], ["other-id"])
_write(tmp_path, "unrelated.h", "int changed;\n")
assert base == pch.pch_checksum(tmp_path, ["root.h"], ["id"])
_write(tmp_path, "nested.h", "int b;\n")
assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"])
@pytest.mark.skipif(
os.name == "nt" or os.geteuid() == 0, reason="chmod is ineffective here"
)
def test_include_closure_fails_closed_on_unreadable(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A marker would truncate the transitive walk; the OSError propagates
so callers compile without a pch."""
_write(tmp_path, "a.h", '#include "locked.h"\n')
locked = tmp_path / "locked.h"
locked.write_text("")
locked.chmod(0)
try:
with pytest.raises(OSError):
pch._include_closure(tmp_path, ["a.h"])
finally:
locked.chmod(0o644)
assert "Could not read locked.h" in caplog.text
def test_pch_extra_scripts_gated(monkeypatch: pytest.MonkeyPatch) -> None:
with patch.dict(os.environ, {}, clear=True):
assert pch.pch_extra_scripts() == ["post:pch.py"]
monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0")
assert pch.pch_extra_scripts() == []
def test_include_closure_raises_when_identity_unknown(
caplog: pytest.LogCaptureFixture,
) -> None:
"""An unreadable header propagates; callers compile without a pch."""
class _BadFile:
def stat(self): # noqa: ANN202 -- regular-file mode only
import os
import stat as stat_mod
return os.stat_result((stat_mod.S_IFREG | 0o644,) + (0,) * 9)
def read_bytes(self) -> bytes:
raise OSError("read failed")
class _FakeSrcDir:
def __truediv__(self, rel: str) -> _BadFile:
return _BadFile()
with pytest.raises(OSError, match="read failed"):
pch._include_closure(_FakeSrcDir(), ["a.h"])
assert "Could not read a.h" in caplog.text
def test_include_closure_survives_non_utf8_include_name(tmp_path: Path) -> None:
"""A non-UTF-8 quoted include must not abort the build; it simply does
not resolve and ends the walk."""
(tmp_path / "a.h").write_bytes(b'#include "bad\xff.h"\n#include "b.h"\n')
(tmp_path / "b.h").write_text("")
closure = pch._include_closure(tmp_path, ["a.h"])
assert set(closure) == {"a.h", "b.h"}
def test_pch_checksum_survives_surrogate_extra(tmp_path: Path) -> None:
"""Install paths from non-UTF-8 filesystems carry surrogates; hashing
them must not raise past the caller's identity-unknown guard."""
assert pch.pch_checksum(tmp_path, [], ["/opt/bad\udcff/framework"])
def test_include_closure_walks_angle_includes_under_src(tmp_path: Path) -> None:
"""An angle include resolving under src/ must enter the digest; one
that does not simply ends the walk."""
_write(tmp_path, "a.h", "#include <local.h>\n#include <Arduino.h>\n")
(tmp_path / "local.h").write_text("")
closure = pch._include_closure(tmp_path, ["a.h"])
assert set(closure) == {"a.h", "local.h"}
def test_ccache_pch_env_warns_on_falsy_extsum(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A user CCACHE_PCH_EXTSUM=false makes ccache hash the .gch bytes."""
pch.mark_pch_emitted()
with patch.dict(os.environ, {"CCACHE_PCH_EXTSUM": "false"}, clear=True):
env = pch.ccache_pch_env()
assert "CCACHE_PCH_EXTSUM" not in env
assert "disables pch caching" in caplog.text
@@ -1,193 +0,0 @@
"""Tests for the native (non-PlatformIO) toolchain config validation."""
from __future__ import annotations
from collections.abc import Generator
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from esphome.components import esp8266
from esphome.components.esp8266 import (
ARDUINO_FRAMEWORK_SCHEMA,
_resolve_toolchain,
_validate_native_toolchain,
)
import esphome.config_validation as cv
from esphome.const import (
CONF_BOARD,
CONF_FRAMEWORK,
CONF_PLATFORM_VERSION,
CONF_SOURCE,
CONF_TOOLCHAIN,
CONF_VERSION,
Toolchain,
)
from esphome.core import CORE, EsphomeError
from esphome.types import ConfigType
@pytest.fixture(autouse=True)
def _arduino_toolchain() -> Generator[None]:
# The suite-wide reset_core fixture clears CORE.toolchain after each test
CORE.toolchain = Toolchain.ARDUINO
esp8266._DECODE_WARNED_AT.clear()
yield
esp8266._DECODE_WARNED_AT.clear()
def _config(
board: str = "nodemcuv2",
version: str = "3.1.2",
source: str | None = None,
platform_version: str | None = None,
) -> ConfigType:
framework: dict[str, str] = {CONF_VERSION: version}
if source is not None:
framework[CONF_SOURCE] = source
if platform_version is not None:
framework[CONF_PLATFORM_VERSION] = platform_version
# The real schema fills the source/platform_version defaults, so these
# tests validate against what config validation actually emits
return {
CONF_FRAMEWORK: ARDUINO_FRAMEWORK_SCHEMA(framework),
CONF_BOARD: board,
}
def test_valid_config_passes() -> None:
config = _config()
assert _validate_native_toolchain(config) is config
def test_platformio_toolchain_skips_checks() -> None:
# 3.0.2 is pio-legal (>= the global 3.0.0 floor) but below the native
# toolchain's own 3.1.1 floor; the bogus board only the native path checks
CORE.toolchain = Toolchain.PLATFORMIO
config = _config(board="not_a_board", version="3.0.2")
assert _validate_native_toolchain(config) is config
def test_version_below_floor_rejected() -> None:
# 3.1.0 has no registry package, so the native floor is 3.1.1
with pytest.raises(cv.Invalid, match="3.1.1 or newer"):
_validate_native_toolchain(_config(version="3.1.0"))
def test_version_at_floor_accepted() -> None:
_validate_native_toolchain(_config(version="3.1.1"))
def test_custom_platform_version_warns_and_is_dropped(
caplog: pytest.LogCaptureFixture,
) -> None:
config = _config(platform_version="platformio/espressif8266@4.0.1")
_validate_native_toolchain(config)
assert "'platform_version' is ignored" in caplog.text
assert CONF_PLATFORM_VERSION not in config[CONF_FRAMEWORK]
def test_default_platform_version_does_not_warn(
caplog: pytest.LogCaptureFixture,
) -> None:
config = _config()
_validate_native_toolchain(config)
assert "'platform_version' is ignored" not in caplog.text
assert CONF_PLATFORM_VERSION not in config[CONF_FRAMEWORK]
def test_custom_source_rejected() -> None:
with pytest.raises(cv.Invalid, match="custom framework source"):
_validate_native_toolchain(
_config(source="https://github.com/esp8266/Arduino.git")
)
def test_unsupported_board_rejected() -> None:
with pytest.raises(cv.Invalid, match="not supported by"):
_validate_native_toolchain(_config(board="not_a_board"))
def test_yaml_toolchain_key_resolves() -> None:
"""The documented `toolchain: arduino` YAML key selects the native path."""
CORE.toolchain = None
_resolve_toolchain({CONF_TOOLCHAIN: Toolchain.ARDUINO})
assert CORE.toolchain == Toolchain.ARDUINO
assert CORE.using_toolchain_arduino
def test_yaml_toolchain_key_defaults_to_platformio() -> None:
CORE.toolchain = None
_resolve_toolchain({})
assert CORE.toolchain == Toolchain.PLATFORMIO
def test_decode_pc_native_missing_tools_warns_once(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A stack dump of many addresses produces one missing-tool warning."""
with (
patch(
"esphome.arduino8266.toolchain.get_addr2line_path",
return_value=tmp_path / "missing-addr2line",
),
patch(
"esphome.arduino8266.toolchain.get_elf_path",
return_value=tmp_path / "missing.elf",
),
):
esp8266._decode_pc({}, "40201234")
esp8266._decode_pc({}, "40201238")
assert caplog.text.count("Cannot decode crash addresses") == 1
def test_decode_pc_platformio_missing_tools_warns_once(
caplog: pytest.LogCaptureFixture,
) -> None:
"""The PlatformIO branch reports a missing addr2line/ELF at the same
warning level as the native one; raw undecoded addresses with no
stated reason are undiagnosable at default log level."""
CORE.toolchain = Toolchain.PLATFORMIO
idedata = SimpleNamespace(addr2line_path=None, firmware_elf_path=None)
with patch("esphome.platformio.toolchain.get_idedata", return_value=idedata):
esp8266._decode_pc({}, "40201234")
esp8266._decode_pc({}, "40201238")
assert caplog.text.count("Cannot decode crash addresses") == 1
def test_resolve_toolchain_rejects_unsupported() -> None:
"""ESP8266 rejects a CLI toolchain it cannot serve, like every platform."""
CORE.toolchain = Toolchain.SDK_NRF
with pytest.raises(cv.Invalid, match="Unsupported toolchain 'sdk-nrf'"):
_resolve_toolchain({})
def test_run_compile_platformio_falls_through() -> None:
"""Under toolchain: platformio the hook returns False without touching
the native backend; this is what keeps existing users on PlatformIO."""
CORE.toolchain = Toolchain.PLATFORMIO
with patch("esphome.arduino8266.toolchain.run_compile") as mock_native:
assert esp8266.run_compile(SimpleNamespace(), {}) is False
mock_native.assert_not_called()
def test_run_compile_arduino_failure_raises() -> None:
"""A non-zero native build fails by name instead of returning success."""
CORE.verbose = False
with (
patch("esphome.arduino8266.toolchain.run_compile", return_value=1),
pytest.raises(EsphomeError, match="native build failed"),
):
esp8266.run_compile(SimpleNamespace(), {})
def test_copy_files_native_skips_platformio_scripts(tmp_path: Path) -> None:
"""The native build writes no PlatformIO extra scripts."""
CORE.build_path = tmp_path
esp8266.copy_files()
assert list(tmp_path.iterdir()) == []
+9
View File
@@ -1429,3 +1429,12 @@ async def test_add_platformio_options_native_arduino(
assert "board_build.ldscript is ignored" in caplog.text
assert "'arduino' toolchain" in caplog.text
assert "upload_speed" not in caplog.text
def test_esp8266_rejects_unsupported_cli_toolchain() -> None:
"""Until the native backend lands, ESP8266 serves only PlatformIO."""
from esphome.components.esp8266 import CONFIG_SCHEMA
CORE.toolchain = Toolchain.ARDUINO
with pytest.raises(cv.Invalid, match="Unsupported toolchain 'arduino'"):
CONFIG_SCHEMA({"board": "nodemcuv2"})
@@ -1,65 +0,0 @@
"""Run the native-toolchain serial-upload path and report heavy modules.
Executed as a subprocess by test_lazy_imports.py: heavy module names come
in on argv, the ones found in sys.modules afterwards go out on stdout.
``upload_using_esptool`` dispatches native toolchains through a
toolchain-keyed table; if someone routes it back through the platform
component packages (esp32 or esp8266), this reports the leak.
"""
import os
from pathlib import Path
import sys
import tempfile
from unittest.mock import patch
from _leak_report import print_leaked_modules
from esphome.__main__ import upload_using_esptool
from esphome.const import (
CONF_ESPHOME,
KEY_CORE,
KEY_ESP32,
KEY_TARGET_PLATFORM,
KEY_VARIANT,
Toolchain,
)
from esphome.core import CORE
# An ambient ESPHOME_USE_SUBPROCESS would route past the patched
# run_external_command into run_external_process and confuse the checks.
os.environ.pop("ESPHOME_USE_SUBPROCESS", None)
config = {CONF_ESPHOME: {"platformio_options": {}}}
with tempfile.TemporaryDirectory() as build_dir:
CORE.name = "leaktest"
CORE.build_path = build_dir
for platform, toolchain, backend in (
("esp8266", Toolchain.ARDUINO, "esphome.arduino8266.toolchain"),
("esp32", Toolchain.ESP_IDF, "esphome.espidf.toolchain"),
):
CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform}
if platform == "esp32":
CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32S3"}
CORE.toolchain = toolchain
import importlib
image = importlib.import_module(backend).get_factory_firmware_path()
image.parent.mkdir(parents=True, exist_ok=True)
image.write_bytes(b"\x00")
with patch("esphome.__main__.run_external_command", return_value=0) as mock_run:
rc = upload_using_esptool(config, "/dev/ttyUSB0", None, None)
# Fail loudly if the upload path stopped doing its work; otherwise
# an empty leak list could just mean nothing ran.
if rc != 0:
sys.exit(f"upload_using_esptool({platform}) returned {rc}")
cmd = list(mock_run.call_args[0][1:])
if str(image) not in [str(Path(c)) for c in cmd]:
sys.exit(f"native factory image did not reach esptool: {cmd}")
print_leaked_modules()
@@ -1,657 +0,0 @@
"""Tests for esphome.arduino8266.toolchain (the ninja build driver)."""
from __future__ import annotations
import os
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from esphome.arduino8266 import framework, toolchain
from esphome.build_helpers.pch import mark_pch_emitted
import esphome.config_validation as cv
from esphome.const import (
CONF_COMPILE_PROCESS_LIMIT,
CONF_ESPHOME,
KEY_CORE,
KEY_FRAMEWORK_VERSION,
)
from esphome.core import CORE, EsphomeError
_SIZE_OUTPUT = """\
firmware.elf :
section size addr
.data 1924 1073643520
.noinit 56 1073645444
.text 496 1074790400
.irom0.text 342804 1075843088
.text1 27489 1074790896
.rodata 2588 1073645504
.bss 26504 1073648096
Total 401861
"""
@pytest.fixture(autouse=True)
def _setup_core(tmp_path: Path) -> None:
CORE.name = "test8266"
CORE.config_path = tmp_path / "test8266.yaml"
CORE.build_path = tmp_path
CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 1, 2)}
# run_compile verifies the produced artifacts; give every test a build
# that "produced" them (tests for the guard delete them again). The
# manifest comes first: artifacts must not be older than build.ninja.
build_dir = CORE.relative_pioenvs_path("test8266")
build_dir.mkdir(parents=True, exist_ok=True)
(build_dir / "build.ninja").write_text("# manifest")
for artifact in (
"firmware.elf",
"firmware.bin",
"firmware.factory.bin",
"firmware.ota.bin",
):
(build_dir / artifact).write_bytes(b"")
def _paths(tmp_path: Path) -> framework.InstalledPaths:
return framework.InstalledPaths(
framework=tmp_path / "framework",
toolchain=tmp_path / "toolchain",
ninja=tmp_path / "ninja",
)
def test_path_getters(tmp_path: Path) -> None:
assert toolchain.get_build_dir() == CORE.relative_pioenvs_path("test8266")
assert toolchain.get_elf_path().name == "firmware.elf"
# The framework accessor owns the layout and the Windows suffix
suffix = ".exe" if os.name == "nt" else ""
assert toolchain.get_addr2line_path().name == f"xtensa-lx106-elf-addr2line{suffix}"
assert toolchain.get_objdump_path().name == f"xtensa-lx106-elf-objdump{suffix}"
assert toolchain.get_readelf_path().name == f"xtensa-lx106-elf-readelf{suffix}"
def test_run_compile_build_failure(tmp_path: Path) -> None:
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project"),
patch.object(
toolchain.subprocess, "run", return_value=MagicMock(returncode=2)
) as mock_run,
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=True) == 2
cmd = mock_run.call_args[0][0]
assert "-v" in cmd
# The compile database is generated before the build runs, so a failed
# build cannot leave a stale database behind.
mock_compdb.assert_called_once()
def test_run_compile_success(tmp_path: Path) -> None:
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
# An unchanged manifest is what makes the -n probe run
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
) as mock_run,
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary") as mock_size,
patch.object(toolchain, "get_idedata") as mock_idedata,
):
rc = toolchain.run_compile(
{CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 4}}, verbose=False
)
assert rc == 0
# The -n probe runs first, then the real build (cwd, no -C banner)
ninja_calls = [c for c in mock_run.call_args_list if "ninja" in str(c[0][0][0])]
assert "-n" in ninja_calls[0][0][0]
# Explicit targets: a manifest missing them fails as "unknown target"
assert ninja_calls[0][0][0][-1] == "firmware.ota.bin"
cmd = ninja_calls[1][0][0]
assert cmd[-4:] == ["-j", "4", "firmware.factory.bin", "firmware.ota.bin"]
assert "-C" not in cmd
assert ninja_calls[1][1]["cwd"] is not None
mock_compdb.assert_called_once()
mock_size.assert_called_once()
mock_idedata.assert_called_once()
def test_run_compile_noop_skips_the_build_spawn(tmp_path: Path) -> None:
"""A no-op rebuild stays quiet: the -n probe answers "no work to do"
and the real ninja spawn (and its banner) never happens."""
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
# An unchanged manifest is what makes the -n probe run
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(
returncode=0, stdout="ninja: no work to do.\n", stderr=""
),
) as mock_run,
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
assert rc == 0
ninja_calls = [c for c in mock_run.call_args_list if "ninja" in str(c[0][0][0])]
assert len(ninja_calls) == 1
assert "-n" in ninja_calls[0][0][0]
# Explicit targets: a manifest missing them fails as "unknown target"
assert ninja_calls[0][0][0][-1] == "firmware.ota.bin"
def test_run_compile_regenerates_stale_compdb(tmp_path: Path) -> None:
"""An interrupted run can leave build.ninja newer than the compile DB;
mere existence must not skip regeneration."""
build_dir = toolchain.get_build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
(build_dir / "build.ninja").write_text("")
compdb = build_dir / "compile_commands.json"
compdb.write_text("[]")
os.utime(compdb, ((build_dir / "build.ninja").stat().st_mtime - 5,) * 2)
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(
returncode=0, stdout="ninja: no work to do.\n", stderr=""
),
),
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
mock_compdb.assert_called_once()
def test_run_compile_surfaces_probe_diagnostics(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A load-time ninja diagnostic (a generator bug signal) reaches the
user even when the no-work branch skips the real spawn."""
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
# An unchanged manifest is what makes the -n probe run
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(
returncode=0,
stdout="ninja: no work to do.\n",
stderr="ninja: warning: multiple rules generate x\n",
),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
assert rc == 0
assert "multiple rules generate x" in caplog.text
def test_run_compile_missing_artifact_fails(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A zero ninja exit that produced no firmware must not be a green
build (size summary and idedata only warn)."""
(toolchain.get_build_dir() / "firmware.elf").unlink()
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project"),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary") as mock_size,
patch.object(toolchain, "get_idedata"),
):
rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
assert rc == 1
assert "Build produced no" in caplog.text
mock_size.assert_not_called()
def test_run_compile_warns_when_idedata_fails(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A failed idedata generation right after a successful build is visible,
not deferred to a misleading error in a later command."""
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project"),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata", return_value=None),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert "No idedata was generated for this build" in caplog.text
def test_write_compile_commands(tmp_path: Path) -> None:
build_dir = tmp_path / "build"
build_dir.mkdir()
entries = '[{"file": "a.cpp", "command": "cc"}]\n'
with patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=entries),
):
toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {})
assert (build_dir / "compile_commands.json").read_text() == entries
@pytest.mark.parametrize(
("stdout", "match"),
[
("[]\n", "empty compile database"),
# A parse failure names its cause, not the rule-name story
("not json", "unparsable compile database.*not json"),
],
)
def test_write_compile_commands_bad_db_raises(
tmp_path: Path, stdout: str, match: str
) -> None:
"""An empty or unparsable compile database fails the build with its
actual cause and drops any stale database."""
build_dir = tmp_path / "build"
build_dir.mkdir()
(build_dir / "compile_commands.json").write_text("[stale]")
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=stdout),
),
pytest.raises(EsphomeError, match=match),
):
toolchain._write_compile_commands(tmp_path / "ninja", build_dir, {})
assert not (build_dir / "compile_commands.json").exists()
def test_write_compile_commands_failure_removes_stale_db(tmp_path: Path) -> None:
"""A failed compdb run must not leave a stale database behind."""
stale = tmp_path / "compile_commands.json"
stale.write_text("[]")
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=1, stderr="boom"),
),
pytest.raises(EsphomeError, match="compile_commands"),
):
toolchain._write_compile_commands(tmp_path / "ninja", tmp_path, {})
assert not stale.exists()
def test_parse_app_size(tmp_path: Path) -> None:
ld = tmp_path / "eagle.flash.4m.ld"
ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0xfeff0\n}\n")
with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld):
assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) == 0xFEFF0
ld.write_text("MEMORY { }\n")
with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld):
assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None
# A zero-length segment is bad data, not a budget; warn and drop it
ld.write_text("MEMORY\n{\n irom0_0_seg : org = 0x40201010, len = 0x0\n}\n")
with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld):
assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None
with patch(
"esphome.build_gen.arduino8266.get_flash_ld_path",
return_value=tmp_path / "missing.ld",
):
assert toolchain._parse_app_size(tmp_path, _paths(tmp_path)) is None
def test_print_size_summary(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=_SIZE_OUTPUT),
),
patch.object(toolchain, "_parse_app_size", return_value=1044464),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
out = capsys.readouterr().out
# Exact PlatformIO shape so script/ci_memory_impact_extract.py can parse it
assert "RAM: [==== ] 37.9% (used 31016 bytes from 81920 bytes)" in out
assert "Flash: [==== ] 35.9% (used 375301 bytes from 1044464 bytes)" in out
def test_print_size_summary_missing_size_tool_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A missing size binary degrades to a warning; the firmware already
linked and must not be discarded."""
with patch.object(
toolchain.subprocess, "run", side_effect=FileNotFoundError("no size")
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
assert "Could not summarize firmware size" in caplog.text
def test_print_size_summary_no_app_size(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=_SIZE_OUTPUT),
),
patch.object(toolchain, "_parse_app_size", return_value=None),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
out = capsys.readouterr().out
# Both lines are skipped together: a RAM line without Flash would skew
# CI's memory-impact sums across builds
assert out == ""
def test_print_size_summary_size_tool_failure(
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
caplog: pytest.LogCaptureFixture,
) -> None:
with patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=1, stdout="", stderr="bad elf"),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
assert capsys.readouterr().out == ""
assert "Could not summarize firmware size" in caplog.text
def test_get_idedata_delegates(tmp_path: Path) -> None:
with (
patch(
"esphome.build_helpers.idedata.load_or_build_idedata",
return_value={"cc_path": "x"},
) as mock_load,
patch.object(toolchain, "resolve_ccache_path", return_value="/cc/ccache"),
):
assert toolchain.get_idedata() == {"cc_path": "x"}
compile_commands, elf, cache = mock_load.call_args[0]
assert compile_commands.name == "compile_commands.json"
assert elf.name == "firmware.elf"
assert cache.name == "test8266.arduino.json"
# The exact configured launcher string is passed for compile DB parsing
# (resolve_ccache_path returns a str, untouched on every platform)
assert mock_load.call_args.kwargs["launcher"] == "/cc/ccache"
def test_get_idedata_no_ccache(tmp_path: Path) -> None:
with (
patch(
"esphome.build_helpers.idedata.load_or_build_idedata", return_value={}
) as mock_load,
patch.object(toolchain, "resolve_ccache_path", return_value=None),
):
toolchain.get_idedata()
assert mock_load.call_args.kwargs["launcher"] is None
def test_run_compile_skips_compdb_when_ninja_unchanged(tmp_path: Path) -> None:
"""An unchanged build.ninja means the compile DB is already current."""
build_dir = toolchain.get_build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
# write_project (stubbed below) always leaves a build.ninja behind
(build_dir / "build.ninja").write_text("# manifest")
def run(regenerate_expected: bool) -> None:
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands") as mock_compdb,
patch.object(toolchain, "_print_size_summary"),
patch.object(toolchain, "get_idedata"),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert mock_compdb.called == regenerate_expected
# Missing compile DB: regenerated even though build.ninja is unchanged
run(regenerate_expected=True)
# Present compile DB + unchanged build.ninja: skipped
(build_dir / "compile_commands.json").write_text("[]")
run(regenerate_expected=False)
def test_print_size_summary_unparsable_section(
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
caplog: pytest.LogCaptureFixture,
) -> None:
"""A totals-relevant section that fails to parse must not produce a
confident wrong number; an irrelevant one only warns."""
bad = _SIZE_OUTPUT.replace(".bss 26504", ".bss abc")
with patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=bad),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
assert capsys.readouterr().out == ""
assert "Unparsable size output" in caplog.text
caplog.clear()
harmless = _SIZE_OUTPUT + ".broken abc 0\n"
with (
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=harmless),
),
patch.object(toolchain, "_parse_app_size", return_value=1044464),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
assert "RAM:" in capsys.readouterr().out
assert "Unparsable size output" in caplog.text
def test_print_size_summary_missing_section_skips_summary(
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
caplog: pytest.LogCaptureFixture,
) -> None:
"""A totals section absent from the output must not default to zero."""
without_bss = "\n".join(
line for line in _SIZE_OUTPUT.splitlines() if ".bss" not in line
)
with patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout=without_bss),
):
toolchain._print_size_summary(tmp_path, _paths(tmp_path))
assert capsys.readouterr().out == ""
assert "missing section(s) .bss" in caplog.text
def test_warn_ignored_platformio_options(caplog: pytest.LogCaptureFixture) -> None:
"""Component-added options the native build drops are warned by name;
the honored ones (lib_ignore, f_cpu, ldscript, build_src_flags,
flash_mode) stay quiet."""
CORE.platformio_options = {
"board_build.ldscript": "eagle.flash.4m2m.ld",
"board_build.f_cpu": "160000000L",
"board_build.filesystem": "littlefs",
"board_build.flash_mode": "dio",
"build_src_flags": "-include throw_stubs.h",
"lib_ignore": ["Updater"],
"upload_speed": "460800",
}
toolchain._warn_ignored_platformio_options()
assert "platformio_options->board_build.filesystem is ignored" in caplog.text
assert "native 'arduino' toolchain" in caplog.text
assert "board_build.ldscript is ignored" not in caplog.text
assert "board_build.f_cpu is ignored" not in caplog.text
assert "lib_ignore" not in caplog.text
assert "build_src_flags" not in caplog.text
assert "flash_mode" not in caplog.text
# Component-added upload_speed never gets read under the native
# toolchain, so it must warn
assert "upload_speed" in caplog.text
def test_run_compile_idedata_error_does_not_fail_build(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""An unusable compile DB after a successful build warns, never fails."""
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project"),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary"),
patch.object(
toolchain,
"get_idedata",
side_effect=EsphomeError("compile database is unusable"),
),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert "Could not generate idedata: compile database is unusable" in caplog.text
def test_run_compile_skipped_size_summary_names_consequence(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A skipped RAM/Flash summary warns about the missing metric; the
per-cause warnings alone are invisible to CI harnesses."""
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project"),
patch.object(
toolchain.subprocess,
"run",
return_value=MagicMock(returncode=0, stdout="", stderr=""),
),
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary", return_value=False),
patch.object(toolchain, "get_idedata", return_value=None),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
assert "Firmware size summary unavailable for this build" in caplog.text
def test_parse_app_size_non_utf8_ld_warns(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A corrupt (non-UTF-8) linker script degrades to the same warning as
an unreadable one, never a traceback after a successful link."""
paths = _paths(tmp_path)
ld = tmp_path / "corrupt.ld"
ld.write_bytes(b"\xff\xfe not utf8")
with patch("esphome.build_gen.arduino8266.get_flash_ld_path", return_value=ld):
assert toolchain._parse_app_size(tmp_path, paths) is None
assert "Cannot read linker script" in caplog.text
def test_run_compile_failed_probe_runs_full_build(tmp_path: Path) -> None:
"""A failing -n probe (e.g. unknown target from a defective manifest)
falls through to the real build so the error prints attributably."""
probe = MagicMock(returncode=1, stdout="", stderr="")
ok = MagicMock(returncode=0, stdout="", stderr="")
def fake_run(cmd, *args, **kwargs):
return probe if "-n" in cmd else ok
with (
patch.object(framework, "check_and_install", return_value=_paths(tmp_path)),
patch.object(framework, "get_build_env", return_value={}),
patch("esphome.build_gen.arduino8266.write_project", return_value=False),
patch.object(toolchain.subprocess, "run", side_effect=fake_run) as mock_run,
patch.object(toolchain, "_write_compile_commands"),
patch.object(toolchain, "_print_size_summary", return_value=True),
patch.object(toolchain, "get_idedata", return_value={}),
):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0
# The real build ran after the failed probe
assert any(
"firmware.ota.bin" in c[0][0] and "-n" not in c[0][0]
for c in mock_run.call_args_list
)
def test_get_idedata_accepts_preresolved_ccache() -> None:
"""run_compile threads its resolved ccache through; the probe must not
run again."""
with (
patch(
"esphome.build_helpers.idedata.load_or_build_idedata",
return_value={"ok": True},
) as mock_build,
patch.object(toolchain, "resolve_ccache_path") as mock_resolve,
):
assert toolchain.get_idedata("/usr/bin/ccache") == {"ok": True}
mock_resolve.assert_not_called()
assert mock_build.call_args.kwargs["launcher"] == "/usr/bin/ccache"
def test_ccache_env_includes_pch_settings() -> None:
"""The native build exports the ccache settings the pch needs."""
mark_pch_emitted()
with patch.dict(os.environ, {}, clear=True):
env = framework.ccache_env("/usr/bin/ccache")
assert env["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
assert env["CCACHE_PCH_EXTSUM"] == "true"
def test_ccache_env_pch_disabled() -> None:
with patch.dict(os.environ, {"ESPHOME_PCH_ENABLE": "0"}, clear=True):
env = framework.ccache_env("/usr/bin/ccache")
assert "CCACHE_SLOPPINESS" not in env
assert "CCACHE_PCH_EXTSUM" not in env
def test_ccache_env_respects_user_sloppiness() -> None:
mark_pch_emitted()
with patch.dict(os.environ, {"CCACHE_SLOPPINESS": "locale"}, clear=True):
env = framework.ccache_env("/usr/bin/ccache")
# The user's tokens survive; the ones the pch needs are unioned on
assert env["CCACHE_SLOPPINESS"] == "locale,pch_defines,time_macros"
assert env["CCACHE_PCH_EXTSUM"] == "true"
-21
View File
@@ -175,27 +175,6 @@ def test_esptool_upload_fast_path_does_not_import_heavy_modules(
)
def test_native_upload_fast_path_does_not_import_heavy_modules(
fixture_path: Path,
probe_env: dict[str, str],
) -> None:
"""The native-toolchain serial upload dispatches through a
toolchain-keyed backend table; resolving the flash image must not drag
in the esp32 or esp8266 component package or the validation stack.
"""
leaked = _leaked_from_fixture(
fixture_path,
probe_env,
"native_upload_fast_path.py",
extra=("esphome.components.esp8266",),
)
assert not leaked, (
f"native upload_using_esptool pulls in heavy modules: {leaked}. "
"The upload fast path skips validation; importing a platform "
"component package executes its codegen module by design."
)
def test_api_client_does_not_import_heavy_modules() -> None:
"""``esphome.api_client`` is on the logs fast path and must stay light.
+3 -262
View File
@@ -13,7 +13,7 @@ import sys
import time
from types import SimpleNamespace
from typing import Any, Self
from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from pytest import CaptureFixture
@@ -25,7 +25,6 @@ from esphome.__main__ import (
Purpose,
_get_configured_xtal_freq,
_make_crystal_freq_callback,
_native_toolchain_module,
_redact_with_legacy_fallback,
_resolve_network_devices,
_should_subscribe_states,
@@ -7133,7 +7132,7 @@ def test_command_run_rp2040_bootsel_redetects_serial_port() -> None:
def test_command_idedata_esp_idf_prints_json(capsys: CaptureFixture) -> None:
"""Under the native ESP-IDF toolchain, idedata is emitted as JSON."""
setup_core(platform=PLATFORM_ESP32)
setup_core()
CORE.toolchain = Toolchain.ESP_IDF
data = {"cxx_path": "g++", "prog_path": "/build/firmware.elf"}
@@ -7147,7 +7146,7 @@ def test_command_idedata_esp_idf_prints_json(capsys: CaptureFixture) -> None:
def test_command_idedata_esp_idf_no_build_errors() -> None:
"""Under ESP-IDF, a missing build (no idedata) returns an error, not a crash."""
setup_core(platform=PLATFORM_ESP32)
setup_core()
CORE.toolchain = Toolchain.ESP_IDF
with patch("esphome.espidf.toolchain.get_idedata", return_value=None):
@@ -7321,201 +7320,6 @@ def test_warn_source_tree_mismatch_falls_back_when_stat_fails(
assert not caplog.text
def test_upload_using_esptool_arduino_toolchain(
tmp_path: Path,
mock_run_external_command_main: Mock,
) -> None:
"""The native ESP8266 Arduino toolchain flashes its factory image at
0x0, resolved from the toolchain-keyed backend table (deliberately not
the platform hook: that import would break the upload fast path)."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test")
CORE.toolchain = Toolchain.ARDUINO
from esphome.arduino8266 import toolchain as native
factory = native.get_factory_firmware_path()
factory.parent.mkdir(parents=True, exist_ok=True)
factory.touch()
config = {CONF_ESPHOME: {"platformio_options": {}}}
result = upload_using_esptool(config, "/dev/ttyUSB0", None, None)
assert result == 0
cmd_list = list(mock_run_external_command_main.call_args[0][1:])
firmware_offset_idx = cmd_list.index("write-flash") + 4
assert cmd_list[firmware_offset_idx] == "0x0"
assert cmd_list[firmware_offset_idx + 1] == str(factory)
@pytest.mark.parametrize(
("toolchain", "pio_project_written"),
[
# The native toolchain generates its project at compile time, so
# write_cpp_file must not write a platformio.ini; the default
# toolchain writes the PlatformIO project files.
(Toolchain.ARDUINO, False),
(None, True),
],
)
def test_write_cpp_file_project_generation_follows_toolchain(
tmp_path: Path, toolchain: Toolchain | None, pio_project_written: bool
) -> None:
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test")
CORE.toolchain = toolchain
with (
patch("esphome.writer.write_cpp") as mock_write_cpp,
patch("esphome.build_gen.platformio.write_project") as mock_pio_project,
patch.object(
type(CORE), "cpp_main_section", new_callable=PropertyMock
) as mock_section,
):
mock_section.return_value = ""
assert main.write_cpp_file() == 0
mock_write_cpp.assert_called_once()
assert mock_pio_project.called is pio_project_written
def test_command_idedata_arduino_prints_json(
tmp_path: Path, capsys: CaptureFixture
) -> None:
"""Under the native ESP8266 Arduino toolchain, idedata is emitted as JSON."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path)
CORE.toolchain = Toolchain.ARDUINO
data = {"cxx_path": "g++", "prog_path": "/build/firmware.elf"}
with patch(
"esphome.arduino8266.toolchain.get_idedata", return_value=data
) as mock_get:
result = command_idedata(MagicMock(), CORE.config)
assert result == 0
mock_get.assert_called_once_with()
assert json.loads(capsys.readouterr().out) == data
def test_command_idedata_arduino_no_build_errors(tmp_path: Path) -> None:
"""A missing native build (no idedata) returns an error, not a crash."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path)
CORE.toolchain = Toolchain.ARDUINO
with patch("esphome.arduino8266.toolchain.get_idedata", return_value=None):
result = command_idedata(MagicMock(), CORE.config)
assert result == 1
@pytest.mark.parametrize(
("platform", "toolchain", "module"),
[
(PLATFORM_ESP8266, Toolchain.ARDUINO, "esphome.arduino8266.toolchain"),
(PLATFORM_ESP32, Toolchain.ESP_IDF, "esphome.espidf.toolchain"),
],
)
def test_command_analyze_memory_native_toolchains(
tmp_path: Path,
mock_write_cpp: Mock,
mock_compile_program: Mock,
mock_get_esphome_components: Mock,
mock_memory_analyzer_cli: Mock,
mock_ram_strings_analyzer: Mock,
platform: str,
toolchain: Toolchain,
module: str,
) -> None:
"""analyze-memory uses the native toolchain's binutils instead of
falling into the PlatformIO branch."""
setup_core(platform=platform, tmp_path=tmp_path, name="test_device")
CORE.toolchain = toolchain
config = {CONF_ESPHOME: {CONF_NAME: "test_device"}}
# The tools must exist: a missing binutils now fails by name instead of
# silently falling back to host tools
objdump = tmp_path / "objdump"
readelf = tmp_path / "readelf"
objdump.write_text("")
readelf.write_text("")
# The ELF must exist too: the analyzer swallows tool failures, so a
# missing image would report zeroes with exit 0
firmware_elf = tmp_path / "firmware.elf"
firmware_elf.write_text("")
with (
patch(f"{module}.get_objdump_path", return_value=objdump),
patch(f"{module}.get_readelf_path", return_value=readelf),
patch(f"{module}.get_elf_path", return_value=firmware_elf),
):
result = command_analyze_memory(MockArgs(), config)
assert result == 0
mock_memory_analyzer_cli.assert_called_once_with(
str(firmware_elf),
str(objdump),
str(readelf),
set(),
idedata=None,
)
def test_command_analyze_memory_native_missing_elf_fails(
tmp_path: Path,
mock_write_cpp: Mock,
mock_compile_program: Mock,
mock_get_esphome_components: Mock,
mock_memory_analyzer_cli: Mock,
mock_ram_strings_analyzer: Mock,
caplog: pytest.LogCaptureFixture,
) -> None:
"""A missing firmware.elf fails by name instead of an exit-0 zeroed
report."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device")
CORE.toolchain = Toolchain.ARDUINO
config = {CONF_ESPHOME: {CONF_NAME: "test_device"}}
objdump = tmp_path / "objdump"
readelf = tmp_path / "readelf"
objdump.write_text("")
readelf.write_text("")
module = "esphome.arduino8266.toolchain"
with (
patch(f"{module}.get_objdump_path", return_value=objdump),
patch(f"{module}.get_readelf_path", return_value=readelf),
patch(f"{module}.get_elf_path", return_value=tmp_path / "missing.elf"),
):
result = command_analyze_memory(MockArgs(), config)
assert result == 1
assert "compile the configuration first" in caplog.text
mock_memory_analyzer_cli.assert_not_called()
def test_command_analyze_memory_missing_binutils_fails_by_name(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A truncated toolchain install fails naming the missing tool instead
of silently analyzing with host binutils."""
setup_core(platform="esp8266", tmp_path=tmp_path, name="test_device")
CORE.toolchain = Toolchain.ARDUINO
config = {CONF_ESPHOME: {CONF_NAME: "test_device"}}
module = "esphome.arduino8266.toolchain"
with (
patch(f"{module}.get_objdump_path", return_value=tmp_path / "missing-objdump"),
patch(f"{module}.get_readelf_path", return_value=tmp_path / "readelf"),
patch("esphome.__main__.write_cpp", return_value=0),
patch("esphome.__main__.compile_program", return_value=0),
):
assert command_analyze_memory(MockArgs(), config) == 1
assert "missing-objdump" in caplog.text
assert "toolchain install may be incomplete" in caplog.text
def test_command_idedata_incompatible_toolchain(tmp_path: Path) -> None:
"""A non-native, non-platformio toolchain errors out cleanly."""
setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path)
CORE.toolchain = Toolchain.SDK_NRF
assert command_idedata(MagicMock(), CORE.config) == 1
@pytest.mark.parametrize(
"error",
[
@@ -7592,46 +7396,6 @@ def test_compile_program_espidf_idedata_none_warns(
assert "No idedata was generated" in caplog.text
def test_native_toolchain_table_serves_every_native_toolchain() -> None:
"""Every member of NATIVE_TOOLCHAINS has a backend entry; a gap would
surface as a targeted EsphomeError on the one affected config, and this
pin keeps the table from drifting when a toolchain is added."""
from esphome.const import NATIVE_TOOLCHAINS
assert {tc for _, tc in main._NATIVE_TOOLCHAIN_MODULES} == set(NATIVE_TOOLCHAINS)
def test_native_toolchain_module_missing_backend_raises(tmp_path: Path) -> None:
"""A native toolchain missing from the backend table is a bug and must
fail, not silently degrade to the PlatformIO path."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device")
CORE.toolchain = Toolchain.ARDUINO
with (
patch.dict(main._NATIVE_TOOLCHAIN_MODULES, clear=True),
pytest.raises(EsphomeError, match="no native build backend"),
):
_native_toolchain_module()
def test_command_analyze_memory_unsupported_toolchain(
tmp_path: Path,
mock_write_cpp: Mock,
mock_compile_program: Mock,
caplog: pytest.LogCaptureFixture,
) -> None:
"""A hook-less non-PlatformIO toolchain is refused by name, never routed
into the PlatformIO branch."""
setup_core(platform=PLATFORM_NRF52, tmp_path=tmp_path, name="test_device")
CORE.toolchain = Toolchain.SDK_NRF
mock_write_cpp.return_value = 0
mock_compile_program.return_value = 0
result = command_analyze_memory(MockArgs(), {CONF_ESPHOME: {CONF_NAME: "t"}})
assert result == 1
assert "analyze-memory is not supported" in caplog.text
def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None:
"""An explicit --toolchain must run the per-platform validators, so the
upload/logs fast path becomes a cache miss."""
@@ -7647,29 +7411,6 @@ def test_cli_toolchain_skips_the_validated_config_cache(tmp_path: Path) -> None:
mock_read.assert_called_once()
def test_upload_using_esptool_native_missing_firmware_raises(
tmp_path: Path,
) -> None:
"""A stale or absent firmware.bin fails by name instead of flashing air."""
setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test")
CORE.toolchain = Toolchain.ARDUINO
with pytest.raises(EsphomeError, match="compile the configuration first"):
upload_using_esptool(
{CONF_ESPHOME: {"platformio_options": {}}}, "/dev/ttyUSB0", None, None
)
def test_compile_program_unclaimed_native_toolchain_raises(
tmp_path: Path,
) -> None:
"""A resolved native toolchain no platform backend claims must fail,
never fall through to the PlatformIO project path."""
setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test_device")
CORE.toolchain = Toolchain.ARDUINO # esp32 has no arduino-native backend
with pytest.raises(EsphomeError, match="no platform backend claimed"):
compile_program(MockArgs(), {})
def test_cli_toolchain_still_refreshes_the_validated_config_cache(
tmp_path: Path,
) -> None:
@@ -1,460 +0,0 @@
"""Tests for esphome/platformio/pch.py.script against a fake SCons env."""
from __future__ import annotations
import os
from pathlib import Path
import stat
from unittest.mock import patch
import pytest
from esphome.platformio import toolchain
pytestmark = pytest.mark.skipif(
os.name == "nt", reason="the fake compiler is a POSIX shell script"
)
_SCRIPT = Path(toolchain.__file__).parent / "pch.py.script"
class _FakePlatform:
packages = {"framework-x": {}, "toolchain-y": {}}
def get_package_version(self, name: str) -> str:
if name == "toolchain-y":
raise KeyError(name)
return "1.2.3"
def get_package(self, name: str) -> object | None:
return None
class _BrokenPlatform(_FakePlatform):
def get_package_version(self, name: str) -> str:
raise RuntimeError("manifest parse error")
class _UnresolvedPlatform(_FakePlatform):
"""KeyError from a package that IS installed: unresolved identity."""
def get_package_version(self, name: str) -> str:
raise KeyError(name)
def get_package(self, name: str) -> object:
return object()
class _FakeSConsEnv(dict):
"""Just enough of a SCons construction environment for pch.py."""
def __init__(
self,
proj_dir: Path,
src_dir: Path,
cxx: str,
flags: list[str],
platform_cls: type[_FakePlatform] = _FakePlatform,
):
super().__init__(ENV={})
self._subst = {
"$PROJECT_DIR": str(proj_dir),
"$PROJECT_SRC_DIR": str(src_dir),
"$CXX": cxx,
}
self._flags = flags
self._platform_cls = platform_cls
self.prepended: list[str] = []
def subst(self, expr: str) -> str: # noqa: N802
return self._subst[expr]
def subst_list(self, expr: str) -> list[list[str]]: # noqa: N802
return [self._flags]
def PioPlatform(self) -> _FakePlatform: # noqa: N802
return self._platform_cls()
def Prepend(self, CXXFLAGS: list[str]) -> None: # noqa: N802, N803
self.prepended = CXXFLAGS
def _fake_cxx(
tmp_path: Path,
fail: bool = False,
fail_msg: str | None = None,
reject_pch: bool = False,
probe_exit: int = 0,
) -> Path:
"""A compiler stand-in that records its argv and writes the -o target.
With reject_pch it builds the .gch fine but, like GCC 10 on macOS arm64,
warns on any consuming compile that the .gch cannot be loaded; probe_exit
sets the exit code of non-header compiles (the load probe).
"""
cxx = tmp_path / "fake-gxx"
body = (
'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n'
)
if fail:
body += f"echo {fail_msg or 'boom'} >&2\nexit 1\n"
else:
# Only the c++-header compile has a -o; the load probe has none
body += 'out=""; prev=""; mf=0; dep=0\nfor a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; [ "$a" = "-MF" ] && mf=1; case "$a" in -M|-MM|-MD|-MMD) dep=1;; esac; done\n'
# Real cc1plus rejects -MF without a dependency flag
body += 'if [ "$mf" = 1 ] && [ "$dep" = 0 ]; then echo "cc1plus: error: to generate dependencies you must specify either \x27-M\x27 or \x27-MM\x27" >&2; exit 1; fi\n'
body += '[ -n "$out" ] && echo gch > "$out"\n'
if reject_pch:
body += 'case " $* " in *c++-header*) ;; *) echo "warning: esphome_pch.h.gch: had text segment at different address" >&2;; esac\n'
body += f'case " $* " in *c++-header*) exit 0;; *) exit {probe_exit};; esac\n'
cxx.write_text("#!/bin/sh\n" + body)
cxx.chmod(cxx.stat().st_mode | stat.S_IEXEC)
return cxx
def _run_script(
tmp_path: Path,
flags: list[str] | None = None,
fail: bool = False,
fail_msg: str | None = None,
reject_pch: bool = False,
probe_exit: int = 0,
missing_cxx: bool = False,
env_vars: dict[str, str] | None = None,
name: str = "dev",
platform_cls: type[_FakePlatform] = _FakePlatform,
) -> _FakeSConsEnv:
proj = tmp_path / name
src = proj / "src"
(src / "esphome" / "core").mkdir(parents=True, exist_ok=True)
(src / "esphome" / "core" / "defines.h").write_text("#define USE_X\n")
cxx = _fake_cxx(
tmp_path,
fail=fail,
fail_msg=fail_msg,
reject_pch=reject_pch,
probe_exit=probe_exit,
)
if missing_cxx:
cxx = tmp_path / "no-such-gxx"
args = (proj, src, str(cxx), flags or ["-DX=1"], platform_cls)
# Distinct objects: the -include flags must land on projenv only
global_env = _FakeSConsEnv(*args)
projenv = _FakeSConsEnv(*args)
projenv.global_env = global_env
source = _SCRIPT.read_text()
with patch.dict(os.environ, env_vars or {}, clear=True):
exec( # noqa: S102
compile(source, "pch.py", "exec"),
{"Import": lambda *_names: None, "env": global_env, "projenv": projenv},
)
return projenv
def test_pch_script_builds_and_prepends_relative_include(tmp_path: Path) -> None:
scons_env = _run_script(tmp_path)
proj = tmp_path / "dev"
assert (proj / "esphome_pch.h").read_text().endswith('"esphome/core/defines.h"\n')
assert (proj / "esphome_pch.h.gch").is_file()
assert len((proj / "esphome_pch.h.gch.sum").read_text().strip()) == 64
# Relative include: an absolute path would poison ccache keys
assert scons_env.prepended == [
"-Winvalid-pch",
"-Wno-error=invalid-pch",
"-include",
"esphome_pch.h",
]
# In production projenv["ENV"] aliases os.environ; only the -include
# flags are genuinely scoped to projenv (src compiles)
assert scons_env["ENV"]["CCACHE_SLOPPINESS"] == "pch_defines,time_macros"
assert scons_env["ENV"]["CCACHE_PCH_EXTSUM"] == "true"
assert scons_env.global_env.prepended == []
def test_pch_script_preserves_spaced_flag_elements(tmp_path: Path) -> None:
"""One SCons element stays one compiler argv; -include pairs are
stripped from the .gch compile."""
spaced = tmp_path / "My Configs"
spaced.mkdir()
(tmp_path / "dev" / "src").mkdir(parents=True, exist_ok=True)
(tmp_path / "dev" / "src" / "other.h").write_text("")
flags = ['-DUSB_PRODUCT=\\"Pico 2W\\"', "-I", str(spaced), "-include", "other.h"]
_run_script(tmp_path, flags=flags)
calls = (tmp_path / "fake-gxx.argv").read_text().split("---call---\n")
gch_call = next(c for c in calls if "c++-header" in c).splitlines()
assert '-DUSB_PRODUCT="Pico 2W"' in gch_call
assert str(spaced) in gch_call
assert "-include" not in gch_call
# The stripped src-resolvable -include is folded into the prefix header
pch = (tmp_path / "dev" / "esphome_pch.h").read_text()
assert pch.splitlines()[0] == '#include "other.h"'
def test_pch_script_folds_joined_force_include_spelling(tmp_path: Path) -> None:
"""-includefoo.h folds like the separated form, matching the native path."""
(tmp_path / "dev" / "src").mkdir(parents=True, exist_ok=True)
(tmp_path / "dev" / "src" / "other.h").write_text("")
_run_script(tmp_path, flags=["-DX=1", "-includeother.h"])
pch = (tmp_path / "dev" / "esphome_pch.h").read_text()
assert pch.splitlines()[0] == '#include "other.h"'
def test_pch_script_leaves_absolute_force_includes_unfolded(
tmp_path: Path,
) -> None:
"""An absolute -include resolves through src_dir / name; it must still
stay consumer-only or the host path enters the .sum."""
outside = tmp_path / "outside.h"
outside.write_text("")
_run_script(tmp_path, flags=["-DX=1", "-include", str(outside)])
pch = (tmp_path / "dev" / "esphome_pch.h").read_text()
assert "outside.h" not in pch
def test_pch_script_leaves_non_src_force_includes_unfolded(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""A user -include outside src/ must not enter the prefix header:
consumers keep their own copy, so folding an unguarded header would
include it twice."""
_run_script(tmp_path, flags=["-DX=1", "-include", "user_extra.h"])
pch = (tmp_path / "dev" / "esphome_pch.h").read_text()
assert "user_extra.h" not in pch
assert pch.splitlines()[-1] == '#include "esphome/core/defines.h"'
assert "not precompiling non-src force-includes" in capsys.readouterr().out
def test_pch_script_sum_is_device_independent(tmp_path: Path) -> None:
"""Regression: identical configs in different dirs share cache keys."""
sums = []
for name in ("dev_a", "dev_b"):
proj = tmp_path / name
_run_script(
tmp_path,
flags=["-DX=1", "-I", str(proj / "include")],
env_vars={"CCACHE_BASEDIR": str(proj)},
name=name,
)
sums.append((proj / "esphome_pch.h.gch.sum").read_text())
(tmp_path / "fake-gxx").unlink()
(tmp_path / "fake-gxx.argv").unlink(missing_ok=True)
assert sums[0] == sums[1]
def test_pch_script_transient_compiler_failure_does_not_latch(
tmp_path: Path,
) -> None:
"""ENOSPC-style failures clear on their own; no .failed marker."""
scons_env = _run_script(tmp_path, fail=True, fail_msg="No space left on device")
proj = tmp_path / "dev"
assert not (proj / "esphome_pch.h.gch.failed").exists()
assert scons_env.prepended == []
def test_pch_script_failure_marker_suppresses_retry(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
scons_env = _run_script(tmp_path, fail=True)
proj = tmp_path / "dev"
assert (proj / "esphome_pch.h.gch.failed").is_file()
assert not (proj / "esphome_pch.h.gch.sum").exists()
assert scons_env.prepended == []
# Second run: same checksum, no compile attempt, but says so
attempts = (tmp_path / "fake-gxx.argv").read_text().count("c++-header")
_run_script(tmp_path, fail=True)
out = capsys.readouterr().out
assert (tmp_path / "fake-gxx.argv").read_text().count("c++-header") == attempts
assert "delete esphome_pch.h.gch.failed to retry" in out
def test_pch_script_probe_rejection_falls_back(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""A toolchain that cannot load its own .gch (GCC 10 on macOS arm64)
must not leave consumers paying for a pch every compile rejects."""
scons_env = _run_script(tmp_path, reject_pch=True)
proj = tmp_path / "dev"
assert not (proj / "esphome_pch.h.gch").exists()
assert not (proj / "esphome_pch.h.gch.sum").exists()
assert (proj / "esphome_pch.h.gch.failed").is_file()
assert scons_env.prepended == []
assert "toolchain cannot load the pch" in capsys.readouterr().out
def test_pch_script_spawn_failure_is_transient(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""A spawn failure must not latch a .failed marker (matches espidf)."""
scons_env = _run_script(tmp_path, missing_cxx=True)
proj = tmp_path / "dev"
assert not (proj / "esphome_pch.h.gch.failed").exists()
assert not (proj / "esphome_pch.h.gch.sum").exists()
assert scons_env.prepended == []
assert "did not run" in capsys.readouterr().out
def test_pch_script_probe_nonzero_exit_falls_back(tmp_path: Path) -> None:
"""A probe failure whose stderr never mentions .gch must still count."""
scons_env = _run_script(tmp_path, probe_exit=1)
proj = tmp_path / "dev"
assert not (proj / "esphome_pch.h.gch").exists()
assert (proj / "esphome_pch.h.gch.failed").is_file()
assert scons_env.prepended == []
def test_pch_script_unresolved_package_version_skips_pch(tmp_path: Path) -> None:
"""A KeyError for an installed package is unresolved identity, not absence."""
scons_env = _run_script(tmp_path, platform_cls=_UnresolvedPlatform)
assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists()
assert scons_env.prepended == []
def test_pch_script_package_version_error_skips_pch(tmp_path: Path) -> None:
"""Without trustworthy package identity a stale .gch could survive an
upgrade, so the script must not build one at all."""
scons_env = _run_script(tmp_path, platform_cls=_BrokenPlatform)
proj = tmp_path / "dev"
assert not (proj / "esphome_pch.h.gch").exists()
assert scons_env.prepended == []
def test_pch_script_corrupt_sidecar_reads_as_stale(tmp_path: Path) -> None:
"""A truncated/corrupt .failed marker must not disable the pch forever."""
_run_script(tmp_path, fail=True)
proj = tmp_path / "dev"
(proj / "esphome_pch.h.gch.failed").write_bytes(b"\xff\xfe corrupt")
_run_script(tmp_path)
assert (proj / "esphome_pch.h.gch").is_file()
assert (proj / "esphome_pch.h.gch.sum").is_file()
def test_pch_script_rebuilds_when_header_missing(tmp_path: Path) -> None:
_run_script(tmp_path)
proj = tmp_path / "dev"
(proj / "esphome_pch.h").unlink()
_run_script(tmp_path)
assert (proj / "esphome_pch.h").is_file()
def test_copy_pch_script(tmp_path: Path) -> None:
from esphome.core import CORE
CORE.build_path = tmp_path
toolchain.copy_pch_script()
assert (tmp_path / "pch.py").read_text() == _SCRIPT.read_text()
def test_pch_script_unions_user_sloppiness(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""A user CCACHE_SLOPPINESS without the pch tokens gets them unioned on,
mirroring ccache_pch_env, or every src TU is a permanent miss."""
scons_env = _run_script(tmp_path, env_vars={"CCACHE_SLOPPINESS": "locale"})
assert scons_env["ENV"]["CCACHE_SLOPPINESS"] == "locale,pch_defines,time_macros"
assert "adding pch_defines,time_macros" in capsys.readouterr().out
def test_pch_script_unmodelable_flag_skips_pch(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""Unbalanced quotes and multi-token elements cannot be reproduced as
one argv; the pch is skipped rather than built with diverging flags."""
for bad in ("-DFOO='bar", "-DA=1\t-DB=2"):
scons_env = _run_script(tmp_path, flags=["-DX=1", bad])
assert scons_env.prepended == []
assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists()
assert "unmodelable flag" in capsys.readouterr().out
@pytest.mark.skipif(
getattr(os, "geteuid", lambda: -1)() == 0, reason="root ignores file modes"
)
def test_pch_script_unlistable_include_dir_skips_pch(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""An unlistable subtree must not silently drop out of the digest."""
proj = tmp_path / "dev"
override = proj / "lwip_override"
hidden = override / "hidden"
hidden.mkdir(parents=True)
(hidden / "gen.h").write_text("")
hidden.chmod(0)
try:
scons_env = _run_script(tmp_path, flags=["-DX=1", "-I", str(override)])
finally:
hidden.chmod(0o755)
assert scons_env.prepended == []
assert "skipping precompiled header" in capsys.readouterr().out
def test_pch_script_nobuild_without_projenv_is_noop(tmp_path: Path) -> None:
"""-t nobuild never exports projenv; the script must not abort."""
proj = tmp_path / "dev"
(proj / "src").mkdir(parents=True)
def strict_import(*names: str) -> None:
if "projenv" in names:
raise RuntimeError("Import of non-existent variable 'projenv'")
env = _FakeSConsEnv(proj, proj / "src", "g++", ["-DX=1"])
exec( # noqa: S102
compile(_SCRIPT.read_text(), "pch.py", "exec"),
{"Import": strict_import, "env": env},
)
assert not (proj / "esphome_pch.h").exists()
def test_pch_script_ignores_library_trees_and_non_headers(tmp_path: Path) -> None:
""".piolibdeps and non-header files must not enter the digest (or be
read at all); package versions already cover library identity."""
proj = tmp_path / "dev"
libdeps = proj / ".piolibdeps" / "lib" / "src"
libdeps.mkdir(parents=True)
(libdeps / "lib.h").write_text("#define A 1\n")
override = proj / "lwip_override"
override.mkdir(parents=True)
(override / "lwipopts.h").write_text("#define TCP_MSS 1460\n")
(override / "notes.txt").write_text("v1\n")
flags = ["-DX=1", "-I", str(libdeps), "-I", str(override)]
_run_script(tmp_path, flags=flags)
first = (proj / "esphome_pch.h.gch.sum").read_text()
(libdeps / "lib.h").write_text("#define A 2\n")
(override / "notes.txt").write_text("v2\n")
(tmp_path / "fake-gxx.argv").unlink(missing_ok=True)
_run_script(tmp_path, flags=flags)
assert (proj / "esphome_pch.h.gch.sum").read_text() == first
def test_pch_script_hashes_project_local_include_dirs(tmp_path: Path) -> None:
"""Generated headers in project-local -I dirs (e.g. rp2's lwip_override)
must invalidate the checksum when they change."""
proj = tmp_path / "dev"
override = proj / "lwip_override"
override.mkdir(parents=True)
(override / "lwipopts.h").write_text("#define TCP_MSS 1460\n")
flags = ["-DX=1", "-I", str(override)]
_run_script(tmp_path, flags=flags)
first = (proj / "esphome_pch.h.gch.sum").read_text()
(override / "lwipopts.h").write_text("#define TCP_MSS 536\n")
(tmp_path / "fake-gxx.argv").unlink(missing_ok=True)
_run_script(tmp_path, flags=flags)
assert (proj / "esphome_pch.h.gch.sum").read_text() != first
@pytest.mark.skipif(
getattr(os, "geteuid", lambda: -1)() == 0, reason="root ignores file modes"
)
def test_pch_script_unreadable_local_header_skips_pch(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
"""An unreadable generated header means unknown identity: no pch."""
proj = tmp_path / "dev"
override = proj / "lwip_override"
override.mkdir(parents=True)
secret = override / "lwipopts.h"
secret.write_text("#define TCP_MSS 1460\n")
secret.chmod(0)
scons_env = _run_script(tmp_path, flags=["-DX=1", "-I", str(override)])
assert not (proj / "esphome_pch.h.gch.sum").exists()
assert scons_env.prepended == []
assert "skipping precompiled header" in capsys.readouterr().out
+1 -31
View File
@@ -516,13 +516,10 @@ def test_clean_build(
dependencies_lock = tmp_path / "dependencies.lock"
dependencies_lock.write_text("lock file")
# idedata caches live under the data dir, not the build path; the
# .arduino.json variant is the native esp8266 toolchain's.
# idedata cache lives under the data dir, not the build path.
idedata_cache = tmp_path / "idedata" / "test.json"
idedata_cache.parent.mkdir()
idedata_cache.write_text("{}")
arduino_idedata_cache = tmp_path / "idedata" / "test.arduino.json"
arduino_idedata_cache.write_text("{}")
# Native ESP-IDF toolchain artifacts.
idf_build_dir = tmp_path / "build"
@@ -583,7 +580,6 @@ def test_clean_build(
assert not piolibdeps_dir.exists()
assert not dependencies_lock.exists()
assert not idedata_cache.exists()
assert not arduino_idedata_cache.exists()
assert not idf_build_dir.exists()
assert not managed_components_dir.exists()
assert not pio_components_dir.exists()
@@ -677,32 +673,6 @@ def test_clean_build_partial_exists(
assert "dependencies.lock" not in caplog.text
@patch("esphome.writer.CORE")
def test_clean_build_partial_removes_pch_artifacts(
mock_core: MagicMock,
tmp_path: Path,
) -> None:
"""The PlatformIO pch sidecars live at the project root and must go in
a partial clean, like the native backend's under .pioenvs."""
names = (
"esphome_pch.h",
"esphome_pch.h.gch",
"esphome_pch.h.gch.sum",
"esphome_pch.h.gch.failed",
)
for name in names:
(tmp_path / name).write_text("x")
mock_core.relative_pioenvs_path.return_value = tmp_path / ".pioenvs"
mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps"
mock_core.relative_build_path.side_effect = lambda name: tmp_path / name
mock_core.relative_internal_path.side_effect = tmp_path.joinpath
clean_build()
for name in names:
assert not (tmp_path / name).exists()
@patch("esphome.writer.CORE")
def test_clean_build_nothing_exists(
mock_core: MagicMock,