From 1ce0bed3f672d3a4699dad0cbfd8617c3b8950e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 18:10:45 +0200 Subject: [PATCH 01/55] [core] Share compiled binaries across modbus integration tests (#18945) --- script/helpers.py | 17 + tests/integration/README.md | 7 + tests/integration/conftest.py | 416 ++++++++++++++---- .../fixtures/sensor_filters_batch_window.yaml | 58 --- .../uart_mock_modbus_client_read_write.yaml | 111 ----- .../fixtures/uart_mock_modbus_custom_pdu.yaml | 88 ---- ...t_mock_modbus_deprecated_write_buffer.yaml | 106 ----- .../uart_mock_modbus_lambda_invert.yaml | 95 ---- .../uart_mock_modbus_lambda_write.yaml | 97 ---- .../fixtures/uart_mock_modbus_loopback.yaml | 233 ++++++++++ ...roller.yaml => uart_mock_modbus_mesh.yaml} | 121 ++++- .../uart_mock_modbus_register_offset.yaml | 138 ------ .../fixtures/uart_mock_modbus_server.yaml | 124 ------ ...ock_modbus_server_controller_multiple.yaml | 116 ----- ... => uart_mock_modbus_server_injected.yaml} | 55 ++- tests/integration/host_prefs.py | 14 +- .../test_api_zero_psk_provisioning.py | 1 - .../test_host_preferences_suspend_resume.py | 11 +- tests/integration/test_light_initial_state.py | 8 - tests/integration/test_uart_mock_modbus.py | 23 +- tests/script/test_helpers.py | 28 ++ 21 files changed, 805 insertions(+), 1062 deletions(-) delete mode 100644 tests/integration/fixtures/sensor_filters_batch_window.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_loopback.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_controller.yaml => uart_mock_modbus_mesh.yaml} (58%) delete mode 100644 tests/integration/fixtures/uart_mock_modbus_register_offset.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_read_write.yaml => uart_mock_modbus_server_injected.yaml} (52%) diff --git a/script/helpers.py b/script/helpers.py index bf22e15808..a8a237118f 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1104,6 +1104,10 @@ def get_components_per_integration_fixture() -> dict[str, set[str]]: _TEST_FUNC_RE = re.compile(r"async def (test_\w+)") +# Any usage form (decorator, pytestmark assignment or list element); only +# test_*.py files are scanned, so the marker docs elsewhere cannot false-hit +_SHARED_YAML_USE_RE = re.compile(r"\bmark\.shared_yaml") +_SHARED_YAML_ARG_RE = re.compile(r"\(\s*[\"'](\w+)[\"']\s*\)") @cache @@ -1123,6 +1127,19 @@ def get_fixture_to_test_files() -> dict[str, frozenset[str]]: for func in _TEST_FUNC_RE.findall(content): base_name = func.replace("test_", "").partition("[")[0] result.setdefault(base_name, set()).add(rel_path) + # Shared fixtures are named by marker, not by a test function; each + # decorator must carry a string literal or its fixture would silently + # map to no tests + for use in _SHARED_YAML_USE_RE.finditer(content): + arg = _SHARED_YAML_ARG_RE.match(content, use.end()) + if arg is None: + line = content.count("\n", 0, use.start()) + 1 + raise ValueError( + f"{rel_path}:{line}: shared_yaml marker must take a " + "single-line string literal so CI test selection can map " + "its fixture" + ) + result.setdefault(arg.group(1), set()).add(rel_path) return {k: frozenset(v) for k, v in result.items()} diff --git a/tests/integration/README.md b/tests/integration/README.md index 790d9a3a11..bee20409e8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -21,6 +21,13 @@ The `yaml_config` fixture automatically loads YAML configurations based on the t - The fixture file must exist or the test will fail with a clear error message - The fixture automatically injects a dynamic port number into the API configuration +Tests marked `@pytest.mark.shared_yaml("name")` load `fixtures/name.yaml` instead +of the test-named file and compile it in a shared, hash-keyed build directory, so +the whole group pays one full compile and each test only a relink. The marker +argument must be a single-line string literal (CI test selection maps fixtures to +test files by scanning for it), and marked tests must hand the `yaml_config` +content to `run_compiled` unmodified. + ### Key Fixtures - `run_compiled` - Combines write, compile, and run operations into a single context manager diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 15c5860879..78e0b1a36c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,17 +4,22 @@ from __future__ import annotations import asyncio from collections.abc import AsyncGenerator, Callable, Generator -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress import fcntl +from functools import cache +import hashlib import logging import os from pathlib import Path import platform +import re +import shutil import signal import socket import subprocess import sys import tempfile +import time from typing import TextIO from aioesphomeapi import APIClient, APIConnectionError, LogParser, ReconnectLogic @@ -23,7 +28,13 @@ import pytest_asyncio import esphome.config from esphome.core import CORE -from esphome.helpers import get_usable_cpu_count +from esphome.helpers import ( + get_usable_cpu_count, + read_file, + rmtree, + write_file, + write_file_if_changed, +) from esphome.platformio.toolchain import get_idedata from .const import ( @@ -56,6 +67,21 @@ import pty # not available on Windows pytest.register_assert_rewrite("tests.integration.entity_utils") +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "shared_yaml(name): load fixtures/.yaml and compile it in a shared, " + "hash-keyed incremental build directory", + ) + + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# CI caches parts of this path; keep in sync with ci.yml integration-tests. +INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests" + + def _get_platformio_env(cache_dir: Path) -> dict[str, str]: """Get environment variables for PlatformIO with shared cache.""" env = os.environ.copy() @@ -78,7 +104,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: ) # Compile with THIS tree's esphome sources, not wherever the venv's editable # install points (which may be a different git worktree or checkout). - repo_root = str(Path(__file__).resolve().parent.parent.parent) + repo_root = str(REPO_ROOT) existing = env.get("PYTHONPATH") env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{existing}" if existing else repo_root return env @@ -88,8 +114,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: def shared_platformio_cache() -> Generator[Path]: """Initialize a shared PlatformIO cache for all integration tests.""" # Use a dedicated directory for integration tests to avoid conflicts. - # CI caches parts of this path; keep in sync with ci.yml integration-tests. - test_cache_dir = Path.home() / ".esphome-integration-tests" + test_cache_dir = INTEGRATION_TESTS_ROOT cache_dir = test_cache_dir / "platformio" # Use a lock file in the home directory to ensure only one process initializes the cache @@ -112,7 +137,9 @@ def shared_platformio_cache() -> Generator[Path]: init_dir = Path(tmpdir) fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml" config_path = init_dir / "cache_init.yaml" - config_path.write_text(fixture_path.read_text()) + config_path.write_text( + fixture_path.read_text(encoding="utf-8"), encoding="utf-8" + ) # Run compilation to populate the cache # We must succeed here to avoid race conditions where multiple @@ -162,13 +189,6 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) -@pytest.fixture -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Host preferences persist per device name; give the test its own so a - provisioned key never leaks into another run.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" @@ -188,21 +208,29 @@ def unused_tcp_port(reserved_tcp_port: tuple[int, socket.socket]) -> int: return reserved_tcp_port[0] +@pytest.fixture(autouse=True) +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Give every test its own host prefs dir; prefs are keyed only by device + name, which tests sharing a fixture also share.""" + prefdir = tmp_path / "prefs" + monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) + return prefdir + + @pytest_asyncio.fixture async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> str: """Load YAML configuration based on test name.""" - # Get the test function name - test_name: str = request.node.name - # Extract the base test name (remove test_ prefix and any parametrization) - base_name = test_name.replace("test_", "").partition("[")[0] + shared_name = _shared_yaml_name(request) + # Base test name: test_ prefix and any parametrization stripped + base_name = shared_name or request.node.name.replace("test_", "").partition("[")[0] # Load the fixture file - fixture_path = Path(__file__).parent / "fixtures" / f"{base_name}.yaml" + fixture_path = FIXTURES_DIR / f"{base_name}.yaml" if not fixture_path.exists(): raise FileNotFoundError(f"Fixture file not found: {fixture_path}") loop = asyncio.get_running_loop() - content = await loop.run_in_executor(None, fixture_path.read_text) + content = await loop.run_in_executor(None, read_file, fixture_path) # Replace the port in the config if it contains api section if "api:" in content: @@ -226,11 +254,13 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s # Replace external component path placeholder if present if "EXTERNAL_COMPONENT_PATH" in content: - external_components_path = str( - Path(__file__).parent / "fixtures" / "external_components" - ) + external_components_path = str(FIXTURES_DIR / "external_components") content = content.replace("EXTERNAL_COMPONENT_PATH", external_components_path) + if shared_name is not None: + # _compile verifies the marked test compiles this content unmodified + request.node._shared_yaml_content = content + return content @@ -240,24 +270,218 @@ async def write_yaml_config( ) -> AsyncGenerator[ConfigWriter]: """Write YAML configuration to a file.""" # Get the test name for default filename - test_name = request.node.name - base_name = test_name.replace("test_", "").split("[")[0] + base_name = request.node.name.replace("test_", "").partition("[")[0] async def _write_config(content: str, filename: str | None = None) -> Path: if filename is None: filename = f"{base_name}.yaml" config_path = integration_test_dir / filename loop = asyncio.get_running_loop() - await loop.run_in_executor(None, config_path.write_text, content) + await loop.run_in_executor(None, write_file, config_path, content) return config_path yield _write_config +# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale +# dirs for a fixture are pruned when its content hash changes. +SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds" + +# In the dir name (not just the hash) so pruning stays inside this checkout +_REPO_KEY = hashlib.sha256(str(REPO_ROOT).encode()).hexdigest()[:8] + +# Give a contended shared build lock time for a full cold compile ahead of us +_SHARED_LOCK_TIMEOUT_S = 900 +_SHARED_LOCK_POLL_S = 0.1 +_SHARED_LOCK_REPORT_S = 30 + +# Reclaims dirs orphaned by fixture renames or deleted checkouts +_STALE_BUILD_MAX_AGE_S = 30 * 24 * 3600 + +# ELF path per shared build dir; constant once compiled, so resolve it only once +_shared_elf_paths: dict[Path, Path] = {} + +# Dirs this process already swept; pruning is session-scoped work +_pruned_dirs: set[Path] = set() + + +def _shared_yaml_name(request: pytest.FixtureRequest) -> str | None: + """Name passed to the shared_yaml marker, or None when unmarked.""" + marker = request.node.get_closest_marker("shared_yaml") + if marker is None: + return None + # Exactly one \w+ positional arg: the name doubles as a build dir + # component, and CI test selection (script/helpers.py) parses the same shape + if ( + len(marker.args) != 1 + or marker.kwargs + or not re.fullmatch(r"\w+", str(marker.args[0])) + ): + raise ValueError( + "shared_yaml marker requires exactly one \\w+ fixture name literal" + ) + return marker.args[0] + + +def _shared_build_prefix(name: str) -> str: + return f"{name}-{_REPO_KEY}-" + + +@cache +def _shared_build_dir(name: str) -> Path: + """Dir keyed by checkout and fixture source, before per-test injections.""" + key = hashlib.sha256((FIXTURES_DIR / f"{name}.yaml").read_bytes()).hexdigest()[:16] + return SHARED_BUILDS_ROOT / (_shared_build_prefix(name) + key) + + +def _read_stamp(stamp: Path, shared_dir: Path) -> Path | None: + """ELF path recorded by the last completed compile, or None.""" + try: + text = stamp.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return None + except OSError as err: + print(f"Cannot read {stamp}: {err}") + return None + if not text: + print(f"Ignoring empty stamp {stamp}") + return None + built = Path(text) + # Never trust a stamp pointing outside its own build dir as an unlink target + if shared_dir.resolve() in built.resolve().parents: + return built + print(f"Ignoring stamp {stamp} pointing outside {shared_dir}") + return None + + +def _unused_since(stale: Path, cutoff: float) -> bool: + """Whether a build dir looks untouched since cutoff; unknown counts as used.""" + # Newest of the .built stamp (rewritten by every completed compile) and the + # dir itself (freshened by a worker claiming the dir before locking) + newest: float | None = None + for probe in (stale / ".built", stale): + try: + mtime = probe.stat().st_mtime + except FileNotFoundError: + continue + except NotADirectoryError: + return True # a stray file where a dir should be; reclaimable + except OSError as err: + print(f"Cannot age-probe {stale}: {err}") + return False # unknown never authorizes deletion + newest = mtime if newest is None else max(newest, mtime) + return newest is not None and newest < cutoff + + +def _prune_stale_builds(name: str, keep: Path) -> None: + """Remove outdated build dirs (blocking, run in executor): this checkout's + other dirs for the fixture, plus anything untouched for 30 days. Tolerates + other workers pruning the same dirs concurrently.""" + cutoff = time.time() - _STALE_BUILD_MAX_AGE_S + prefix = _shared_build_prefix(name) + for stale in SHARED_BUILDS_ROOT.iterdir(): + if stale == keep: + continue + same_fixture = stale.name.startswith(prefix) + if not same_fixture and not _unused_since(stale, cutoff): + continue + # Creating .lock bumps the dir mtime, so remember whether the re-probe + # under the lock can trust it + lock_preexisting = (stale / ".lock").exists() + try: + lock_file = (stale / ".lock").open("w") + except FileNotFoundError: + continue # pruned by another worker meanwhile + except NotADirectoryError: + print(f"Removing stray file {stale}") + stale.unlink(missing_ok=True) + continue + except OSError as err: + print(f"Cannot prune {stale}: {err}") + continue + with lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue # still in use by another run + # Re-probe under the lock: a worker freshens its dir before + # locking, so a just-claimed dir no longer looks unused. A dir + # whose .lock we just created cannot be held by anyone, and our + # own open bumped its mtime, so its pre-open probe stands + if ( + lock_preexisting + and not same_fixture + and not _unused_since(stale, cutoff) + ): + continue + # rmtree tolerates races; a leftover partial tree only costs a + # rebuild, since the ELF is deleted before every compile + try: + rmtree(stale) + except OSError as err: + print(f"Failed to prune {stale}: {err}") + + +async def _run_esphome_compile( + config_path: Path, cwd: Path, env: dict[str, str] +) -> None: + """Run `esphome compile`, retrying up to 3 times on a segfault.""" + max_retries = 3 + for attempt in range(max_retries): + # Compile using subprocess, inheriting stdout/stderr to show progress + proc = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "esphome", + "compile", + str(config_path), + cwd=cwd, + stdout=None, # Inherit stdout + stderr=None, # Inherit stderr + stdin=asyncio.subprocess.DEVNULL, + # Start in a new process group to isolate signal handling + start_new_session=True, + env=env, + close_fds=False, + ) + await proc.wait() + + if proc.returncode == 0: + break + if proc.returncode == -11 and attempt < max_retries - 1: + # Segfault (-11 = SIGSEGV), retry + print( + f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." + ) + await asyncio.sleep(1) # Brief pause before retry + continue + raise RuntimeError( + f"Failed to compile {config_path}, return code: {proc.returncode}. " + f"Run with 'pytest -s' to see compilation output." + ) + + +def _resolve_compiled_binary(config_path: Path) -> Path: + """Load the config to learn the compiled ELF path (blocking, run in executor).""" + CORE.reset() # Reset CORE state between test runs + CORE.config_path = config_path + config = esphome.config.read_config( + {"command": "compile", "config": str(config_path)} + ) + if config is None: + raise RuntimeError(f"Failed to read config from {config_path}") + idedata = get_idedata(config) + binary_path = Path(idedata.firmware_elf_path) + if not binary_path.exists(): + raise RuntimeError(f"Compiled binary not found at {binary_path}") + return binary_path + + @pytest_asyncio.fixture async def compile_esphome( integration_test_dir: Path, shared_platformio_cache: Path, + request: pytest.FixtureRequest, ) -> AsyncGenerator[CompileFunction]: """Compile an ESPHome configuration and return the binary path.""" @@ -265,66 +489,96 @@ async def compile_esphome( # Use the shared PlatformIO cache for faster compilation # This avoids re-downloading dependencies for each test env = _get_platformio_env(shared_platformio_cache) - - # Retry compilation up to 3 times if we get a segfault - max_retries = 3 - for attempt in range(max_retries): - # Compile using subprocess, inheriting stdout/stderr to show progress - proc = await asyncio.create_subprocess_exec( - sys.executable, - "-m", - "esphome", - "compile", - str(config_path), - cwd=integration_test_dir, - stdout=None, # Inherit stdout - stderr=None, # Inherit stderr - stdin=asyncio.subprocess.DEVNULL, - # Start in a new process group to isolate signal handling - start_new_session=True, - env=env, - close_fds=False, - ) - await proc.wait() - - if proc.returncode == 0: - # Success! - break - if proc.returncode == -11 and attempt < max_retries - 1: - # Segfault (-11 = SIGSEGV), retry - print( - f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." - ) - await asyncio.sleep(1) # Brief pause before retry - continue - # Other error or final retry - raise RuntimeError( - f"Failed to compile {config_path}, return code: {proc.returncode}. " - f"Run with 'pytest -s' to see compilation output." - ) - - # Load the config to get idedata (blocking call, must use executor) loop = asyncio.get_running_loop() - def _read_config_and_get_binary(): - CORE.reset() # Reset CORE state between test runs - CORE.config_path = config_path - config = esphome.config.read_config( - {"command": "compile", "config": str(config_path)} + name = _shared_yaml_name(request) + if name is None: + await _run_esphome_compile(config_path, integration_test_dir, env) + return await loop.run_in_executor( + None, _resolve_compiled_binary, config_path ) - if config is None: - raise RuntimeError(f"Failed to read config from {config_path}") - # Get the compiled binary path - idedata = get_idedata(config) - return Path(idedata.firmware_elf_path) - - binary_path = await loop.run_in_executor(None, _read_config_and_get_binary) - - if not binary_path.exists(): - raise RuntimeError(f"Compiled binary not found at {binary_path}") - - return binary_path + # Shared fixture: build in a hash-keyed dir so tests sharing a config + # pay one full compile and later only a main.cpp (port) rebuild + relink + shared_dir = _shared_build_dir(name) + shared_dir.mkdir(parents=True, exist_ok=True) + # Freshen the dir before locking so a concurrent age sweep, which + # re-probes under the lock, never reaps a dir a worker just claimed; + # if a peer reaped it already, the guarded lock open recreates it + with suppress(FileNotFoundError): + os.utime(shared_dir) + if shared_dir not in _pruned_dirs: + _pruned_dirs.add(shared_dir) + await loop.run_in_executor(None, _prune_stale_builds, name, shared_dir) + shared_config = shared_dir / f"{name}.yaml" + private_binary = integration_test_dir / f"{name}.elf" + content = await loop.run_in_executor(None, read_file, config_path) + if content != getattr(request.node, "_shared_yaml_content", None): + # The dir is keyed by the fixture source; a mutated config would be + # cached under a hash that does not describe it + raise RuntimeError( + "shared_yaml tests must compile the yaml_config content unmodified" + ) + # flock serializes concurrent xdist workers; closing the fd releases it. + # Hand-rolled rather than filelock.FileLock: non-blocking retries keep + # the wait cancellable, while a blocking acquire in an executor thread + # would survive test cancellation holding the fd + try: + lock_file = (shared_dir / ".lock").open("w") + except FileNotFoundError: + # A peer run pruning divergent hashes reaped the dir between our + # mkdir and this open; recreate it and pay a full rebuild + shared_dir.mkdir(parents=True, exist_ok=True) + lock_file = (shared_dir / ".lock").open("w") + with lock_file: + start = time.monotonic() + last_report = start + while True: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + now = time.monotonic() + if now - start > _SHARED_LOCK_TIMEOUT_S: + raise RuntimeError( + f"Timed out waiting for the {shared_dir} lock" + ) from None + if now - last_report >= _SHARED_LOCK_REPORT_S: + last_report = now + print( + f"Waited {now - start:.0f}s for another worker's " + f"build of {shared_dir.name}" + ) + await asyncio.sleep(_SHARED_LOCK_POLL_S) + # .built carries the ELF path of the last completed compile, so + # later workers skip the config re-read in _resolve_compiled_binary + stamp = shared_dir / ".built" + if (built := _shared_elf_paths.get(shared_dir)) is None: + built = await loop.run_in_executor(None, _read_stamp, stamp, shared_dir) + # Delete the ELF before compiling: whatever exists afterwards is + # this compile's output, so no staleness check is ever needed. + # With no usable stamp, sweep any leftover at the known layout + if built is not None: + built.unlink(missing_ok=True) + else: + # Layout-agnostic: ESPHOME_BUILD_PATH can move the build tree + for leftover in shared_dir.rglob("program"): + if leftover.is_file(): + leftover.unlink() + await loop.run_in_executor( + None, write_file_if_changed, shared_config, content + ) + await _run_esphome_compile(shared_config, shared_dir, env) + if built is None or not built.exists(): + built = await loop.run_in_executor( + None, _resolve_compiled_binary, shared_config + ) + _shared_elf_paths[shared_dir] = built + await loop.run_in_executor(None, write_file, stamp, str(built)) + # Copy out before unlocking: another worker may relink firmware.elf + # while this test is still running its private copy + await loop.run_in_executor(None, shutil.copy2, built, private_binary) + return private_binary yield _compile diff --git a/tests/integration/fixtures/sensor_filters_batch_window.yaml b/tests/integration/fixtures/sensor_filters_batch_window.yaml deleted file mode 100644 index 58a254c215..0000000000 --- a/tests/integration/fixtures/sensor_filters_batch_window.yaml +++ /dev/null @@ -1,58 +0,0 @@ -esphome: - name: test-batch-window-filters - -host: -api: - batch_delay: 0ms # Disable batching to receive all state updates -logger: - level: DEBUG - -# Template sensor that we'll use to publish values -sensor: - - platform: template - name: "Source Sensor" - id: source_sensor - accuracy_decimals: 2 - - # Batch window filters (window_size == send_every) - use streaming filters - - platform: copy - source_id: source_sensor - name: "Min Sensor" - id: min_sensor - filters: - - min: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Max Sensor" - id: max_sensor - filters: - - max: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Moving Avg Sensor" - id: moving_avg_sensor - filters: - - sliding_window_moving_average: - window_size: 5 - send_every: 5 - send_first_at: 1 - -# Button to trigger publishing test values -button: - - platform: template - name: "Publish Values Button" - id: publish_button - on_press: - - lambda: |- - // Publish 10 values: 1.0, 2.0, ..., 10.0 - for (int i = 1; i <= 10; i++) { - id(source_sensor).publish_state(float(i)); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml deleted file mode 100644 index 1f89889c95..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml +++ /dev/null @@ -1,111 +0,0 @@ -esphome: - name: uart-mock-modbus-cli-rw - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -# Two virtual buses looped back to each other: the client's transmissions reach the server and the -# server's replies reach the client. auto_start so forwarding is active before the button fires. -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_client - data: !lambda return data; - - id: virtual_uart_client - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: stored_1 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_client - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - # Writable + readable register: the read publishes what it returns, so the test can confirm the - # write half of the 0x17 ran before the read half (Modbus 6.17). - - address: 0x01 - value_type: U_WORD - read_lambda: |- - id(srv_read_1).publish_state(id(stored_1)); - return id(stored_1); - write_lambda: |- - id(stored_1) = x; - id(srv_write_1).publish_state(x); - return true; - # Read-only register, returned together with 0x01 by the 2-register read half. - - address: 0x02 - value_type: U_WORD - read_lambda: return 0x00AA; - -sensor: - # Server-side observations. - - platform: template - name: "srv_write_1" - id: srv_write_1 - - platform: template - name: "srv_read_1" - id: srv_read_1 - # Client-side read-back: the values the client's on_response received. - - platform: template - name: "client_read_0" - id: client_read_0 - - platform: template - name: "client_read_1" - id: client_read_1 - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. - - modbus_client.read_write_multiple_registers: - address: 0x01 - read_address: 0x0001 - read_count: 2 - write_address: 0x0001 - values: [0x1234] - on_response: - then: - - lambda: |- - // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. - if (values.size() >= 2) { - id(client_read_0).publish_state(values[0]); - id(client_read_1).publish_state(values[1]); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml b/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml deleted file mode 100644 index 188abf90f1..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml +++ /dev/null @@ -1,88 +0,0 @@ -esphome: - name: uart-mock-modbus-custom-pdu - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 259; - -sensor: - # Plain read to confirm the controller <-> server link is up. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "plain_read" - address: 0x01 - register_type: holding - value_type: U_WORD - # Custom PDU: read holding register 0x0001, count 1. The PDU is - # {function code, address hi, address lo, count hi, count lo}; the device - # address and CRC are added by the hub. The lambda parses the response payload - # (the register value, big-endian). - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "custom_read" - custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] - lambda: |- - if (data.size() < 2) return {}; - return (float) ((data[0] << 8) | data[1]); - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml deleted file mode 100644 index f378e3de43..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml +++ /dev/null @@ -1,106 +0,0 @@ -esphome: - name: uart-mock-modbus-dep-buffer - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: |- - id(reg10) = x; - return true; - -# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw -# frame as words: device address + function code + data) instead of the new item->write_* API. The write -# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per -# entity no matter how many writes happen. -number: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "buf_number" - id: buf_number - address: 0x10 - register_type: holding - value_type: U_WORD - min_value: 0 - max_value: 1000 - step: 1 - write_lambda: |- - // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value. - payload.push_back(0x0106); - payload.push_back(0x0010); - payload.push_back((uint16_t) x); - return {}; - -# Reports the server-side register so the test can observe that the deprecated buffer write landed. -sensor: - - platform: template - name: "written_value" - id: written_value - update_interval: 0.5s - lambda: "return id(reg10);" - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # The test drives the writes via number_command; the mock is autostart. diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml deleted file mode 100644 index 41afce70d6..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml +++ /dev/null @@ -1,95 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-invert - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg40 - type: uint16_t - initial_value: "5" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x40 - value_type: U_WORD - read_lambda: return id(reg40); - write_lambda: id(reg40) = x; return true; - -# An active-low holding switch: the write_lambda inverts the wire value, but the entity must still -# report the REQUESTED state. assumed_state keeps the register unpolled, so the published state comes -# only from write_state() - turning ON writes 0x0000 yet the switch shows ON. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "invert_switch" - register_type: holding - address: 0x40 - assumed_state: true - write_lambda: |- - return !x; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_40" - address: 0x40 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml deleted file mode 100644 index 86e17ea0d7..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml +++ /dev/null @@ -1,97 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-write - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg30 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x30 - value_type: U_WORD - read_lambda: return id(reg30); - write_lambda: id(reg30) = x; return true; - -# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead -# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so -# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing -# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "cross_switch" - register_type: coil - address: 0x00 - assumed_state: true - write_lambda: |- - item->write_single_register(0x30, x ? 1234 : 0); - return {}; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_30" - address: 0x30 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_loopback.yaml b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml new file mode 100644 index 0000000000..7212bfb2b2 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml @@ -0,0 +1,233 @@ +esphome: + name: uart-mock-modbus-loopback + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Shared loopback fixture (see the shared_yaml markers in the test file); +# register spaces are disjoint so each test only observes its own entities. +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "100" + - id: reg11 + type: uint16_t + initial_value: "200" + - id: reg12 + type: uint16_t + initial_value: "300" + - id: reg13 + type: uint16_t + initial_value: "0xABCD" + - id: reg30 + type: uint16_t + initial_value: "0" + - id: reg40 + type: uint16_t + initial_value: "5" + - id: reg50 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 259; + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: id(reg10) = x; return true; + - address: 0x11 + value_type: U_WORD + read_lambda: return id(reg11); + write_lambda: id(reg11) = x; return true; + - address: 0x12 + value_type: U_WORD + read_lambda: return id(reg12); + write_lambda: id(reg12) = x; return true; + - address: 0x13 + value_type: U_WORD + read_lambda: return id(reg13); + - address: 0x30 + value_type: U_WORD + read_lambda: return id(reg30); + write_lambda: id(reg30) = x; return true; + - address: 0x40 + value_type: U_WORD + read_lambda: return id(reg40); + write_lambda: id(reg40) = x; return true; + - address: 0x50 + value_type: U_WORD + read_lambda: return id(reg50); + write_lambda: id(reg50) = x; return true; + +# Byte-based offset: 2 bytes -> register 0x11 (the old code folded it in as a +# register count, hitting 0x12). assumed_state keeps the switch write-only. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "offset_switch" + register_type: holding + address: 0x10 + offset: 2 + assumed_state: true + # Reading switch, byte offset 6 -> register 0x13; the pre-fix resolution (0x16) + # would draw ILLEGAL_DATA_ADDRESS and never publish. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "read_offset_switch" + register_type: holding + address: 0x10 + offset: 6 + bitmask: 0x1 + # Coil switch whose write_lambda dispatches a holding-register write via `item`; + # returning an empty optional suppresses the default coil write. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "cross_switch" + register_type: coil + address: 0x00 + assumed_state: true + write_lambda: |- + item->write_single_register(0x30, x ? 1234 : 0); + return {}; + # Active-low: the write_lambda inverts the wire value but the entity must still + # report the requested state (assumed_state keeps the register unpolled). + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "invert_switch" + register_type: holding + address: 0x40 + assumed_state: true + write_lambda: |- + return !x; + +# Uses the deprecated buffer parameter (legacy raw frame as words); the write +# must land and the deprecation warning must fire only once per entity. +number: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "buf_number" + id: buf_number + address: 0x50 + register_type: holding + value_type: U_WORD + min_value: 0 + max_value: 1000 + step: 1 + write_lambda: |- + // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0050, value. + payload.push_back(0x0106); + payload.push_back(0x0050); + payload.push_back((uint16_t) x); + return {}; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "plain_read" + address: 0x01 + register_type: holding + value_type: U_WORD + # Custom PDU: read holding register 0x0001; device address and CRC are added + # by the hub. The lambda parses the big-endian register value. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "custom_read" + custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] + lambda: |- + if (data.size() < 2) return {}; + return (float) ((data[0] << 8) | data[1]); + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_10" + address: 0x10 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_11" + address: 0x11 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_12" + address: 0x12 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_30" + address: 0x30 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_40" + address: 0x40 + register_type: holding + value_type: U_WORD + # Reports the server-side register so the test can observe that the deprecated buffer write landed. + - platform: template + name: "written_value" + id: written_value + update_interval: 0.5s + lambda: "return id(reg50);" + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # Nothing to start (mock is autostart); tests drive entities directly diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml similarity index 58% rename from tests/integration/fixtures/uart_mock_modbus_server_controller.yaml rename to tests/integration/fixtures/uart_mock_modbus_mesh.yaml index 4a5d280a2f..69edd614d7 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-server-contro + name: uart-mock-modbus-mesh host: api: @@ -17,13 +17,14 @@ uart: baud_rate: 115200 port: /dev/null +# Shared 3-bus mesh (see the shared_yaml markers): addr 1 = typed read-only +# registers, addr 5 = the read/write 0x17 target, addr 2/3 on the second +# server hub. auto_start everywhere: the controller polls at boot, so the +# forwarding must already be live or early requests generate warnings. +# Every test presses Start Scenario, so all merged actions fire in every test. uart_mock: - id: virtual_uart_server baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. auto_start: true debug: on_tx: @@ -31,35 +32,68 @@ uart_mock: - uart_mock.inject_rx: id: virtual_uart_controller data: !lambda return data; - - id: virtual_uart_controller + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + - id: virtual_uart_server_2 baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above + auto_start: true debug: on_tx: - then: - uart_mock.inject_rx: id: virtual_uart_server data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + +globals: + - id: stored_1 + type: uint16_t + initial_value: "0" modbus: - uart_id: virtual_uart_server id: virtual_modbus_server role: server + - uart_id: virtual_uart_server_2 + id: virtual_modbus_server_2 + role: server - uart_id: virtual_uart_controller - id: virtual_modbus_controller + id: virtual_modbus_client role: client turnaround_time: 10ms modbus_controller: - address: 1 - modbus_id: virtual_modbus_controller + modbus_id: virtual_modbus_client id: modbus_controller_1 update_interval: 1s + - address: 2 + modbus_id: virtual_modbus_client + id: modbus_controller_2 + update_interval: 1s + - address: 3 + modbus_id: virtual_modbus_client + id: modbus_controller_3 + update_interval: 1s modbus_server: - address: 1 modbus_id: virtual_modbus_server - id: modbus_server_1 registers: - address: 0x01 value_type: U_WORD @@ -103,6 +137,34 @@ modbus_server: - address: 0x28 value_type: FP32_R read_lambda: return 3.14; + - address: 5 + modbus_id: virtual_modbus_server + registers: + # Writable + readable register: srv_write_1 plus the client's read-back + # confirm the write half of the 0x17 ran before the read half (Modbus 6.17). + - address: 0x01 + value_type: U_WORD + read_lambda: return id(stored_1); + write_lambda: |- + id(stored_1) = x; + id(srv_write_1).publish_state(x); + return true; + # Read-only register, returned together with 0x01 by the 2-register read half. + - address: 0x02 + value_type: U_WORD + read_lambda: return 0x00AA; + - address: 2 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 919; + - address: 3 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 929; sensor: - platform: modbus_controller @@ -195,9 +257,46 @@ sensor: address: 0x28 register_type: holding value_type: FP32_R + - platform: modbus_controller + modbus_controller_id: modbus_controller_2 + name: "multi_reg_a" + address: 0x01 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_3 + name: "multi_reg_b" + address: 0x01 + register_type: holding + value_type: U_WORD + # client_read_write observations, server- and client-side. + - platform: template + name: "srv_write_1" + id: srv_write_1 + - platform: template + name: "client_read_0" + id: client_read_0 + - platform: template + name: "client_read_1" + id: client_read_1 button: - platform: template name: "Start Scenario" id: start_scenario_btn - # This test does not have anything to start (mock is autostart) + on_press: + # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. + - modbus_client.read_write_multiple_registers: + address: 5 + read_address: 0x0001 + read_count: 2 + write_address: 0x0001 + values: [0x1234] + on_response: + then: + - lambda: |- + // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. + if (values.size() >= 2) { + id(client_read_0).publish_state(values[0]); + id(client_read_1).publish_state(values[1]); + } diff --git a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml deleted file mode 100644 index 21c451aa99..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml +++ /dev/null @@ -1,138 +0,0 @@ -esphome: - name: uart-mock-modbus-reg-offset - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "100" - - id: reg11 - type: uint16_t - initial_value: "200" - - id: reg12 - type: uint16_t - initial_value: "300" - - id: reg13 - type: uint16_t - initial_value: "0xABCD" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: id(reg10) = x; return true; - - address: 0x11 - value_type: U_WORD - read_lambda: return id(reg11); - write_lambda: id(reg11) = x; return true; - - address: 0x12 - value_type: U_WORD - read_lambda: return id(reg12); - write_lambda: id(reg12) = x; return true; - - address: 0x13 - value_type: U_WORD - read_lambda: return id(reg13); - write_lambda: id(reg13) = x; return true; - -# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target -# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register -# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "offset_switch" - register_type: holding - address: 0x10 - offset: 2 - assumed_state: true - # A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix - # the switch itself resolves to 0x13 (the even byte offset folds into the address as whole registers) and - # joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds - # into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "read_offset_switch" - register_type: holding - address: 0x10 - offset: 6 - bitmask: 0x1 - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_10" - address: 0x10 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_11" - address: 0x11 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_12" - address: 0x12 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server.yaml b/tests/integration/fixtures/uart_mock_modbus_server.yaml deleted file mode 100644 index cc5a59e242..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server.yaml +++ /dev/null @@ -1,124 +0,0 @@ -esphome: - name: uart-mock-modbus-server-test - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_dev - baud_rate: 9600 - rx_full_threshold: 120 - rx_timeout: 2 - auto_start: false - debug: - injections: - - delay: 100ms - inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) - - delay: 100ms - # Read holding register 7 on device 2 - # Reply from device 2 - # Read holding register 5 on device 1 (read_after_peer_response) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x02, - 0x03, - 0x02, - 0x00, - 0xF0, - 0xFC, - 0x00, - 0x01, - 0x03, - 0x00, - 0x05, - 0x00, - 0x01, - 0x94, - 0x0B, - ] - - delay: 100ms - inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response - - delay: 100ms - # Read holding register 7 on device 2, with no response - # Read holding register A on device 1 (read_after_peer_timeout) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x01, - 0x03, - 0x00, - 0x0A, - 0x00, - 0x01, - 0xA4, - 0x08, - ] - -modbus: - uart_id: virtual_uart_dev - role: server - -modbus_server: - - address: 1 - registers: - - address: 0x03 - value_type: U_WORD - read_lambda: |- - id(basic_read).publish_state(1); - return 1; - - address: 0x05 - value_type: U_WORD - read_lambda: |- - id(read_after_peer_response).publish_state(1); - return 1; - - address: 0x0A - value_type: U_WORD - read_lambda: |- - id(read_after_peer_timeout).publish_state(1); - return 1; - -sensor: - - platform: template - name: "basic_read" - id: basic_read - - platform: template - name: "read_after_peer_response" - id: read_after_peer_response - - platform: template - name: "read_after_peer_timeout" - id: read_after_peer_timeout - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - - lambda: "id(virtual_uart_dev).start_scenario();" diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml b/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml deleted file mode 100644 index 18423be6d5..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml +++ /dev/null @@ -1,116 +0,0 @@ -esphome: - name: uart-mock-modbus-server-mult - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - - id: virtual_uart_server_2 - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_server_2 - id: virtual_modbus_server_2 - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_1 - - address: 2 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_2 - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 919; - - address: 2 - modbus_id: virtual_modbus_server_2 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 929; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_u_word" - address: 0x01 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_2 - name: "reg_u_word_2" - address: 0x01 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml similarity index 52% rename from tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml rename to tests/integration/fixtures/uart_mock_modbus_server_injected.yaml index e998861c2d..2cd1c610f1 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-srv-rw + name: uart-mock-modbus-srv-injected host: api: @@ -17,6 +17,8 @@ uart: baud_rate: 115200 port: /dev/null +# Shared server-role fixture (see the shared_yaml markers in the test file); +# the injections concatenate and each test waits only on its own sensors. uart_mock: - id: virtual_uart_dev baud_rate: 9600 @@ -25,18 +27,31 @@ uart_mock: auto_start: false debug: injections: - # FC 0x17 Read/Write Multiple Registers on device 1: - # write reg 0x0001 = 0x1234 (qty 1), then read regs 0x0001..0x0002 (qty 2). - # Per Modbus 6.17 the write is performed before the read, so reg 0x0001 must - # read back the just-written 0x1234 in the same request. + - delay: 100ms + inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) + - delay: 100ms + # Read holding register 7 on device 2, its reply, then read holding + # register 5 on device 1 (read_after_peer_response) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x02, 0x03, 0x02, 0x00, 0xF0, 0xFC, + 0x00, 0x01, 0x03, 0x00, 0x05, 0x00, 0x01, 0x94, 0x0B] + - delay: 100ms + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response + - delay: 100ms + # Read holding register 7 on device 2 with no response, then read + # holding register A on device 1 (read_after_peer_timeout) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x01, 0x03, 0x00, 0x0A, 0x00, 0x01, 0xA4, 0x08] + # FC 0x17 on device 1: write reg 0x0001 = 0x1234 then read 0x0001..0x0002; + # per Modbus 6.17 the write runs first, so 0x0001 must read back 0x1234. - delay: 100ms inject_rx: [0x01, 0x17, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x02, 0x12, 0x34, 0x49, 0xD8] - # FC 0x17: write reg 0x0003 = 0x5678 (qty 1), then read reg 0x0003 (qty 1) - + # FC 0x17: write reg 0x0006 = 0x5678 (qty 1), then read reg 0x0006 (qty 1) - # a write and read targeting a different register block. - delay: 100ms inject_rx: - [0x01, 0x17, 0x00, 0x03, 0x00, 0x01, 0x00, 0x03, 0x00, 0x01, 0x02, 0x56, 0x78, 0x9B, 0x10] + [0x01, 0x17, 0x00, 0x06, 0x00, 0x01, 0x00, 0x06, 0x00, 0x01, 0x02, 0x56, 0x78, 0x8B, 0x55] globals: - id: stored_1 @@ -70,8 +85,18 @@ modbus_server: read_lambda: |- id(rw_read_2).publish_state(0x00AA); return 0x00AA; - # Second writable + readable register, targeted by the second request. - address: 0x03 + value_type: U_WORD + read_lambda: |- + id(basic_read).publish_state(1); + return 1; + - address: 0x05 + value_type: U_WORD + read_lambda: |- + id(read_after_peer_response).publish_state(1); + return 1; + # Second writable + readable register, targeted by the second FC 0x17 request. + - address: 0x06 value_type: U_WORD read_lambda: |- id(rw_read_3).publish_state(id(stored_3)); @@ -80,8 +105,22 @@ modbus_server: id(stored_3) = x; id(rw_write_3).publish_state(x); return true; + - address: 0x0A + value_type: U_WORD + read_lambda: |- + id(read_after_peer_timeout).publish_state(1); + return 1; sensor: + - platform: template + name: "basic_read" + id: basic_read + - platform: template + name: "read_after_peer_response" + id: read_after_peer_response + - platform: template + name: "read_after_peer_timeout" + id: read_after_peer_timeout - platform: template name: "rw_write_1" id: rw_write_1 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a01..5f526dce5f 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -1,7 +1,7 @@ """Helpers for manipulating the host platform's preferences file. ESPHome's host platform stores preferences in -``~/.esphome/prefs/.prefs`` using a simple binary layout that +``$ESPHOME_PREFDIR/.prefs`` using a simple binary layout that mirrors ``HostPreferences::sync()``: ``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry. @@ -11,13 +11,21 @@ boot (e.g. forcing safe mode) or to clear stale state between runs. from __future__ import annotations +import os from pathlib import Path import struct def host_prefs_path(device_name: str) -> Path: - """Return the on-disk prefs file path for a host-platform device.""" - return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs" + """Return the on-disk prefs file path for a host-platform device. + + Requires ESPHOME_PREFDIR, which the autouse isolated_preferences fixture + sets; refusing the ~/.esphome/prefs fallback keeps tests off real user + data if the fixture is ever bypassed.""" + prefdir = os.environ.get("ESPHOME_PREFDIR") + if not prefdir: + raise RuntimeError("ESPHOME_PREFDIR is not set; refusing the real prefs dir") + return Path(prefdir) / f"{device_name}.prefs" def clear_host_prefs(device_name: str) -> None: diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index f315335d1b..d103167a00 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -24,7 +24,6 @@ from .types import ( RunCompiledFunction, ) -pytestmark = pytest.mark.usefixtures("isolated_preferences") NEW_KEY = PROVISIONING_PSK diff --git a/tests/integration/test_host_preferences_suspend_resume.py b/tests/integration/test_host_preferences_suspend_resume.py index ab08d5c440..5f08d5519e 100644 --- a/tests/integration/test_host_preferences_suspend_resume.py +++ b/tests/integration/test_host_preferences_suspend_resume.py @@ -41,15 +41,6 @@ async def _poll_until_exists(path: Path) -> None: await asyncio.sleep(0.05) -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> Path: - """Keep host preferences per-test so this test never touches the real - ~/.esphome/prefs and never races other tests over ESPHOME_PREFDIR.""" - prefdir = tmp_path / "prefs" - monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) - return prefdir / f"{DEVICE_NAME}.prefs" - - @pytest.mark.asyncio async def test_host_preferences_suspend_resume( yaml_config: str, @@ -58,7 +49,7 @@ async def test_host_preferences_suspend_resume( isolated_preferences: Path, ) -> None: """Test that a running syncer flushes, a suspended one doesn't, and resume restores flushing.""" - pref_file = isolated_preferences + pref_file = isolated_preferences / f"{DEVICE_NAME}.prefs" loop = asyncio.get_running_loop() saved_in_memory = loop.create_future() diff --git a/tests/integration/test_light_initial_state.py b/tests/integration/test_light_initial_state.py index 657e273fe7..12ebf7c4a1 100644 --- a/tests/integration/test_light_initial_state.py +++ b/tests/integration/test_light_initial_state.py @@ -11,14 +11,6 @@ from .state_utils import InitialStateHelper, require_entity from .types import APIClientConnectedFactory, RunCompiledFunction -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so RESTORE_AND_ON never loads a stale value left - behind by a previous run (host preferences otherwise persist to ~/.esphome/prefs, - keyed only by device name).""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.mark.asyncio async def test_light_initial_state( yaml_config: str, diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 864275f5ed..232e1fb654 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -173,6 +173,7 @@ async def test_uart_mock_modbus_no_threshold( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server( yaml_config: str, @@ -203,6 +204,7 @@ async def test_uart_mock_modbus_server( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server_read_write( yaml_config: str, @@ -231,8 +233,8 @@ async def test_uart_mock_modbus_server_read_write( "rw_write_1": 4660, # 0x1234 written to reg 0x0001 "rw_read_1": 4660, # reg 0x0001 reads back the just-written value "rw_read_2": 170, # 0x00AA read from reg 0x0002 in the same request - "rw_write_3": 22136, # 0x5678 written to reg 0x0003 - "rw_read_3": 22136, # reg 0x0003 reads back the just-written value + "rw_write_3": 22136, # 0x5678 written to reg 0x0006 + "rw_read_3": 22136, # reg 0x0006 reads back the just-written value } ) @@ -241,7 +243,8 @@ async def test_uart_mock_modbus_server_read_write( api_client_connected() as client, ): await tracker.setup_and_start_scenario(client) - await tracker.await_all(futures) + # The FC 0x17 injections fire last, behind four earlier 100ms delays + await tracker.await_all(futures, timeout=4.0) _assert_no_modbus_errors(error_log_lines, warning_log_lines) @@ -296,6 +299,7 @@ async def test_uart_mock_modbus_server_read_write_invalid( ) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller( yaml_config: str, @@ -485,6 +489,7 @@ async def test_uart_mock_modbus_server_controller_bits( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller_multiple( yaml_config: str, @@ -495,7 +500,7 @@ async def test_uart_mock_modbus_server_controller_multiple( line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - expected_values = {"reg_u_word": 919, "reg_u_word_2": 929} + expected_values = {"multi_reg_a": 919, "multi_reg_b": 929} tracker = SensorTracker(list(expected_values.keys())) futures = tracker.expect_all(expected_values) @@ -706,6 +711,7 @@ async def test_uart_mock_modbus_shared_address( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_custom_pdu( yaml_config: str, @@ -932,6 +938,7 @@ async def test_uart_mock_modbus_broadcast_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_client_read_write( yaml_config: str, @@ -947,9 +954,7 @@ async def test_uart_mock_modbus_client_read_write( """ line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - tracker = SensorTracker( - ["srv_write_1", "srv_read_1", "client_read_0", "client_read_1"] - ) + tracker = SensorTracker(["srv_write_1", "client_read_0", "client_read_1"]) futures = tracker.expect_all( { "srv_write_1": 4660, # server wrote 0x1234 to reg 0x0001 @@ -967,6 +972,7 @@ async def test_uart_mock_modbus_client_read_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_register_offset( yaml_config: str, @@ -1022,6 +1028,7 @@ async def test_uart_mock_modbus_register_offset( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_write( yaml_config: str, @@ -1058,6 +1065,7 @@ async def test_uart_mock_modbus_lambda_write( await tracker.await_change(wrote_30, "reg_30", timeout=4.0) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_invert( yaml_config: str, @@ -1113,6 +1121,7 @@ async def test_uart_mock_modbus_lambda_invert( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_deprecated_write_buffer( yaml_config: str, diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 7d4059da2f..8f82a121c6 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2122,6 +2122,34 @@ def test_get_cpp_changed_components_independent_of_cwd( ) == ["time"] +def test_fixture_map_includes_shared_yaml_markers() -> None: + """Fixtures named only by shared_yaml markers must map to their test file.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + for fixture in ( + "uart_mock_modbus_loopback", + "uart_mock_modbus_mesh", + "uart_mock_modbus_server_injected", + ): + assert mapping[fixture] == frozenset( + {"tests/integration/test_uart_mock_modbus.py"} + ) + + +def test_no_orphan_integration_fixtures() -> None: + """Every fixture must reach CI test selection; an orphan selects nothing.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve() + fixtures = list(fixtures_dir.glob("*.yaml")) + assert fixtures, f"no fixtures found under {fixtures_dir}" + # cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead + orphans = [ + f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping + ] + assert not orphans, f"fixtures invisible to CI test selection: {orphans}" + + def test_lpt_partition_balances_skewed_weights() -> None: """Heavy items spread across groups instead of clustering.""" items = [f"i{n}" for n in range(6)] From 3926612281789284df820f21f159f8cf1bb24969 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 02/55] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 4ab9298ab3eedddbd45507785b4d2453ae867bba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:11:36 +0200 Subject: [PATCH 03/55] Bump esptool from 5.3.1 to 5.4.0 (#19023) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd3f7446f3..dfddbed00b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ tzlocal==5.4.4 # from time tzdata>=2026.3 # from time pyserial==3.5 platformio==6.1.19 -esptool==5.3.1 +esptool==5.4.0 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi From 5bb112f407e8edac9576a7eea1eafb9d94cb1f47 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 04/55] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From 006f31af9308fd85212cec8b5a9816273608dbba Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 05/55] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 8f511a365a471d1614e7578a03ceb3c0dbc4470f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 06/55] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 6c5ab89d5f818ac501855479ea776984c5d3f16a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 07/55] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From b947094f45f7bc8b193db6a75b732c9bdbcce41b Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 08/55] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 823d79c948eb4474423200d5a251210c31482b68 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 09/55] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 628ebe23ec389d770e822f18de22753c167dff6f Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 10/55] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From e7f45a0d315442dcf789997d6e28de72f082e28a Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 11/55] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From d5cff6e9dfcdfce156483eecf205e64169a56dee Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 12/55] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From 934086365217965f95c21bda0593b3f2ec960615 Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 13/55] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From 199acdf5222a923d5c7951af2e0ea632f45ba220 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 14/55] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 442e4a1ec2c70bfc8507aa1cf5f471402806d9b0 Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 15/55] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From c9729244af79e5b36a2b712c2fa5b91efa6a504f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 16/55] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From ca864c22b4c0810e9e4779bfa6d95b3597cc8abe Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 17/55] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 866ddb6e5729f11e2a3a107fa7145dc596b9a17a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 18/55] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From f8a4cfa945ef765e469daa03edbe263346a03154 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 19/55] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From c3ce07755f32292af3da6466aa1fc4a2cfeca07d Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 20/55] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 7660dd7fa7059a6154e65797c26e45d27aa8bf78 Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 21/55] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From f8b2e53609051bf6ac9a626305a7e6304c67393f Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 22/55] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f89b9e704c7dfabce1e8ce670dd2c6e7aa9ea086 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 23/55] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \ From 9b6facb20d5461dfaf47fd3993a7b4601f082c34 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 15:46:17 -0400 Subject: [PATCH 24/55] [core] Fix use-after-free when deleting a running StaticTask (#19048) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../micro_wake_word/micro_wake_word.cpp | 4 +-- .../mixer/speaker/mixer_speaker.cpp | 4 +-- .../resampler/speaker/resampler_speaker.cpp | 4 +-- .../speaker/media_player/audio_pipeline.cpp | 11 +++++-- esphome/core/static_task.cpp | 30 ++++++++++++++----- esphome/core/static_task.h | 17 +++++++---- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 3dadb78077..cebfe8e791 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -446,9 +446,9 @@ void MicroWakeWord::loop() { xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STOPPING); } - if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & EventGroupBits::TASK_STOPPED) && this->inference_task_.deallocate()) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 6128dc3767..0b79010773 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -382,8 +382,8 @@ void MixerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, MIXER_TASK_STATE_STOPPING); } - if (event_group_bits & MIXER_TASK_STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); this->all_stopped_since_ms_ = 0; diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index f1ebd180cc..edda00ae06 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -153,8 +153,8 @@ void ResamplerSpeaker::loop() { ESP_LOGV(TAG, "Stopping"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::STATE_STOPPING); } - if (event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) { - this->task_.deallocate(); + // Retries on a subsequent loop if the task is still running on the other core + if ((event_group_bits & ResamplingEventGroupBits::STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); xEventGroupClearBits(this->event_group_, ResamplingEventGroupBits::ALL_BITS); } diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index 010f0c50b3..c286a9d7d6 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -202,8 +202,15 @@ AudioPipelineState AudioPipeline::process_state() { if (!this->is_playing_) { // The tasks have been stopped for two ``process_state`` calls in a row, so delete the tasks if (this->read_task_.is_created() || this->decode_task_.is_created()) { - this->read_task_.deallocate(); - this->decode_task_.deallocate(); + // Both are attempted every time; a task that is still running on the other core is freed by a + // subsequent call, and freeing an already freed task succeeds without doing anything + bool read_task_freed = this->read_task_.deallocate(); + bool decode_task_freed = this->decode_task_.deallocate(); + if (!read_task_freed || !decode_task_freed) { + // A task is still running on the other core, so keep the pipeline in its current state and try + // again on the next call + return AudioPipelineState::PLAYING; + } if (this->hard_stop_) { // Stop command was sent, so immediately end the playback this->speaker_->stop(); diff --git a/esphome/core/static_task.cpp b/esphome/core/static_task.cpp index 4cfead44c2..4301108315 100644 --- a/esphome/core/static_task.cpp +++ b/esphome/core/static_task.cpp @@ -40,16 +40,31 @@ bool StaticTask::create(TaskFunction_t fn, const char *name, uint32_t stack_size return true; } -void StaticTask::destroy() { - if (this->handle_ != nullptr) { - TaskHandle_t handle = this->handle_; - this->handle_ = nullptr; - vTaskDelete(handle); +bool StaticTask::destroy() { + if (this->handle_ == nullptr) { + return true; } + + // Suspending takes the task off the ready and event lists, so nothing can schedule it again. It only asks + // the other core to yield though, so the task may still be running on it for a moment. + vTaskSuspend(this->handle_); + if (eTaskGetState(this->handle_) != eSuspended) { + // The task is still running on the other core and using its stack. Deleting it now would only put it on + // the termination list and return, so the caller has to try again once it has been swapped out. + return false; + } + + // The task cannot run again, so the delete completes right away instead of being left to the idle task. + TaskHandle_t handle = this->handle_; + this->handle_ = nullptr; + vTaskDelete(handle); + return true; } -void StaticTask::deallocate() { - this->destroy(); +bool StaticTask::deallocate() { + if (!this->destroy()) { + return false; + } if (this->stack_buffer_ != nullptr) { RAMAllocator allocator(this->use_psram_ ? RAMAllocator::ALLOC_EXTERNAL : RAMAllocator::ALLOC_INTERNAL); @@ -57,6 +72,7 @@ void StaticTask::deallocate() { this->stack_buffer_ = nullptr; this->stack_size_ = 0; } + return true; } } // namespace esphome diff --git a/esphome/core/static_task.h b/esphome/core/static_task.h index 5fd5b38f9e..e2996abeda 100644 --- a/esphome/core/static_task.h +++ b/esphome/core/static_task.h @@ -11,6 +11,7 @@ namespace esphome { /** Helper for FreeRTOS static task management. * Bundles TaskHandle_t, StaticTask_t, and the stack buffer into one object with create/destroy methods. + * Call destroy() and deallocate() from another task: a task cannot free the stack it is still running on. */ class StaticTask { public: @@ -23,7 +24,7 @@ class StaticTask { /// @brief Allocate stack and create task. /// @param fn Task function /// @param name Task name (for debug) - /// @param stack_size Stack size in StackType_t words + /// @param stack_size Stack size in bytes (StackType_t is a byte on ESP-IDF) /// @param param Parameter passed to task function /// @param priority FreeRTOS task priority /// @param use_psram If true, allocate stack in PSRAM; otherwise internal RAM @@ -31,11 +32,17 @@ class StaticTask { bool create(TaskFunction_t fn, const char *name, uint32_t stack_size, void *param, UBaseType_t priority, bool use_psram); - /// @brief Delete the task but keep the stack buffer allocated for reuse by a subsequent create() call. - void destroy(); + /// @brief Delete the task, keeping the stack buffer allocated for reuse by a subsequent create() call. + /// The task must have finished its work and parked itself, either suspended or blocked indefinitely: it is + /// suspended here so that it cannot be scheduled again, and it is given no chance to clean up. + /// @return true if the task was deleted; false if it is still running on another core, in which case the + /// caller should try again later. + bool destroy(); - /// @brief Delete the task (if running) and free the stack buffer. - void deallocate(); + /// @brief Delete the task (if created) and free the stack buffer. + /// @return true if the stack buffer was freed; false if the task is still running on another core, in + /// which case the caller should try again later. + bool deallocate(); protected: TaskHandle_t handle_{nullptr}; From 0a1e2acbcba4521391742824c617c8cf206beb63 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:04:34 -0400 Subject: [PATCH 25/55] [audio][i2s_audio][micro_wake_word][microphone][mixer][resampler][speaker] Replace use_count() checks with lock and null test (#19046) --- esphome/components/audio/audio_reader.cpp | 3 +++ esphome/components/audio/audio_transfer_buffer.cpp | 12 ++++++------ .../i2s_audio/speaker/i2s_audio_speaker.cpp | 4 ++-- .../components/micro_wake_word/micro_wake_word.cpp | 2 +- esphome/components/microphone/microphone_source.h | 2 +- esphome/components/mixer/speaker/mixer_speaker.cpp | 12 ++++++------ .../resampler/speaker/resampler_speaker.cpp | 6 +++--- .../speaker/media_player/audio_pipeline.cpp | 12 +++++++----- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/esphome/components/audio/audio_reader.cpp b/esphome/components/audio/audio_reader.cpp index 4678ed548c..e69f33ac2d 100644 --- a/esphome/components/audio/audio_reader.cpp +++ b/esphome/components/audio/audio_reader.cpp @@ -58,6 +58,9 @@ esp_err_t AudioReader::add_sink(const std::weak_ptr &ou if (current_audio_file_ != nullptr) { // A transfer buffer isn't ncessary for a local file this->file_ring_buffer_ = output_ring_buffer.lock(); + if (this->file_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; + } return ESP_OK; } diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index a611549e58..01fd4bb68a 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -51,14 +51,14 @@ void AudioTransferBuffer::increase_buffer_length(size_t bytes) { this->buffer_le void AudioTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } } void AudioSinkTransferBuffer::clear_buffered_data() { this->buffer_length_ = 0; - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { this->ring_buffer_->reset(); } #ifdef USE_SPEAKER @@ -69,7 +69,7 @@ void AudioSinkTransferBuffer::clear_buffered_data() { } bool AudioTransferBuffer::has_buffered_data() const { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); @@ -144,7 +144,7 @@ size_t AudioSourceTransferBuffer::transfer_data_from_source(TickType_t ticks_to_ size_t bytes_to_read = AudioTransferBuffer::free(); size_t bytes_read = 0; if (bytes_to_read > 0) { - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_read = this->ring_buffer_->read((void *) this->get_buffer_end(), bytes_to_read, ticks_to_wait); } @@ -161,7 +161,7 @@ size_t AudioSinkTransferBuffer::transfer_data_to_sink(TickType_t ticks_to_wait, bytes_written = this->speaker_->play(this->data_start_, this->available(), ticks_to_wait); } else #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { bytes_written = this->ring_buffer_->write_without_replacement((void *) this->data_start_, this->available(), ticks_to_wait); } else if (this->sink_callback_ != nullptr) { @@ -186,7 +186,7 @@ bool AudioSinkTransferBuffer::has_buffered_data() const { return (this->speaker_->has_buffered_data() || (this->available() > 0)); } #endif - if (this->ring_buffer_.use_count() > 0) { + if (this->ring_buffer_ != nullptr) { return ((this->ring_buffer_->available() > 0) || (this->available() > 0)); } return (this->available() > 0); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 1c2eb12904..b78a151ee4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -218,8 +218,8 @@ size_t I2SAudioSpeakerBase::play(const uint8_t *data, size_t length, TickType_t } bool I2SAudioSpeakerBase::has_buffered_data() const { - if (this->audio_ring_buffer_.use_count() > 0) { - std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + std::shared_ptr temp_ring_buffer = this->audio_ring_buffer_.lock(); + if (temp_ring_buffer != nullptr) { return temp_ring_buffer->available() > 0; } return false; diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index cebfe8e791..cf239be696 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -129,7 +129,7 @@ void MicroWakeWord::setup() { return; } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (this->ring_buffer_.use_count() > 1) { + if (temp_ring_buffer != nullptr) { // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task // to drain it - reset() is a consumer operation and must run on the inference task's thread. // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. diff --git a/esphome/components/microphone/microphone_source.h b/esphome/components/microphone/microphone_source.h index 7be3b8cdb5..d7a3352432 100644 --- a/esphome/components/microphone/microphone_source.h +++ b/esphome/components/microphone/microphone_source.h @@ -48,7 +48,7 @@ class MicrophoneSource final { template void add_data_callback(F &&data_callback) { this->mic_->add_data_callback([this, data_callback](const std::vector &data) { if (this->enabled_ || this->passive_) { - if (this->processed_samples_.use_count() == 0) { + if (this->processed_samples_ == nullptr) { // Create vector if its unused this->processed_samples_ = std::make_shared>(); } diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 0b79010773..ef21da65c5 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -218,7 +218,7 @@ size_t SourceSpeaker::play(const uint8_t *data, size_t length, TickType_t ticks_ } size_t bytes_written = 0; std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer.use_count() > 0) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); if (bytes_written > 0) { @@ -250,14 +250,14 @@ esp_err_t SourceSpeaker::start_() { // avoids unnecessary single-frame splices. const size_t ring_buffer_size = (this->audio_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; - if (this->audio_source_.use_count() == 0) { + if (this->audio_source_ == nullptr) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); this->ring_buffer_ = temp_ring_buffer; } - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { return ESP_ERR_NO_MEM; } @@ -278,7 +278,7 @@ void SourceSpeaker::stop() { this->send_command_(SOURCE_SPEAKER_COMMAND_STOP); } void SourceSpeaker::finish() { this->send_command_(SOURCE_SPEAKER_COMMAND_FINISH); } bool SourceSpeaker::has_buffered_data() const { - return ((this->audio_source_.use_count() > 0) && this->audio_source_->has_buffered_data()); + return ((this->audio_source_ != nullptr) && this->audio_source_->has_buffered_data()); } void SourceSpeaker::set_mute_state(bool mute_state) { @@ -496,7 +496,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (speaker->is_running() && !speaker->get_pause_state()) { // Speaker is running and not paused, so it possibly can provide audio data std::shared_ptr audio_source = speaker->get_audio_source().lock(); - if (audio_source.use_count() == 0) { + if (audio_source == nullptr) { // No audio source allocated, so skip processing this speaker continue; } diff --git a/esphome/components/resampler/speaker/resampler_speaker.cpp b/esphome/components/resampler/speaker/resampler_speaker.cpp index edda00ae06..16d2d5dc9e 100644 --- a/esphome/components/resampler/speaker/resampler_speaker.cpp +++ b/esphome/components/resampler/speaker/resampler_speaker.cpp @@ -235,7 +235,7 @@ size_t ResamplerSpeaker::play(const uint8_t *data, size_t length, TickType_t tic bytes_written = this->output_speaker_->play(data, length, ticks_to_wait); } else { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { // Only write to the ring buffer if the reference is valid bytes_written = temp_ring_buffer->write_without_replacement(data, length, ticks_to_wait); } else { @@ -299,7 +299,7 @@ bool ResamplerSpeaker::has_buffered_data() const { bool has_ring_buffer_data = false; if (this->requires_resampling_()) { std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); - if (temp_ring_buffer) { + if (temp_ring_buffer != nullptr) { has_ring_buffer_data = (temp_ring_buffer->available() > 0); } } @@ -342,7 +342,7 @@ void ResamplerSpeaker::resample_task(void *params) { std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( this_resampler->audio_stream_info_.ms_to_bytes(this_resampler->buffer_duration_ms_)); - if (!temp_ring_buffer) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { this_resampler->ring_buffer_ = temp_ring_buffer; diff --git a/esphome/components/speaker/media_player/audio_pipeline.cpp b/esphome/components/speaker/media_player/audio_pipeline.cpp index c286a9d7d6..509984cfa2 100644 --- a/esphome/components/speaker/media_player/audio_pipeline.cpp +++ b/esphome/components/speaker/media_player/audio_pipeline.cpp @@ -322,17 +322,17 @@ void AudioPipeline::read_task(void *params) { if (err == ESP_OK) { size_t file_ring_buffer_size = this_pipeline->buffer_size_; - std::shared_ptr temp_ring_buffer; + std::shared_ptr temp_ring_buffer = this_pipeline->raw_file_ring_buffer_.lock(); - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { temp_ring_buffer = ring_buffer::RingBuffer::create(file_ring_buffer_size); this_pipeline->raw_file_ring_buffer_ = temp_ring_buffer; } - if (!this_pipeline->raw_file_ring_buffer_.use_count()) { + if (temp_ring_buffer == nullptr) { err = ESP_ERR_NO_MEM; } else { - reader->add_sink(this_pipeline->raw_file_ring_buffer_); + err = reader->add_sink(temp_ring_buffer); } } @@ -403,7 +403,9 @@ void AudioPipeline::decode_task(void *params) { make_unique(this_pipeline->transfer_buffer_size_, this_pipeline->transfer_buffer_size_); esp_err_t err = decoder->start(this_pipeline->current_audio_file_type_); - decoder->add_source(this_pipeline->raw_file_ring_buffer_); + if (err == ESP_OK) { + err = decoder->add_source(this_pipeline->raw_file_ring_buffer_); + } if (err != ESP_OK) { // Send specific error message From ac79173f4ae83ff10c6a571140091be6ec7878b1 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:05:02 -0400 Subject: [PATCH 26/55] [i2s_audio] Fix spurious driver failure (#19045) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index b78a151ee4..1382a87046 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -118,21 +118,24 @@ void I2SAudioSpeakerBase::loop() { break; } + // Still starting up or winding down from a previous run + if ((this->tx_handle_ != nullptr) || (this->speaker_task_handle_ != nullptr)) { + break; + } + if (this->start_i2s_driver(this->audio_stream_info_) != ESP_OK) { ESP_LOGE(TAG, "Driver failed to start; retrying in 1 second"); this->status_momentary_error("driver-failure", 1000); break; } - if (this->speaker_task_handle_ == nullptr) { - xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, - &this->speaker_task_handle_); + xTaskCreate(I2SAudioSpeakerBase::speaker_task, "speaker_task", TASK_STACK_SIZE, (void *) this, TASK_PRIORITY, + &this->speaker_task_handle_); - if (this->speaker_task_handle_ == nullptr) { - ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); - this->status_momentary_error("task-failure", 1000); - this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt - } + if (this->speaker_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Task failed to start, retrying in 1 second"); + this->status_momentary_error("task-failure", 1000); + this->stop_i2s_driver_(); // Stops the driver to return the lock; will be reloaded in next attempt } break; case speaker::STATE_RUNNING: // Intentional fallthrough From 9c16aba6f78af2657cd9e5876d0e771c72692fcd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:07:06 +0200 Subject: [PATCH 27/55] [noise] Bump noise-c to 0.1.26 and libsodium to 1.10021.8 (#19030) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 4de706120e..d17ebf235e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.24") + cg.add_library("esphome/noise-c", "0.1.26") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.6") + cg.add_library("esphome/libsodium", "1.10021.8") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index 779a05e7de..738773d1b5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.24 ; noise (api, ota) + esphome/noise-c@0.1.26 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.24 ; used by noise (api, ota) + esphome/noise-c@0.1.26 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index 4f7f5a4a4c..00f22ca138 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.26") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.26") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.24\n" + " esphome/noise-c @ 0.1.26\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.24"] + assert libs == ["esphome/noise-c @ 0.1.26"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.24", - "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.26", + "esphome/noise-c @ 0.1.26", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.24"] + assert cls.calls == ["esphome/noise-c @ 0.1.26"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.24"] is None + assert compats["esphome/noise-c @ 0.1.26"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.26"}) cls.deps = { - "esphome/noise-c @ 0.1.24": [ + "esphome/noise-c @ 0.1.26": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.26"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 14c52dda8d..b03bff19a2 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1663,7 +1663,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1683,7 +1683,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.26", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 42fffd16fef3d08f58b5ace59b0545e551c07e64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 23:08:16 +0200 Subject: [PATCH 28/55] [esphome][core] Give a lost OTA chunk ack time to be retransmitted (#19041) --- esphome/components/esphome/ota/ota_esphome.cpp | 5 ++++- esphome/espota2.py | 9 ++++++--- tests/unit_tests/test_espota2.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 1005ed214b..f853ed6a2d 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -41,7 +41,10 @@ const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake -static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer +// Milliseconds for data transfer. Covers the lwIP retransmit run seen in +// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits +// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries +static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/espota2.py b/esphome/espota2.py index ce403c398d..c683ffa323 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -96,6 +96,10 @@ UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # across the addresses on top of that. EXTRA_UPLOAD_ATTEMPTS = 2 UPLOAD_RETRY_DELAY = 5.0 +# Data phase timeout; must stay longer than the device's OTA_SOCKET_TIMEOUT_DATA +# (105 s) so a stalled session is gone before a retry, and long enough for lwIP +# to get a lost chunk ack through after the retransmit run seen in practice +DATA_PHASE_TIMEOUT = 160.0 _LOGGER = logging.getLogger(__name__) @@ -694,8 +698,7 @@ def perform_ota( _LOGGER.info("Handshake complete") - # Timeout must match device-side OTA_SOCKET_TIMEOUT_DATA to prevent premature failures - sock.settimeout(90.0) + sock.settimeout(DATA_PHASE_TIMEOUT) if extended_proto: send_check(sock, ota_type, "ota type") @@ -854,7 +857,7 @@ def run_ota_impl_( # clean up a half-open connection (its handshake watchdog runs at 20s); # moving on to the next address family stays immediate. Known limitation: # a silent mid-transfer drop with no reset can wedge the device until its - # 90s data timeout, which outlasts this budget; the retries target the + # 105s data timeout, which outlasts this budget; the retries target the # common failures where the device resets or closes the link promptly. total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 8867e2c215..2d65e8e079 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -416,6 +416,9 @@ def test_perform_ota_no_auth( "Update took 14.00 seconds (prepare 2.00, upload 5.00, commit 7.00)" in caplog.text ) + # The data phase timeout must outlast the device's 105 s data timeout + mock_socket.settimeout.assert_any_call(espota2.DATA_PHASE_TIMEOUT) + assert espota2.DATA_PHASE_TIMEOUT > 105.0 @pytest.mark.usefixtures("mock_time") From d2bc056f0ab3ea93e62c7ca468f7a3da17a3f421 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 8 Sep 2026 17:31:18 -0400 Subject: [PATCH 29/55] [sendspin] Add codec preference list to the media source (#19047) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/sendspin/__init__.py | 26 ++++-- .../sendspin/media_source/__init__.py | 31 +++++++ .../sendspin/test_media_source.py | 90 +++++++++++++++++++ .../sendspin/common-media_source.yaml | 1 + 4 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/component_tests/sendspin/test_media_source.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 570fd3fadd..8ef11a7f90 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -30,6 +30,7 @@ CONF_SENDSPIN_ID = "sendspin_id" CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" +CONF_CODECS = "codecs" # Matches ARTWORK_MAX_SLOTS in sendspin-cpp. MAX_ARTWORK_SLOTS = 4 @@ -44,6 +45,20 @@ CODEC_FORMAT_OPUS = SendspinCodecFormat.enum("OPUS") CODEC_FORMAT_PCM = SendspinCodecFormat.enum("PCM") CODEC_FORMAT_UNSUPPORTED = SendspinCodecFormat.enum("UNSUPPORTED") +CODEC_FLAC = "flac" +CODEC_OPUS = "opus" +CODEC_PCM = "pcm" + +CODECS = { + CODEC_FLAC: CODEC_FORMAT_FLAC, + CODEC_OPUS: CODEC_FORMAT_OPUS, + CODEC_PCM: CODEC_FORMAT_PCM, +} + +# Opus only supports 48 kHz audio, so it is left out of the default list at other rates. +DEFAULT_CODECS = [CODEC_FLAC, CODEC_OPUS, CODEC_PCM] +OPUS_SAMPLE_RATE = 48000 + SendspinImageFormat = sendspin_library_ns.enum("SendspinImageFormat", is_class=True) IMAGE_FORMAT_JPEG = SendspinImageFormat.enum("JPEG") IMAGE_FORMAT_PNG = SendspinImageFormat.enum("PNG") @@ -286,16 +301,13 @@ async def to_code(config: ConfigType) -> None: if data.player_support: cg.add_define("USE_SENDSPIN_PLAYER", True) - # Configures the player role. We always assume support for 16 bits per sample mono and stereo FLAC, Opus, and PCM at the configured sample rate - # (with Opus only supported at 48 kHz since that's the only sample rate it supports). Users can configure the specific formats via the Sendspin server + # Configures the player role. Each configured codec is advertised for 16 bits per sample + # mono and stereo at the configured sample rate. The order is a preference order, both for + # the codecs themselves and for stereo over mono. player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - # OPUS only supports 48 kHz audio - codecs = [CODEC_FORMAT_FLAC] - if sample_rate == 48000: - codecs.append(CODEC_FORMAT_OPUS) - codecs.append(CODEC_FORMAT_PCM) + codecs = player_cfg[CONF_CODECS] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index 6af244d41f..6a9f1f18ba 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -13,11 +13,16 @@ from esphome.cpp_generator import MockObj, TemplateArgsType from esphome.types import ConfigType from .. import ( + CODEC_OPUS, + CODECS, + CONF_CODECS, CONF_DECODE_MEMORY, CONF_FIXED_DELAY, CONF_INITIAL_STATIC_DELAY, CONF_SENDSPIN_ID, + DEFAULT_CODECS, MEMORY_LOCATIONS, + OPUS_SAMPLE_RATE, SendspinHub, register_player_config, request_controller_support, @@ -49,10 +54,32 @@ DisableStaticDelayAdjustmentAction = sendspin_ns.class_( ) +def _resolve_codecs(config: ConfigType) -> ConfigType: + """Validate the codec preference list, filling in the default when it is not set.""" + sample_rate = config[CONF_SAMPLE_RATE] + if (codecs := config.get(CONF_CODECS)) is None: + config[CONF_CODECS] = [ + codec + for codec in DEFAULT_CODECS + if codec != CODEC_OPUS or sample_rate == OPUS_SAMPLE_RATE + ] + return config + + if len(set(codecs)) != len(codecs): + raise cv.Invalid("Each codec may only be listed once", path=[CONF_CODECS]) + if CODEC_OPUS in codecs and sample_rate != OPUS_SAMPLE_RATE: + raise cv.Invalid( + f"Codec '{CODEC_OPUS}' requires a {CONF_SAMPLE_RATE} of {OPUS_SAMPLE_RATE}", + path=[CONF_CODECS], + ) + return config + + def _register(config: ConfigType) -> ConfigType: request_controller_support() register_player_config( { + CONF_CODECS: config[CONF_CODECS], CONF_SAMPLE_RATE: config[CONF_SAMPLE_RATE], CONF_BUFFER_SIZE: config[CONF_BUFFER_SIZE], CONF_INITIAL_STATIC_DELAY: config[CONF_INITIAL_STATIC_DELAY], @@ -85,9 +112,13 @@ CONFIG_SCHEMA = cv.All( min=16000, max=96000 ), cv.Optional(CONF_DECODE_MEMORY): cv.one_of(*MEMORY_LOCATIONS, lower=True), + cv.Optional(CONF_CODECS): cv.All( + cv.ensure_list(cv.enum(CODECS, lower=True)), cv.Length(min=1) + ), } ), cv.only_on_esp32, + _resolve_codecs, _register, ) diff --git a/tests/component_tests/sendspin/test_media_source.py b/tests/component_tests/sendspin/test_media_source.py new file mode 100644 index 0000000000..6c2f79198d --- /dev/null +++ b/tests/component_tests/sendspin/test_media_source.py @@ -0,0 +1,90 @@ +"""Validation tests for the sendspin media_source platform. + +These cover the codec preference list, whose rejection branches a compile test +cannot reach: a `test*.yaml` can only assert that a configuration is accepted. +""" + +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import CONF_CODECS, _get_data +from esphome.components.sendspin.media_source import CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _media_source_config(**overrides: Any) -> ConfigType: + """Build a minimal valid media source config, allowing field overrides.""" + config: ConfigType = { + "id": "sendspin_media_source", + "sendspin_id": "sendspin_hub", + } + config.update(overrides) + return config + + +def test_default_codecs_at_48_khz(set_core_config: SetCoreConfigCallable) -> None: + """Every codec is advertised when the sample rate suits all of them.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config()) + + assert config[CONF_CODECS] == ["flac", "opus", "pcm"] + + +def test_default_codecs_drop_opus_at_other_rates( + set_core_config: SetCoreConfigCallable, +) -> None: + """Opus only supports 48 kHz, so it leaves the default list at other rates.""" + set_core_config(PlatformFramework.ESP32_IDF) + + config = CONFIG_SCHEMA(_media_source_config(sample_rate=44100)) + + assert config[CONF_CODECS] == ["flac", "pcm"] + + +def test_configured_order_is_preserved(set_core_config: SetCoreConfigCallable) -> None: + """The list is a preference order, so it reaches the player role as written.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA(_media_source_config(codecs=["pcm", "flac"])) + + assert _get_data().player_config[CONF_CODECS] == ["pcm", "flac"] + + +def test_empty_codec_list_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A player with no codecs at all could never be given a stream.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="length of value must be at least 1"): + CONFIG_SCHEMA(_media_source_config(codecs=[])) + + +def test_duplicate_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """A repeated codec has no meaning in a preference order.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="may only be listed once"): + CONFIG_SCHEMA(_media_source_config(codecs=["flac", "flac"])) + + +def test_unknown_codec_rejected(set_core_config: SetCoreConfigCallable) -> None: + """Only codecs the player role can decode are accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="Unknown value"): + CONFIG_SCHEMA(_media_source_config(codecs=["mp3"])) + + +def test_opus_at_wrong_sample_rate_rejected( + set_core_config: SetCoreConfigCallable, +) -> None: + """Asking for Opus at a rate it cannot handle fails rather than silently + dropping the stated preference.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid, match="requires a sample_rate of 48000"): + CONFIG_SCHEMA(_media_source_config(codecs=["opus"], sample_rate=44100)) diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 1977b79c04..0c136fbd43 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,3 +9,4 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal + codecs: [pcm, opus, flac] From 008677298ada0a95adeef14c5ac88d1895d5fb2f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:54:44 +1200 Subject: [PATCH 30/55] Bump version to 2026.9.0b3 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 060de51d3a..97ce92240c 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b2 +PROJECT_NUMBER = 2026.9.0b3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 287804ace3..b013098f33 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b2" +__version__ = "2026.9.0b3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From fd598057efdfa689a10b53329753a936a15016a1 Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 9 Sep 2026 14:22:23 +0200 Subject: [PATCH 31/55] [sendspin] Fix codec enum codegen when codecs is not set (#19055) --- esphome/components/sendspin/__init__.py | 2 +- tests/components/sendspin/common-media_source.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 8ef11a7f90..c1970ab132 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -307,7 +307,7 @@ async def to_code(config: ConfigType) -> None: player_cfg = data.player_config sample_rate = player_cfg[CONF_SAMPLE_RATE] - codecs = player_cfg[CONF_CODECS] + codecs = [CODECS[codec] for codec in player_cfg[CONF_CODECS]] def _audio_format(codec: MockObj, channels: int) -> cg.StructInitializer: return cg.StructInitializer( diff --git a/tests/components/sendspin/common-media_source.yaml b/tests/components/sendspin/common-media_source.yaml index 0c136fbd43..1977b79c04 100644 --- a/tests/components/sendspin/common-media_source.yaml +++ b/tests/components/sendspin/common-media_source.yaml @@ -9,4 +9,3 @@ media_source: static_delay_adjustable: true fixed_delay: 480us decode_memory: internal - codecs: [pcm, opus, flac] From 58ca3456845ca130ac106796225e8ea4cb9c5107 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:42:09 -0400 Subject: [PATCH 32/55] [ci] Refresh integration test durations (#19049) --- .../integration_test_durations.json | 293 +++++++++--------- 1 file changed, 152 insertions(+), 141 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index 5a5aac3b22..b4a7f4e1ae 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,143 +1,154 @@ { - "tests/integration/test_action_concurrent_reentry.py": 57.91, - "tests/integration/test_addressable_light_transition.py": 21.25, - "tests/integration/test_alarm_control_panel_state_transitions.py": 70.71, - "tests/integration/test_api_action_metadata.py": 66.6, - "tests/integration/test_api_action_responses.py": 36.1, - "tests/integration/test_api_action_timeout.py": 68.86, - "tests/integration/test_api_conditional_memory.py": 15.48, - "tests/integration/test_api_custom_services.py": 18.77, - "tests/integration/test_api_get_time_response_timezone.py": 21.08, - "tests/integration/test_api_homeassistant.py": 65.59, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 18.44, - "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 15.05, - "tests/integration/test_api_list_entities_backpressure.py": 13.88, - "tests/integration/test_api_message_size_batching.py": 29.98, - "tests/integration/test_api_reboot_timeout.py": 16.05, - "tests/integration/test_api_string_lambda.py": 15.31, - "tests/integration/test_api_vv_logging.py": 19.28, - "tests/integration/test_api_zero_psk_provisioning.py": 31.5, - "tests/integration/test_areas_and_devices.py": 24.95, - "tests/integration/test_automation_wait_actions.py": 20.92, - "tests/integration/test_automations.py": 35.19, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 17.99, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 20.39, - "tests/integration/test_binary_sensor_invalidate_state.py": 18.41, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 24.69, - "tests/integration/test_build_info.py": 18.7, - "tests/integration/test_camera_mock.py": 16.23, - "tests/integration/test_climate_control_action.py": 21.14, - "tests/integration/test_climate_custom_modes.py": 20.74, - "tests/integration/test_continuation_actions.py": 16.81, - "tests/integration/test_cover_control_action.py": 20.34, - "tests/integration/test_crc8_helper.py": 9.36, - "tests/integration/test_device_id_in_state.py": 44.67, - "tests/integration/test_duplicate_entities.py": 23.58, - "tests/integration/test_entity_icon.py": 34.35, - "tests/integration/test_fan_turn_on_action.py": 24.23, - "tests/integration/test_fnv1_hash_object_id.py": 16.21, - "tests/integration/test_fnv1a_hash.py": 13.38, - "tests/integration/test_gpio_expander_cache.py": 13.06, - "tests/integration/test_host_logger_thread_safety.py": 23.66, - "tests/integration/test_host_mode_basic.py": 8.01, - "tests/integration/test_host_mode_batch_delay.py": 21.0, - "tests/integration/test_host_mode_climate_basic_state.py": 22.14, - "tests/integration/test_host_mode_climate_control.py": 19.39, - "tests/integration/test_host_mode_empty_string_options.py": 21.76, - "tests/integration/test_host_mode_entity_fields.py": 29.61, - "tests/integration/test_host_mode_fan_preset.py": 20.01, - "tests/integration/test_host_mode_many_entities.py": 39.08, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.92, - "tests/integration/test_host_mode_noise_encryption.py": 42.42, - "tests/integration/test_host_mode_reconnect.py": 3.41, - "tests/integration/test_host_mode_sensor.py": 22.96, - "tests/integration/test_host_ota.py": 29.5, - "tests/integration/test_host_preferences.py": 16.06, - "tests/integration/test_host_preferences_suspend_resume.py": 18.71, - "tests/integration/test_improv_serial_uart.py": 20.22, - "tests/integration/test_large_message_batching.py": 26.56, - "tests/integration/test_legacy_area.py": 22.72, - "tests/integration/test_legacy_climate_compat.py": 14.13, - "tests/integration/test_legacy_fan_compat.py": 14.33, - "tests/integration/test_light_automations.py": 18.81, - "tests/integration/test_light_binary_effect_off_phase.py": 8.38, - "tests/integration/test_light_calls.py": 21.88, - "tests/integration/test_light_constant_brightness.py": 59.45, - "tests/integration/test_light_control_action.py": 31.91, - "tests/integration/test_light_dim_relative_action.py": 14.43, - "tests/integration/test_light_effect_zero_brightness.py": 25.05, - "tests/integration/test_light_initial_state.py": 18.97, - "tests/integration/test_light_toggle_action.py": 17.44, - "tests/integration/test_lock_automations.py": 18.9, - "tests/integration/test_logger_buffered_recursion_guard.py": 18.2, - "tests/integration/test_loop_disable_enable.py": 63.35, - "tests/integration/test_loop_interval_decoupling.py": 17.7, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.56, - "tests/integration/test_micros_to_millis.py": 15.89, - "tests/integration/test_multi_click_trigger.py": 17.23, - "tests/integration/test_multi_device_preferences.py": 19.4, - "tests/integration/test_noise_encryption_key_protection.py": 72.59, - "tests/integration/test_object_id_api_verification.py": 19.22, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.77, - "tests/integration/test_object_id_no_friendly_name.py": 45.8, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 86.73, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.4, - "tests/integration/test_online_image_bmp.py": 37.24, - "tests/integration/test_oversized_payloads.py": 55.75, - "tests/integration/test_preference_key_stability.py": 25.49, - "tests/integration/test_runtime_stats.py": 29.81, - "tests/integration/test_safe_mode_loop_runs.py": 6.26, - "tests/integration/test_scheduler_blocking_warning.py": 37.98, - "tests/integration/test_scheduler_bulk_cleanup.py": 18.67, - "tests/integration/test_scheduler_defer_cancel.py": 18.46, - "tests/integration/test_scheduler_defer_cancel_regular.py": 16.34, - "tests/integration/test_scheduler_defer_fifo_simple.py": 18.26, - "tests/integration/test_scheduler_defer_stress.py": 17.74, - "tests/integration/test_scheduler_heap_stress.py": 3.89, - "tests/integration/test_scheduler_internal_id_no_collision.py": 20.01, - "tests/integration/test_scheduler_interval_reschedule.py": 16.29, - "tests/integration/test_scheduler_interval_zero_coerced.py": 16.09, - "tests/integration/test_scheduler_null_name.py": 14.69, - "tests/integration/test_scheduler_numeric_id_test.py": 17.08, - "tests/integration/test_scheduler_pool.py": 19.88, - "tests/integration/test_scheduler_rapid_cancellation.py": 4.42, - "tests/integration/test_scheduler_recursive_timeout.py": 4.3, - "tests/integration/test_scheduler_removed_item_race.py": 15.49, - "tests/integration/test_scheduler_self_keyed.py": 25.77, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 14.84, - "tests/integration/test_scheduler_string_test.py": 15.42, - "tests/integration/test_script_array_params.py": 12.73, - "tests/integration/test_script_delay_params.py": 12.69, - "tests/integration/test_script_queued.py": 20.38, - "tests/integration/test_script_queued_idle_loop.py": 25.06, - "tests/integration/test_script_wait_on_boot.py": 15.67, - "tests/integration/test_select_stringref_trigger.py": 19.48, - "tests/integration/test_sensor_filters_delta.py": 27.62, - "tests/integration/test_sensor_filters_ring_buffer.py": 20.27, - "tests/integration/test_sensor_filters_sliding_window.py": 56.28, - "tests/integration/test_sensor_filters_value_list.py": 20.6, - "tests/integration/test_sensor_timeout_filter.py": 22.21, - "tests/integration/test_socket_wake_gate_tcp.py": 16.37, - "tests/integration/test_status_flags.py": 29.68, - "tests/integration/test_strftime_to.py": 17.42, - "tests/integration/test_syslog.py": 18.39, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 25.61, - "tests/integration/test_template_text_save.py": 19.16, - "tests/integration/test_text_command.py": 16.43, - "tests/integration/test_text_sensor_raw_state.py": 17.19, - "tests/integration/test_uart_mock_ld2410.py": 37.0, - "tests/integration/test_uart_mock_ld2412.py": 40.82, - "tests/integration/test_uart_mock_ld2420.py": 32.7, - "tests/integration/test_uart_mock_ld2450.py": 32.84, - "tests/integration/test_uart_mock_modbus.py": 548.87, - "tests/integration/test_udp.py": 16.67, - "tests/integration/test_use_address_runtime.py": 27.26, - "tests/integration/test_valve_control_action.py": 24.58, - "tests/integration/test_varint_five_byte_device_id.py": 22.5, - "tests/integration/test_wait_until_mid_loop_timing.py": 22.05, - "tests/integration/test_wait_until_on_boot.py": 10.37, - "tests/integration/test_wait_until_ordering.py": 18.23, - "tests/integration/test_wait_until_reentrant_restart.py": 19.35, - "tests/integration/test_wake_loop_forces_phase_b.py": 17.83, - "tests/integration/test_water_heater_template.py": 25.7 + "tests/integration/test_action_concurrent_reentry.py": 30.48, + "tests/integration/test_addressable_light_transition.py": 42.1, + "tests/integration/test_alarm_control_panel_state_transitions.py": 35.76, + "tests/integration/test_api_action_metadata.py": 22.35, + "tests/integration/test_api_action_responses.py": 30.31, + "tests/integration/test_api_action_timeout.py": 34.73, + "tests/integration/test_api_conditional_memory.py": 18.35, + "tests/integration/test_api_custom_services.py": 15.99, + "tests/integration/test_api_get_time_response_timezone.py": 24.21, + "tests/integration/test_api_homeassistant.py": 33.77, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 20.8, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 23.55, + "tests/integration/test_api_list_entities_backpressure.py": 23.04, + "tests/integration/test_api_message_size_batching.py": 27.31, + "tests/integration/test_api_reboot_timeout.py": 29.32, + "tests/integration/test_api_string_lambda.py": 14.9, + "tests/integration/test_api_vv_logging.py": 26.25, + "tests/integration/test_api_zero_psk_provisioning.py": 38.19, + "tests/integration/test_areas_and_devices.py": 27.52, + "tests/integration/test_automation_wait_actions.py": 24.25, + "tests/integration/test_automations.py": 36.02, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 18.46, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 17.47, + "tests/integration/test_binary_sensor_invalidate_state.py": 14.79, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 21.52, + "tests/integration/test_build_info.py": 21.42, + "tests/integration/test_camera_mock.py": 17.02, + "tests/integration/test_climate_control_action.py": 26.56, + "tests/integration/test_climate_custom_modes.py": 18.82, + "tests/integration/test_continuation_actions.py": 20.39, + "tests/integration/test_cover_control_action.py": 19.91, + "tests/integration/test_crc8_helper.py": 16.73, + "tests/integration/test_device_id_in_state.py": 58.41, + "tests/integration/test_duplicate_entities.py": 30.76, + "tests/integration/test_entity_icon.py": 25.34, + "tests/integration/test_fan_turn_on_action.py": 23.64, + "tests/integration/test_fnv1_hash_object_id.py": 25.44, + "tests/integration/test_fnv1a_hash.py": 20.85, + "tests/integration/test_gpio_expander_cache.py": 14.42, + "tests/integration/test_host_logger_thread_safety.py": 21.31, + "tests/integration/test_host_mode_basic.py": 2.65, + "tests/integration/test_host_mode_batch_delay.py": 22.21, + "tests/integration/test_host_mode_climate_basic_state.py": 27.12, + "tests/integration/test_host_mode_climate_control.py": 21.57, + "tests/integration/test_host_mode_empty_string_options.py": 27.17, + "tests/integration/test_host_mode_entity_fields.py": 30.1, + "tests/integration/test_host_mode_fan_preset.py": 17.55, + "tests/integration/test_host_mode_many_entities.py": 38.98, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.82, + "tests/integration/test_host_mode_noise_encryption.py": 39.84, + "tests/integration/test_host_mode_reconnect.py": 13.1, + "tests/integration/test_host_mode_sensor.py": 22.17, + "tests/integration/test_host_ota.py": 92.05, + "tests/integration/test_host_preferences.py": 20.29, + "tests/integration/test_host_preferences_suspend_resume.py": 15.02, + "tests/integration/test_improv_serial_uart.py": 30.15, + "tests/integration/test_large_message_batching.py": 25.84, + "tests/integration/test_legacy_area.py": 21.24, + "tests/integration/test_legacy_climate_compat.py": 17.34, + "tests/integration/test_legacy_fan_compat.py": 22.6, + "tests/integration/test_light_automations.py": 29.13, + "tests/integration/test_light_binary_effect_off_phase.py": 33.99, + "tests/integration/test_light_calls.py": 26.81, + "tests/integration/test_light_constant_brightness.py": 25.0, + "tests/integration/test_light_control_action.py": 25.57, + "tests/integration/test_light_dim_relative_action.py": 21.4, + "tests/integration/test_light_effect_zero_brightness.py": 19.65, + "tests/integration/test_light_initial_state.py": 17.58, + "tests/integration/test_light_toggle_action.py": 28.28, + "tests/integration/test_lock_automations.py": 23.3, + "tests/integration/test_logger_buffered_recursion_guard.py": 22.96, + "tests/integration/test_loop_disable_enable.py": 16.18, + "tests/integration/test_loop_interval_decoupling.py": 25.19, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 20.59, + "tests/integration/test_lvgl_headless_render.py": 87.78, + "tests/integration/test_micros_to_millis.py": 18.73, + "tests/integration/test_multi_click_trigger.py": 24.2, + "tests/integration/test_multi_device_preferences.py": 20.52, + "tests/integration/test_noise_encryption_key_protection.py": 19.1, + "tests/integration/test_object_id_api_verification.py": 26.24, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 14.88, + "tests/integration/test_object_id_no_friendly_name.py": 61.27, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 82.32, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 46.03, + "tests/integration/test_online_image_bmp.py": 34.21, + "tests/integration/test_oversized_payloads.py": 62.75, + "tests/integration/test_preference_key_stability.py": 26.8, + "tests/integration/test_runtime_stats.py": 28.26, + "tests/integration/test_safe_mode_loop_runs.py": 18.14, + "tests/integration/test_scheduler_blocking_warning.py": 28.7, + "tests/integration/test_scheduler_bulk_cleanup.py": 20.73, + "tests/integration/test_scheduler_defer_cancel.py": 22.99, + "tests/integration/test_scheduler_defer_cancel_regular.py": 21.97, + "tests/integration/test_scheduler_defer_fifo_simple.py": 24.15, + "tests/integration/test_scheduler_defer_stress.py": 23.11, + "tests/integration/test_scheduler_heap_stress.py": 20.2, + "tests/integration/test_scheduler_internal_id_no_collision.py": 23.75, + "tests/integration/test_scheduler_interval_reschedule.py": 15.32, + "tests/integration/test_scheduler_interval_zero_coerced.py": 20.1, + "tests/integration/test_scheduler_null_name.py": 17.43, + "tests/integration/test_scheduler_numeric_id_test.py": 25.51, + "tests/integration/test_scheduler_pool.py": 24.22, + "tests/integration/test_scheduler_rapid_cancellation.py": 24.01, + "tests/integration/test_scheduler_recursive_timeout.py": 22.94, + "tests/integration/test_scheduler_removed_item_race.py": 23.07, + "tests/integration/test_scheduler_self_keyed.py": 18.43, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 21.99, + "tests/integration/test_scheduler_string_test.py": 17.27, + "tests/integration/test_script_array_params.py": 4.59, + "tests/integration/test_script_delay_params.py": 22.46, + "tests/integration/test_script_queued.py": 25.24, + "tests/integration/test_script_queued_idle_loop.py": 5.04, + "tests/integration/test_script_wait_on_boot.py": 21.77, + "tests/integration/test_sdl_headless_screenshot.py": 19.23, + "tests/integration/test_select_stringref_trigger.py": 19.31, + "tests/integration/test_sensor_filters_delta.py": 25.92, + "tests/integration/test_sensor_filters_ring_buffer.py": 22.39, + "tests/integration/test_sensor_filters_sliding_window.py": 57.93, + "tests/integration/test_sensor_filters_value_list.py": 20.32, + "tests/integration/test_sensor_timeout_filter.py": 25.35, + "tests/integration/test_snapshot_display.py": 19.7, + "tests/integration/test_socket_wake_gate_tcp.py": 14.5, + "tests/integration/test_status_flags.py": 33.83, + "tests/integration/test_strftime_to.py": 17.64, + "tests/integration/test_syslog.py": 24.49, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 24.81, + "tests/integration/test_template_climate_basic.py": 15.28, + "tests/integration/test_template_climate_custom_modes.py": 25.07, + "tests/integration/test_template_climate_nonoptimistic.py": 24.25, + "tests/integration/test_template_climate_on_control_ordering.py": 24.09, + "tests/integration/test_template_climate_publish_all_fields.py": 17.78, + "tests/integration/test_template_climate_sensor_push.py": 17.42, + "tests/integration/test_template_climate_set_actions.py": 23.63, + "tests/integration/test_template_climate_two_point_temperature.py": 25.19, + "tests/integration/test_template_text_save.py": 17.88, + "tests/integration/test_text_command.py": 22.71, + "tests/integration/test_text_sensor_raw_state.py": 25.17, + "tests/integration/test_uart_mock_ld2410.py": 58.15, + "tests/integration/test_uart_mock_ld2412.py": 61.14, + "tests/integration/test_uart_mock_ld2420.py": 33.87, + "tests/integration/test_uart_mock_ld2450.py": 26.06, + "tests/integration/test_uart_mock_modbus.py": 391.79, + "tests/integration/test_udp.py": 7.38, + "tests/integration/test_use_address_runtime.py": 24.09, + "tests/integration/test_valve_control_action.py": 23.22, + "tests/integration/test_varint_five_byte_device_id.py": 17.93, + "tests/integration/test_wait_until_mid_loop_timing.py": 22.26, + "tests/integration/test_wait_until_on_boot.py": 17.46, + "tests/integration/test_wait_until_ordering.py": 11.89, + "tests/integration/test_wait_until_reentrant_restart.py": 22.88, + "tests/integration/test_wake_loop_forces_phase_b.py": 16.6, + "tests/integration/test_water_heater_template.py": 19.66 } From c66fa812086f20aead9e56bf42b15f541b2e5bc8 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:49 +1200 Subject: [PATCH 33/55] [core] Consolidate setup scripts into a cross-platform setup.py (#18856) --- script/git-hooks/post-checkout | 46 ++- script/setup | 74 +---- script/setup.bat | 29 +- script/setup.py | 222 +++++++++++++ tests/script/test_setup.py | 562 +++++++++++++++++++++++++++++++++ 5 files changed, 827 insertions(+), 106 deletions(-) create mode 100755 script/setup.py create mode 100644 tests/script/test_setup.py diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 853c2b0352..73c1cb0f13 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -1,27 +1,49 @@ #!/bin/sh # Prepare the dev environment for a new checkout or worktree. # -# Installed into the git hooks directory by script/setup. Deliberately tiny and -# self-contained: it stays valid on branches where script/setup does not exist, -# and simply does nothing there. +# Installed into the git hooks directory by script/setup.py. Deliberately tiny +# and self-contained: it stays valid on branches where the setup script does not +# exist, and simply does nothing there. # $3 is 1 for a branch checkout, 0 for a file checkout. [ "$3" = "1" ] || exit 0 top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 -# This also runs on ordinary branch switches, where there is nothing to do. +# This also runs on ordinary branch switches, where there is nothing to do. Both +# layouts are checked because git for Windows runs hooks under its own bundled +# shell, where the environment lives in venv/Scripts rather than venv/bin. [ -x "$top/venv/bin/python" ] && exit 0 -[ -x "$top/script/setup" ] || exit 0 +[ -f "$top/venv/Scripts/python.exe" ] && exit 0 + +# Branches from before the setup script moved to Python carry only the shell +# entry point, so whichever one the checked out branch has is used. +py= +if [ -f "$top/script/setup.py" ]; then + # The interpreter goes by different names across platforms, and on Windows + # "python3" is often a stub that opens the app store instead of running + # anything, so each candidate is tried before it is used. Doing nothing is the + # right outcome when none of them work. + for candidate in "python3" "python" "py -3"; do + # Unquoted on purpose: the launcher candidate is a command plus a flag. + if $candidate -c "" >/dev/null 2>&1; then + py=$candidate + break + fi + done + [ -n "$py" ] || exit 0 +elif ! [ -x "$top/script/setup" ]; then + exit 0 +fi # Every worktree shares the hooks directory of the checkout it was created -# from, and the script/setup run below is the one from whichever branch was just +# from, and the setup script run below is the one from whichever branch was just # checked out. Older branches install their own pre-commit hook without checking # for a worktree: that moves the shared hook aside as pre-commit.legacy and # replaces it with one tied to this worktree's virtual environment, so commits # break in every checkout. To rule that out, the hooks directory is copied -# before script/setup runs and put back exactly as it was afterwards, including -# removing any file script/setup added. +# before the setup script runs and put back exactly as it was afterwards, +# including removing any file the setup script added. hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 cp -p "$hooks"/* "$snap"/ 2>/dev/null @@ -29,7 +51,13 @@ cp -p "$hooks"/* "$snap"/ 2>/dev/null # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -env -u VIRTUAL_ENV "$top/script/setup" +unset VIRTUAL_ENV +if [ -n "$py" ]; then + # Unquoted on purpose, as above. + $py "$top/script/setup.py" +else + "$top/script/setup" +fi status=$? for f in "$hooks"/*; do diff --git a/script/setup b/script/setup index b96af6e8f3..91bcb88154 100755 --- a/script/setup +++ b/script/setup @@ -1,71 +1,7 @@ #!/usr/bin/env bash -# Set up ESPHome dev environment +# Set up ESPHome dev environment. +# +# The work is done by setup.py, which script/setup.bat also runs, so the Unix +# and Windows entry points share one implementation. -set -e - -cd "$(dirname "$0")/.." -if [ -n "$VIRTUAL_ENV" ]; then - # A virtual environment is already active (e.g. the devcontainer's pre-provisioned - # esphome-venv). Install into it rather than creating a ./venv in the workspace. - venv_state=active -elif [ -x venv/bin/python ]; then - # Reuse the environment from an earlier run, so this script can be run again - # at any time to pick up dependency changes. - venv_state=reused - source venv/bin/activate -else - venv_state=created - # --clear replaces a partial environment left behind by an interrupted run. - if [ -x "$(command -v uv)" ]; then - uv venv --clear --seed venv - else - python3 -m venv --clear venv - fi - source venv/bin/activate -fi - -if ! [ -x "$(command -v uv)" ]; then - python3 -m pip install uv -fi - -uv pip install setuptools wheel -uv pip install -e ".[dev,test]" --config-settings editable_mode=compat - -# A worktree shares one git hooks directory with the main checkout it was -# created from, so hooks are installed from the main checkout only. Installing -# from a worktree would point the shared hook at that worktree's virtual -# environment, breaking it for everyone once the worktree is removed. -git_dir="$(git rev-parse --absolute-git-dir 2>/dev/null || true)" -common_dir="$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" -if [ -n "$common_dir" ] && [ "$git_dir" = "$common_dir" ]; then - # --overwrite replaces any hook already in place. Without it, prek finds a - # previously installed pre-commit hook, moves it aside to - # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would - # run both tools. - prek install --overwrite - - # Prepares the virtual environment for new checkouts and worktrees. Installed - # once here, it covers every worktree created from this checkout. - if [ -d "$common_dir/hooks" ]; then - cp script/git-hooks/post-checkout "$common_dir/hooks/post-checkout" - chmod +x "$common_dir/hooks/post-checkout" - fi -fi - -mkdir -p .temp - -echo -echo -case "$venv_state" in - created) - echo "Virtual environment created at ./venv. Run 'source venv/bin/activate' to use it." - ;; - reused) - echo "Dependencies updated in the existing ./venv. Run 'source venv/bin/activate' to use it." - ;; - active) - echo "Dependencies installed into the active virtual environment:" - echo " $VIRTUAL_ENV" - echo "It is already active in this shell, so no 'source venv/bin/activate' is needed." - ;; -esac +exec python3 "$(dirname "$0")/setup.py" "$@" diff --git a/script/setup.bat b/script/setup.bat index 809d05ae93..405121b139 100644 --- a/script/setup.bat +++ b/script/setup.bat @@ -1,28 +1 @@ -@echo off - -if defined VIRTUAL_ENV goto :install - -echo Starting the Virtual Environment -python -m venv venv -call venv/Scripts/activate -echo Running the Virtual Environment - -:install - -echo Installing required packages... - -python.exe -m pip install --upgrade pip - -pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.txt -pip3 install setuptools wheel -pip3 install -e ".[dev,test]" --config-settings editable_mode=compat - -rem --overwrite replaces any hook already in place. Without it, prek finds a -rem previously installed pre-commit hook, moves it aside to -rem .git/hooks/pre-commit.legacy and keeps calling it, so every commit would -rem run both tools. -prek install --overwrite - -echo . -echo . -echo Virtual environment created. Run 'venv/Scripts/activate' to use it. +@python "%~dp0setup.py" %* diff --git a/script/setup.py b/script/setup.py new file mode 100755 index 0000000000..62129b8c05 --- /dev/null +++ b/script/setup.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Set up the ESPHome development environment. + +Shared implementation behind script/setup and script/setup.bat, so the Unix and +Windows entry points cannot drift apart. Uses only the standard library: it runs +before any dependency has been installed. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig + +MIN_PYTHON = (3, 12) + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_VENV = ROOT / "venv" +POST_CHECKOUT_HOOK = ROOT / "script" / "git-hooks" / "post-checkout" + +# State of the environment the dependencies end up in, used for the closing +# message. +VENV_ACTIVE = "active" +VENV_REUSED = "reused" +VENV_CREATED = "created" + + +def bin_dir(venv: Path) -> Path: + """Return the directory holding a virtual environment's executables. + + The "venv" scheme resolves to bin on Unix and Scripts on Windows, so the + layout does not have to be hardcoded here. + """ + base = str(venv) + return Path( + sysconfig.get_path("scripts", "venv", vars={"base": base, "platbase": base}) + ) + + +def venv_python(venv: Path) -> Path: + """Return the path to a virtual environment's interpreter.""" + name = "python.exe" if os.name == "nt" else "python" + return bin_dir(venv) / name + + +def run(command: list[str], env: dict[str, str] | None = None) -> None: + """Run a command, aborting the whole script if it fails.""" + print(f"+ {' '.join(command)}", flush=True) + result = subprocess.run(command, cwd=ROOT, env=env, check=False) + if result.returncode != 0: + # Some tools fail without printing anything, so name the step that broke. + print( + f"Failed with exit code {result.returncode}: {command[0]}", file=sys.stderr + ) + raise SystemExit(result.returncode) + + +def git_output(*args: str) -> str: + """Return the trimmed output of a git command, or "" if it cannot be run.""" + try: + result = subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False + ) + except OSError: + # Git is not required to install the dependencies, only to install hooks. + return "" + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def create_venv(venv: Path) -> None: + """Create a virtual environment, replacing anything already at the path.""" + # --clear replaces a partial environment left behind by an interrupted run. + if (uv := shutil.which("uv")) is not None: + run([uv, "venv", "--clear", "--seed", str(venv)]) + else: + run([sys.executable, "-m", "venv", "--clear", str(venv)]) + + +def venv_environment(venv: Path) -> dict[str, str]: + """Return the environment child processes need to target a virtual env. + + Equivalent to sourcing the environment's activate script: tools such as uv + and prek pick the environment up from VIRTUAL_ENV and PATH. + """ + env = dict(os.environ) + env["VIRTUAL_ENV"] = str(venv) + env.pop("PYTHONHOME", None) + path = str(bin_dir(venv)) + # An empty entry would be appended if PATH is unset, and on Unix that means + # the working directory is searched for executables. + if existing := env.get("PATH"): + path = os.pathsep.join([path, existing]) + env["PATH"] = path + return env + + +def find_uv(venv: Path, env: dict[str, str]) -> str: + """Return the path to uv, installing it into the environment if needed.""" + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + run([str(venv_python(venv)), "-m", "pip", "install", "uv"], env=env) + if (uv := shutil.which("uv", path=env["PATH"])) is not None: + return uv + raise SystemExit("uv could not be installed, aborting.") + + +def install_dependencies(venv: Path, env: dict[str, str]) -> None: + """Install ESPHome and its development dependencies into the environment.""" + uv = find_uv(venv, env) + run([uv, "pip", "install", "setuptools", "wheel"], env=env) + # The dev and test extras pull in requirements_dev.txt and + # requirements_test.txt, and the package itself pulls in requirements.txt, + # so this single install covers every requirements file. + run( + [ + uv, + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + env=env, + ) + + +def install_git_hooks(env: dict[str, str]) -> None: + """Install the git hooks, but only when run from the main checkout. + + A worktree shares one git hooks directory with the main checkout it was + created from. Installing from a worktree would point the shared hook at that + worktree's virtual environment, breaking it for everyone once the worktree is + removed. + """ + git_dir = git_output("rev-parse", "--absolute-git-dir") + common_dir = git_output("rev-parse", "--path-format=absolute", "--git-common-dir") + if not git_dir or not common_dir or Path(git_dir) != Path(common_dir): + return + + prek = shutil.which("prek", path=env["PATH"]) + if prek is None: + raise SystemExit("prek was not installed, aborting.") + # --overwrite replaces any hook already in place. Without it, prek finds a + # previously installed pre-commit hook, moves it aside to + # .git/hooks/pre-commit.legacy and keeps calling it, so every commit would + # run both tools. + run([prek, "install", "--overwrite"], env=env) + + # Prepares the virtual environment for new checkouts and worktrees. Installed + # once here, it covers every worktree created from this checkout. + hooks_dir = Path(common_dir) / "hooks" + if hooks_dir.is_dir(): + installed = hooks_dir / "post-checkout" + shutil.copyfile(POST_CHECKOUT_HOOK, installed) + installed.chmod(0o755) + + +def activate_hint() -> str: + """Return the command that activates the environment this script creates.""" + activate = bin_dir(DEFAULT_VENV).relative_to(ROOT) / "activate" + if os.name == "nt": + return str(activate) + return f"source {activate.as_posix()}" + + +def report(state: str, venv: Path) -> None: + """Print the closing message for the environment that was set up.""" + location = f"./{DEFAULT_VENV.name}" + print() + print() + if state == VENV_ACTIVE: + print("Dependencies installed into the active virtual environment:") + print(f" {venv}") + print( + f"It is already active in this shell, so no '{activate_hint()}' is needed." + ) + elif state == VENV_REUSED: + print( + f"Dependencies updated in the existing {location}. " + f"Run '{activate_hint()}' to use it." + ) + else: + print( + f"Virtual environment created at {location}. " + f"Run '{activate_hint()}' to use it." + ) + + +def main() -> None: + """Set up the development environment.""" + if sys.version_info < MIN_PYTHON: + raise SystemExit( + f"ESPHome needs Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer, " + f"but this is Python {sys.version.split()[0]}." + ) + + # A virtual environment that is already active (for example the + # devcontainer's pre-provisioned esphome-venv) is installed into rather than + # creating a ./venv in the workspace. + if active := os.environ.get("VIRTUAL_ENV"): + state, venv = VENV_ACTIVE, Path(active) + elif venv_python(DEFAULT_VENV).is_file(): + # Reuse the environment from an earlier run, so this script can be run + # again at any time to pick up dependency changes. + state, venv = VENV_REUSED, DEFAULT_VENV + else: + state, venv = VENV_CREATED, DEFAULT_VENV + create_venv(venv) + + env = venv_environment(venv) + install_dependencies(venv, env) + install_git_hooks(env) + (ROOT / ".temp").mkdir(exist_ok=True) + report(state, venv) + + +if __name__ == "__main__": + main() diff --git a/tests/script/test_setup.py b/tests/script/test_setup.py new file mode 100644 index 0000000000..3e816c4b05 --- /dev/null +++ b/tests/script/test_setup.py @@ -0,0 +1,562 @@ +"""Tests for script/setup.py.""" + +import importlib.util +import os +from pathlib import Path, PurePosixPath, PureWindowsPath +import runpy +import sys +from types import ModuleType +from unittest.mock import Mock, call, patch + +import pytest + +_SCRIPT = Path(__file__).parents[2] / "script" / "setup.py" + + +def _load_module() -> ModuleType: + spec = importlib.util.spec_from_file_location("script_setup", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def script_setup() -> ModuleType: + """Fresh import of script/setup.py, isolated from other tests.""" + return _load_module() + + +# --- bin_dir / venv_python / activate_hint ----------------------------------- + + +def test_bin_dir_matches_host_layout(script_setup: ModuleType, tmp_path: Path) -> None: + """The venv scheme resolves to Scripts on Windows and bin everywhere else.""" + expected = "Scripts" if os.name == "nt" else "bin" + assert script_setup.bin_dir(tmp_path) == tmp_path / expected + + +# Both flavours are exercised on every host. Pure paths are used because a real +# Path refuses to change flavour: PosixPath cannot be built on Windows, and +# WindowsPath cannot be built on Unix. + + +def test_venv_python_posix(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PurePosixPath("/x/venv/bin/python") + + +def test_venv_python_nt(script_setup: ModuleType, tmp_path: Path) -> None: + with ( + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + result = script_setup.venv_python(tmp_path) + assert result == PureWindowsPath(r"C:\x\venv\Scripts\python.exe") + + +def test_activate_hint_posix(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PurePosixPath("/x")), + patch.object( + script_setup, "bin_dir", return_value=PurePosixPath("/x/venv/bin") + ), + patch.object(script_setup.os, "name", "posix"), + ): + hint = script_setup.activate_hint() + assert hint == "source venv/bin/activate" + + +def test_activate_hint_nt(script_setup: ModuleType) -> None: + with ( + patch.object(script_setup, "ROOT", PureWindowsPath(r"C:\x")), + patch.object( + script_setup, "bin_dir", return_value=PureWindowsPath(r"C:\x\venv\Scripts") + ), + patch.object(script_setup.os, "name", "nt"), + ): + hint = script_setup.activate_hint() + # The nt branch returns str(activate) as-is, skipping the "source " prefix. + assert hint == r"venv\Scripts\activate" + + +# --- run ----------------------------------------------------------------- + + +def test_run_success(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run: + script_setup.run(["echo", "hi"]) + mock_run.assert_called_once_with( + ["echo", "hi"], cwd=script_setup.ROOT, env=None, check=False + ) + + +def test_run_failure_raises_system_exit_with_code( + script_setup: ModuleType, capsys: pytest.CaptureFixture[str] +) -> None: + with ( + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=7)), + pytest.raises(SystemExit) as excinfo, + ): + script_setup.run(["false"]) + assert excinfo.value.code == 7 + assert "Failed with exit code 7: false" in capsys.readouterr().err + + +# --- git_output ------------------------------------------------------------ + + +def test_git_output_success_strips_stdout(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=0, stdout=" /repo/.git \n"), + ) as mock_run: + result = script_setup.git_output("rev-parse", "--absolute-git-dir") + assert result == "/repo/.git" + mock_run.assert_called_once_with( + ["git", "rev-parse", "--absolute-git-dir"], + cwd=script_setup.ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_git_output_nonzero_returncode_is_empty(script_setup: ModuleType) -> None: + with patch.object( + script_setup.subprocess, + "run", + return_value=Mock(returncode=1, stdout="whatever"), + ): + assert script_setup.git_output("status") == "" + + +def test_git_output_oserror_is_empty(script_setup: ModuleType) -> None: + with patch.object(script_setup.subprocess, "run", side_effect=OSError("no git")): + assert script_setup.git_output("status") == "" + + +# --- create_venv ----------------------------------------------------------- + + +def test_create_venv_uses_uv_when_present( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + ["/usr/bin/uv", "venv", "--clear", "--seed", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +def test_create_venv_falls_back_to_venv_module( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + with ( + patch.object(script_setup.shutil, "which", return_value=None), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.create_venv(venv) + mock_run.assert_called_once_with( + [sys.executable, "-m", "venv", "--clear", str(venv)], + cwd=script_setup.ROOT, + env=None, + check=False, + ) + + +# --- venv_environment -------------------------------------------------------- + + +def test_venv_environment_sets_virtual_env_and_prepends_path( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.setenv("PYTHONHOME", "/somewhere") + monkeypatch.setenv("PATH", "/usr/bin:/bin") + env = script_setup.venv_environment(venv) + assert env["VIRTUAL_ENV"] == str(venv) + assert "PYTHONHOME" not in env + expected_prefix = str(script_setup.bin_dir(venv)) + os.pathsep + assert env["PATH"] == expected_prefix + "/usr/bin:/bin" + + +def test_venv_environment_path_fallback_when_unset( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + venv = tmp_path / "venv" + monkeypatch.delenv("PATH", raising=False) + env = script_setup.venv_environment(venv) + # No trailing separator: an empty PATH entry means "search the cwd". + assert env["PATH"] == str(script_setup.bin_dir(venv)) + + +# --- find_uv ----------------------------------------------------------------- + + +def test_find_uv_found_immediately(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_not_called() + + +def test_find_uv_installed_then_found(script_setup: ModuleType, tmp_path: Path) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, "/usr/bin/uv"]), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + result = script_setup.find_uv(venv, env) + assert result == "/usr/bin/uv" + mock_run.assert_called_once_with( + [str(script_setup.venv_python(venv)), "-m", "pip", "install", "uv"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + + +def test_find_uv_still_missing_raises_system_exit( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", side_effect=[None, None]), + patch.object(script_setup.subprocess, "run", return_value=Mock(returncode=0)), + pytest.raises(SystemExit, match="uv could not be installed"), + ): + script_setup.find_uv(venv, env) + + +# --- install_dependencies ----------------------------------------------------- + + +def test_install_dependencies_installs_setuptools_then_project( + script_setup: ModuleType, tmp_path: Path +) -> None: + venv = tmp_path / "venv" + env = {"PATH": "/usr/bin"} + with ( + patch.object(script_setup.shutil, "which", return_value="/usr/bin/uv"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_dependencies(venv, env) + assert mock_run.call_args_list == [ + call( + ["/usr/bin/uv", "pip", "install", "setuptools", "wheel"], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + call( + [ + "/usr/bin/uv", + "pip", + "install", + "-e", + ".[dev,test]", + "--config-settings", + "editable_mode=compat", + ], + cwd=script_setup.ROOT, + env=env, + check=False, + ), + ] + + +# --- install_git_hooks --------------------------------------------------------- + + +def _fake_git_output(git_dir: str, common_dir: str): + def _run(*args: str) -> str: + if "--absolute-git-dir" in args: + return git_dir + return common_dir + + return _run + + +def test_install_git_hooks_returns_early_when_git_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("", "/repo/.git") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_when_common_dir_empty( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, "git_output", side_effect=_fake_git_output("/repo/.git", "") + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_returns_early_for_worktree( + script_setup: ModuleType, +) -> None: + """A worktree's git-dir differs from the shared common-dir.""" + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git/worktrees/wt", "/repo/.git"), + ), + patch.object(script_setup.subprocess, "run") as mock_run, + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_missing_prek_raises_system_exit( + script_setup: ModuleType, +) -> None: + env = {"PATH": "/usr/bin"} + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output("/repo/.git", "/repo/.git"), + ), + patch.object(script_setup.shutil, "which", return_value=None), + patch.object(script_setup.subprocess, "run") as mock_run, + pytest.raises(SystemExit, match="prek was not installed"), + ): + script_setup.install_git_hooks(env) + mock_run.assert_not_called() + + +def test_install_git_hooks_happy_path_installs_hook( + script_setup: ModuleType, tmp_path: Path +) -> None: + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + hooks_dir = common_dir / "hooks" + hooks_dir.mkdir(parents=True) + source_hook = tmp_path / "post-checkout" + source_hook.write_text("#!/bin/sh\necho post-checkout\n") + + with ( + patch.object(script_setup, "POST_CHECKOUT_HOOK", source_hook), + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once_with( + ["/usr/bin/prek", "install", "--overwrite"], + cwd=script_setup.ROOT, + env=env, + check=False, + ) + installed = hooks_dir / "post-checkout" + assert installed.read_text() == source_hook.read_text() + if os.name != "nt": + # Windows has no POSIX permission bits for chmod to set. + assert (installed.stat().st_mode & 0o777) == 0o755 + + +def test_install_git_hooks_skips_copy_when_hooks_dir_missing( + script_setup: ModuleType, tmp_path: Path +) -> None: + """The prek install still runs when the hooks directory does not exist.""" + env = {"PATH": "/usr/bin"} + common_dir = tmp_path / "repo" / ".git" + common_dir.mkdir(parents=True) # no "hooks" subdirectory created + + with ( + patch.object( + script_setup, + "git_output", + side_effect=_fake_git_output(str(common_dir), str(common_dir)), + ), + patch.object(script_setup.shutil, "which", return_value="/usr/bin/prek"), + patch.object( + script_setup.subprocess, "run", return_value=Mock(returncode=0) + ) as mock_run, + ): + script_setup.install_git_hooks(env) + + mock_run.assert_called_once() + assert not (common_dir / "hooks").exists() + + +# --- report ------------------------------------------------------------------ + + +def test_report_active_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + venv = Path("/opt/esphome-venv") + script_setup.report(script_setup.VENV_ACTIVE, venv) + out = capsys.readouterr().out + assert "Dependencies installed into the active virtual environment:" in out + assert str(venv) in out + assert "is already active in this shell" in out + + +def test_report_reused_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_REUSED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Dependencies updated in the existing ./venv" in out + + +def test_report_created_state( + script_setup: ModuleType, capsys: pytest.CaptureFixture +) -> None: + script_setup.report(script_setup.VENV_CREATED, script_setup.DEFAULT_VENV) + out = capsys.readouterr().out + assert "Virtual environment created at ./venv" in out + + +# --- main -------------------------------------------------------------------- + + +def test_main_raises_system_exit_when_python_too_old( + script_setup: ModuleType, +) -> None: + with ( + patch.object(script_setup.sys, "version_info", (3, 11, 5)), + pytest.raises(SystemExit, match="ESPHome needs Python 3.12"), + ): + script_setup.main() + + +def test_main_uses_active_virtual_env( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + active_venv = tmp_path / "active-venv" + monkeypatch.setenv("VIRTUAL_ENV", str(active_venv)) + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_ACTIVE, active_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_reuses_existing_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" + python_path = script_setup.venv_python(default_venv) + python_path.parent.mkdir(parents=True) + python_path.touch() + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_not_called() + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_REUSED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_main_creates_new_venv( + script_setup: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + default_venv = tmp_path / "venv" # does not exist yet + + with ( + patch.object(script_setup, "ROOT", tmp_path), + patch.object(script_setup, "DEFAULT_VENV", default_venv), + patch.object(script_setup, "create_venv") as mock_create_venv, + patch.object(script_setup, "install_dependencies") as mock_install_deps, + patch.object(script_setup, "install_git_hooks") as mock_install_hooks, + patch.object(script_setup, "report") as mock_report, + ): + script_setup.main() + mock_create_venv.assert_called_once_with(default_venv) + mock_install_deps.assert_called_once() + mock_install_hooks.assert_called_once() + mock_report.assert_called_once_with(script_setup.VENV_CREATED, default_venv) + assert (tmp_path / ".temp").is_dir() + + +def test_run_as_script_calls_main(tmp_path: Path) -> None: + """The __main__ guard runs the whole flow, with every side effect stubbed.""" + completed = Mock(returncode=0, stdout="") + with ( + patch("subprocess.run", return_value=completed) as mock_run, + patch("shutil.which", return_value="/usr/bin/uv"), + patch("pathlib.Path.mkdir") as mock_mkdir, + patch.dict(os.environ, {"VIRTUAL_ENV": str(tmp_path / "env")}), + ): + runpy.run_path(str(_SCRIPT), run_name="__main__") + + # The dependency install ran, and git reported no hooks directory to touch. + assert mock_run.called + mock_mkdir.assert_called_once_with(exist_ok=True) From 4868b498cf80cf6fb6c59544a84797f74736bbfe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:09:58 +1200 Subject: [PATCH 34/55] [ci] Ask stale PR authors to merge dev instead of rebasing (#19064) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index aa31094f81..38d2418ac6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -33,7 +33,7 @@ jobs: and will be closed if no further activity occurs within 7 days. If you are the author of this PR, please leave a comment if you want - to keep it open. Also, please rebase your PR onto the latest dev + to keep it open. Also, please merge the latest dev branch into your branch to ensure that it's up to date with the latest changes. Thank you for your contribution! From 5ed59af9204af5e4a4638d10379375b598dd8ace Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:11:13 +1200 Subject: [PATCH 35/55] [template] Surface value metadata on template entity forms (#17545) --- .../template/binary_sensor/__init__.py | 14 +++- .../components/template/button/__init__.py | 6 +- esphome/components/template/cover/__init__.py | 7 +- esphome/components/template/event/__init__.py | 6 +- .../components/template/number/__init__.py | 9 ++- .../components/template/sensor/__init__.py | 22 +++++- .../components/template/switch/__init__.py | 7 +- .../template/text_sensor/__init__.py | 8 +- esphome/components/template/valve/__init__.py | 7 +- esphome/config_validation.py | 32 ++++++++ .../template/test_template_visibility.py | 76 +++++++++++++++++++ tests/unit_tests/test_config_validation.py | 29 +++++++ 12 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 tests/component_tests/template/test_template_visibility.py diff --git a/esphome/components/template/binary_sensor/__init__.py b/esphome/components/template/binary_sensor/__init__.py index 8f57df91c5..07028f7dff 100644 --- a/esphome/components/template/binary_sensor/__init__.py +++ b/esphome/components/template/binary_sensor/__init__.py @@ -2,7 +2,13 @@ from esphome import automation import esphome.codegen as cg from esphome.components import binary_sensor import esphome.config_validation as cv -from esphome.const import CONF_CONDITION, CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_CONDITION, + CONF_DEVICE_CLASS, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, +) from esphome.cpp_generator import LambdaExpression from .. import template_ns @@ -12,7 +18,11 @@ TemplateBinarySensor = template_ns.class_( ) CONFIG_SCHEMA = ( - binary_sensor.binary_sensor_schema(TemplateBinarySensor) + cv.with_visibility( + binary_sensor.binary_sensor_schema(TemplateBinarySensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Exclusive(CONF_LAMBDA, CONF_CONDITION): cv.returning_lambda, diff --git a/esphome/components/template/button/__init__.py b/esphome/components/template/button/__init__.py index e0101dfc8f..9c6fa13c19 100644 --- a/esphome/components/template/button/__init__.py +++ b/esphome/components/template/button/__init__.py @@ -1,10 +1,14 @@ from esphome.components import button +import esphome.config_validation as cv +from esphome.const import CONF_DEVICE_CLASS from .. import template_ns TemplateButton = template_ns.class_("TemplateButton", button.Button) -CONFIG_SCHEMA = button.button_schema(TemplateButton) +CONFIG_SCHEMA = cv.with_visibility( + button.button_schema(TemplateButton), cv.Visibility.UI, CONF_DEVICE_CLASS +) async def to_code(config): diff --git a/esphome/components/template/cover/__init__.py b/esphome/components/template/cover/__init__.py index 7cb50df84c..0e6f96e9f5 100644 --- a/esphome/components/template/cover/__init__.py +++ b/esphome/components/template/cover/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -38,7 +39,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - cover.cover_schema(TemplateCover) + cv.with_visibility( + cover.cover_schema(TemplateCover), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/event/__init__.py b/esphome/components/template/event/__init__.py index cf9c7f4c3d..bdcbd456d5 100644 --- a/esphome/components/template/event/__init__.py +++ b/esphome/components/template/event/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import event import esphome.config_validation as cv -from esphome.const import CONF_EVENT_TYPES +from esphome.const import CONF_DEVICE_CLASS, CONF_EVENT_TYPES from .. import template_ns @@ -9,7 +9,9 @@ CODEOWNERS = ["@nohat"] TemplateEvent = template_ns.class_("TemplateEvent", event.Event, cg.Component) -CONFIG_SCHEMA = event.event_schema(TemplateEvent).extend( +CONFIG_SCHEMA = cv.with_visibility( + event.event_schema(TemplateEvent), cv.Visibility.UI, CONF_DEVICE_CLASS +).extend( { cv.Required(CONF_EVENT_TYPES): cv.ensure_list(cv.string_strict), } diff --git a/esphome/components/template/number/__init__.py b/esphome/components/template/number/__init__.py index 2f4c9cbffe..3b6485fec3 100644 --- a/esphome/components/template/number/__init__.py +++ b/esphome/components/template/number/__init__.py @@ -3,6 +3,7 @@ import esphome.codegen as cg from esphome.components import number import esphome.config_validation as cv from esphome.const import ( + CONF_DEVICE_CLASS, CONF_ID, CONF_INITIAL_VALUE, CONF_LAMBDA, @@ -12,6 +13,7 @@ from esphome.const import ( CONF_RESTORE_VALUE, CONF_SET_ACTION, CONF_STEP, + CONF_UNIT_OF_MEASUREMENT, ) from .. import template_ns @@ -46,7 +48,12 @@ def validate(config): CONFIG_SCHEMA = cv.All( - number.number_schema(TemplateNumber) + cv.with_visibility( + number.number_schema(TemplateNumber), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + ) .extend( { cv.Required(CONF_MAX_VALUE): cv.float_, diff --git a/esphome/components/template/sensor/__init__.py b/esphome/components/template/sensor/__init__.py index 0c875bba0f..55537a5636 100644 --- a/esphome/components/template/sensor/__init__.py +++ b/esphome/components/template/sensor/__init__.py @@ -2,7 +2,16 @@ from esphome import automation import esphome.codegen as cg from esphome.components import sensor import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import ( + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_FORCE_UPDATE, + CONF_ID, + CONF_LAMBDA, + CONF_STATE, + CONF_STATE_CLASS, + CONF_UNIT_OF_MEASUREMENT, +) from .. import template_ns @@ -11,9 +20,14 @@ TemplateSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - sensor.sensor_schema( - TemplateSensor, - accuracy_decimals=1, + cv.with_visibility( + sensor.sensor_schema(TemplateSensor, accuracy_decimals=1), + cv.Visibility.UI, + CONF_UNIT_OF_MEASUREMENT, + CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, + CONF_STATE_CLASS, + CONF_FORCE_UPDATE, ) .extend( { diff --git a/esphome/components/template/switch/__init__.py b/esphome/components/template/switch/__init__.py index ca986365ed..37303abb0d 100644 --- a/esphome/components/template/switch/__init__.py +++ b/esphome/components/template/switch/__init__.py @@ -4,6 +4,7 @@ from esphome.components import switch import esphome.config_validation as cv from esphome.const import ( CONF_ASSUMED_STATE, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC, @@ -31,7 +32,11 @@ def validate(config): CONFIG_SCHEMA = cv.All( - switch.switch_schema(TemplateSwitch) + cv.with_visibility( + switch.switch_schema(TemplateSwitch), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/components/template/text_sensor/__init__.py b/esphome/components/template/text_sensor/__init__.py index ddbdd6dadb..77f5c2ff7c 100644 --- a/esphome/components/template/text_sensor/__init__.py +++ b/esphome/components/template/text_sensor/__init__.py @@ -3,7 +3,7 @@ import esphome.codegen as cg from esphome.components import text_sensor from esphome.components.text_sensor import TextSensorPublishAction import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_LAMBDA, CONF_STATE +from esphome.const import CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_STATE from .. import template_ns @@ -12,7 +12,11 @@ TemplateTextSensor = template_ns.class_( ) CONFIG_SCHEMA = ( - text_sensor.text_sensor_schema() + cv.with_visibility( + text_sensor.text_sensor_schema(), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.GenerateID(): cv.declare_id(TemplateTextSensor), diff --git a/esphome/components/template/valve/__init__.py b/esphome/components/template/valve/__init__.py index a2d0c19880..11b35dad23 100644 --- a/esphome/components/template/valve/__init__.py +++ b/esphome/components/template/valve/__init__.py @@ -6,6 +6,7 @@ from esphome.const import ( CONF_ASSUMED_STATE, CONF_CLOSE_ACTION, CONF_CURRENT_OPERATION, + CONF_DEVICE_CLASS, CONF_ID, CONF_LAMBDA, CONF_OPEN_ACTION, @@ -36,7 +37,11 @@ CONF_HAS_POSITION = "has_position" CONF_TOGGLE_ACTION = "toggle_action" CONFIG_SCHEMA = ( - valve.valve_schema(TemplateValve) + cv.with_visibility( + valve.valve_schema(TemplateValve), + cv.Visibility.UI, + CONF_DEVICE_CLASS, + ) .extend( { cv.Optional(CONF_LAMBDA): cv.returning_lambda, diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 685a9d04b3..a38fb2ed82 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Callable from contextlib import contextmanager, suppress +import copy from datetime import datetime from ipaddress import ( AddressValueError, @@ -419,6 +420,37 @@ class Required(vol.Required): self.visibility: Visibility | None = visibility +def with_visibility(schema: Schema, visibility: Visibility, *keys: str) -> Schema: + """Return a copy of ``schema`` with the given ``keys`` re-marked at ``visibility``. + + Lets a platform override the editor :class:`Visibility` of fields it + inherits from a shared schema builder — without that builder needing a + visibility parameter of its own. The canonical use is a ``template`` + platform promoting the value metadata its user is expected to define + (``device_class``, ``unit_of_measurement``, …) onto the main form: + + CONFIG_SCHEMA = cv.with_visibility( + sensor.sensor_schema(TemplateSensor), + cv.Visibility.UI, + CONF_DEVICE_CLASS, CONF_UNIT_OF_MEASUREMENT, + ) + + The original marker's key, default and validator are preserved; only the + visibility changes, and the input ``schema`` is left untouched. Raises if + a requested key is not present so typos fail at schema-build time. + """ + wanted = {str(k) for k in keys} + overrides = {} + for marker, validator in schema.schema.items(): + if str(marker) in wanted: + marker = copy.copy(marker) + marker.visibility = visibility + overrides[marker] = validator + if missing := wanted - {str(m) for m in overrides}: + raise ValueError(f"with_visibility: keys not in schema: {sorted(missing)}") + return schema.extend(overrides) + + class FinalExternalInvalid(Invalid): """Represents an invalid value in the final validation phase where the path should not be prepended.""" diff --git a/tests/component_tests/template/test_template_visibility.py b/tests/component_tests/template/test_template_visibility.py new file mode 100644 index 0000000000..a50a27e1f7 --- /dev/null +++ b/tests/component_tests/template/test_template_visibility.py @@ -0,0 +1,76 @@ +"""The template platforms surface value-describing metadata on the main form. + +Hardware platforms get sensible defaults for unit/device_class/etc., so those +fields fall through to the editor's advanced disclosure. A ``template`` entity +has no such defaults -- the user is expected to define them -- so the template +platforms pass ``visibility=cv.Visibility.UI`` to promote them onto the form. +""" + +from __future__ import annotations + +import importlib + +import pytest + +import esphome.config_validation as cv + + +def _markers(schema: cv.Schema) -> dict[str, object]: + s = schema + if hasattr(s, "validators"): + # cv.All -> the schema is the first validator. + s = s.validators[0] + return {str(k): k for k in s.schema} + + +@pytest.mark.parametrize( + ("platform", "fields"), + [ + ( + "sensor", + [ + "unit_of_measurement", + "accuracy_decimals", + "device_class", + "state_class", + "force_update", + ], + ), + ("binary_sensor", ["device_class"]), + ("switch", ["device_class"]), + ("cover", ["device_class"]), + ("button", ["device_class"]), + ("valve", ["device_class"]), + ("event", ["device_class"]), + ("text_sensor", ["device_class"]), + ("number", ["device_class", "unit_of_measurement"]), + ], +) +def test_template_metadata_is_ui(platform: str, fields: list[str]) -> None: + mod = importlib.import_module(f"esphome.components.template.{platform}") + markers = _markers(mod.CONFIG_SCHEMA) + for field in fields: + assert markers[field].visibility is cv.Visibility.UI, f"{platform}.{field}" + + +def test_template_sensor_promotion_preserves_defaults() -> None: + """Promoting to UI must not drop the fields' defaults.""" + from esphome.components.template.sensor import CONFIG_SCHEMA + + markers = _markers(CONFIG_SCHEMA) + assert markers["accuracy_decimals"].default() == 1 + assert markers["force_update"].default() is False + + +def test_hardware_platform_metadata_not_promoted() -> None: + """Without ``visibility=`` the builders leave metadata unset. + + Unset markers fall through to the consumer's ``Optional`` default of + advanced, so hardware platforms are unaffected by the template promotion. + """ + from esphome.components import binary_sensor, sensor + + hw_sensor = _markers(sensor.sensor_schema(device_class="temperature")) + assert hw_sensor["device_class"].visibility is None + hw_bs = _markers(binary_sensor.binary_sensor_schema(device_class="motion")) + assert hw_bs["device_class"].visibility is None diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 457b9d017b..4092b4c0d5 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -1394,6 +1394,35 @@ def test_entity_metadata_visibility_hints() -> None: assert web["web_server"].visibility is advanced +def test_with_visibility_remarks_keys() -> None: + """``with_visibility`` re-marks the named keys, preserving each field's + default and validator, without touching the other keys or the input schema. + """ + base = cv.Schema( + { + cv.Optional("a", default=7): cv.int_, + cv.Optional("b", visibility=cv.Visibility.ADVANCED): cv.string, + } + ) + promoted = cv.with_visibility(base, cv.Visibility.UI, "a") + + pm = {str(k): k for k in promoted.schema} + assert pm["a"].visibility is cv.Visibility.UI # re-marked + assert pm["a"].default() == 7 # default preserved + assert pm["b"].visibility is cv.Visibility.ADVANCED # sibling untouched + assert promoted({}) == {"a": 7} # validator/default still applied + + # The input schema is left untouched (no shared-marker mutation). + assert {str(k): k for k in base.schema}["a"].visibility is None + + +def test_with_visibility_unknown_key_raises() -> None: + """A key not present in the schema is a typo — fail at build time.""" + base = cv.Schema({cv.Optional("a"): cv.int_}) + with pytest.raises(ValueError, match="not in schema"): + cv.with_visibility(base, cv.Visibility.UI, "nope") + + def _wrap_str(value: str) -> ESPHomeDataBase: """Wrap a raw string as an ESPHomeDataBase, mimicking a YAML-loaded value.""" return make_data_base(value) From ddbd89dd2a5bb35b24217dae93a4a45ce5da476f Mon Sep 17 00:00:00 2001 From: Robin Thoni Date: Thu, 10 Sep 2026 06:06:44 +0200 Subject: [PATCH 36/55] [network] Improve `network::is_connected()` to better handle multiple interfaces (#18999) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/network/util.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index 65a578c22f..57c5a66833 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -26,30 +26,34 @@ namespace esphome::network { /// Return whether the node is connected to the network (through wifi, eth, ...) ESPHOME_ALWAYS_INLINE inline bool is_connected() { + // With a single interface enabled the checks below collapse to `if (x) return true; return false;`, which + // clang-tidy wants folded into one return. Keep the per-interface form so every enabled interface is checked. + // NOLINTBEGIN(readability-simplify-boolean-expr) #ifdef USE_ETHERNET if (ethernet::global_eth_component != nullptr && ethernet::global_eth_component->is_connected()) return true; #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->is_connected(); + if (modem::global_modem_component != nullptr && modem::global_modem_component->is_connected()) + return true; #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->is_connected(); + if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->is_connected()) + return true; #endif #ifdef USE_OPENTHREAD - if (openthread::global_openthread_component != nullptr) - return openthread::global_openthread_component->is_connected(); + if (openthread::global_openthread_component != nullptr && openthread::global_openthread_component->is_connected()) + return true; #endif #ifdef USE_HOST return true; // Assume it's connected #endif return false; + // NOLINTEND(readability-simplify-boolean-expr) } /// Return whether the network is disabled: every configured interface with a From 05f7d5e4f1b1d5ff14f0d4c30ce984ed319ca112 Mon Sep 17 00:00:00 2001 From: Anton Sergunov Date: Thu, 10 Sep 2026 10:14:43 +0600 Subject: [PATCH 37/55] [mlx90614] pec validation (#6689) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/mlx90614/mlx90614.cpp | 185 +++++++++++++++++------ esphome/components/mlx90614/mlx90614.h | 7 +- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/esphome/components/mlx90614/mlx90614.cpp b/esphome/components/mlx90614/mlx90614.cpp index 2d3b6631bc..508b3743d1 100644 --- a/esphome/components/mlx90614/mlx90614.cpp +++ b/esphome/components/mlx90614/mlx90614.cpp @@ -26,44 +26,129 @@ static const uint8_t MLX90614_ID4 = 0x3F; static const char *const TAG = "mlx90614"; +// The EEPROM cell has a limited number of write cycles, so stop retrying after a few failures +static constexpr uint8_t EMISSIVITY_WRITE_ATTEMPTS = 3; + +// SMBus packet error code: CRC-8 with polynomial 0x07, MSB first +static uint8_t crc8_pec(const uint8_t *data, uint8_t len) { return crc8(data, len, 0x00, 0x07, true); } + void MLX90614Component::setup() { - if (!this->write_emissivity_()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); - this->mark_failed(); + if (std::isnan(this->emissivity_)) { return; } + this->emissivity_write_attempts_ = EMISSIVITY_WRITE_ATTEMPTS; + this->try_write_emissivity_(); + if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } +} + +void MLX90614Component::try_write_emissivity_() { + if (this->emissivity_write_attempts_ == 0) { + return; + } + if (this->write_emissivity_()) { + this->emissivity_write_attempts_ = 0; + return; + } + if (--this->emissivity_write_attempts_ == 0) { + ESP_LOGE(TAG, "Giving up on writing emissivity after %u attempts", EMISSIVITY_WRITE_ATTEMPTS); + this->emissivity_write_failed_ = true; + } } bool MLX90614Component::write_emissivity_() { - if (std::isnan(this->emissivity_)) + // Skip the write when the EEPROM already holds the desired value to save write cycles + uint16_t current_emissivity; + if (this->read_register_(MLX90614_EMISSIVITY, current_emissivity) != i2c::ERROR_OK) { + return false; + } + + const auto desired_emissivity = static_cast(this->emissivity_ * 0xFFFF); + if (current_emissivity == desired_emissivity) { return true; - uint16_t value = (uint16_t) (this->emissivity_ * 65535); - if (!this->write_bytes_(MLX90614_EMISSIVITY, 0)) { - return false; } - delay(10); - if (!this->write_bytes_(MLX90614_EMISSIVITY, value)) { - return false; - } - delay(10); - return true; + + return this->write_register_(MLX90614_EMISSIVITY, desired_emissivity); } -bool MLX90614Component::write_bytes_(uint8_t reg, uint16_t data) { +bool MLX90614Component::write_register_(uint8_t reg, uint16_t data) { + // The PEC covers the whole write transaction: SLA+W, command, data low, data high uint8_t buf[5]; buf[0] = this->address_ << 1; buf[1] = reg; - buf[2] = data & 0xFF; - buf[3] = data >> 8; - buf[4] = crc8(buf, 4, 0x00, 0x07, true); - return this->write_bytes(reg, buf + 2, 3); + + // See datasheet 8.3.3.1 EEPROM write sequence + // 1. Write 0x0000 into the cell of interest (erases the cell) + buf[2] = buf[3] = 0; + buf[4] = crc8_pec(buf, 4); + auto ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't erase register 0x%02X, error %d", reg, ec); + return false; + } + + // 2. Wait at least 5ms + delay(10); + + // 3. Write the new value + if (data != 0) { + buf[2] = data & 0xFF; + buf[3] = data >> 8; + buf[4] = crc8_pec(buf, 4); + ec = this->write_register(reg, buf + 2, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't write register 0x%02X, error %d", reg, ec); + return false; + } + // 4. Wait at least 5ms + delay(10); + } + + // 5. Read back to confirm the value was stored + uint16_t read_back; + ec = this->read_register_(reg, read_back); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "Can't check register 0x%02X value, error %d", reg, ec); + return false; + } + + if (read_back != data) { + ESP_LOGW(TAG, "Read back mismatch on register 0x%02X. Expected 0x%04X, got 0x%04X", reg, data, read_back); + return false; + } + + return true; +} + +i2c::ErrorCode MLX90614Component::read_register_(uint8_t reg, uint16_t &data) { + // The PEC covers the whole read transaction: SLA+W, command, SLA+R, data low, data high + uint8_t buf[6]; + buf[0] = this->address_ << 1; + buf[1] = reg; + buf[2] = (this->address_ << 1) | 0x01; + + const auto ec = this->read_register(reg, buf + 3, 3); + if (ec != i2c::ERROR_OK) { + ESP_LOGW(TAG, "i2c read error %d", ec); + return ec; + } + + const auto expected_pec = crc8_pec(buf, 5); + if (buf[5] != expected_pec) { + ESP_LOGW(TAG, "i2c CRC error. Expected 0x%02X, got 0x%02X", expected_pec, buf[5]); + return i2c::ERROR_CRC; + } + + data = encode_uint16(buf[4], buf[3]); + return i2c::ERROR_OK; } void MLX90614Component::dump_config() { ESP_LOGCONFIG(TAG, "MLX90614:"); LOG_I2C_DEVICE(this); - if (this->is_failed()) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); + if (this->emissivity_write_attempts_ != 0) { + ESP_LOGW(TAG, " Emissivity not written yet, will retry"); } LOG_UPDATE_INTERVAL(this); LOG_SENSOR(" ", "Ambient", this->ambient_sensor_); @@ -71,33 +156,41 @@ void MLX90614Component::dump_config() { } void MLX90614Component::update() { - uint8_t emissivity[3]; - if (this->read_register(MLX90614_EMISSIVITY, emissivity, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; + // Temperature reads run regardless of the emissivity state so a failure still shows up as NAN + this->try_write_emissivity_(); + + // Publishes NAN on a bus or CRC failure so a stuck reading is visible instead of silently stale + auto publish_sensor = [this](sensor::Sensor *sensor, uint8_t reg) { + if (sensor == nullptr) { + return i2c::ERROR_OK; + } + + uint16_t raw; + const auto ec = this->read_register_(reg, raw); + if (ec != i2c::ERROR_OK) { + sensor->publish_state(NAN); + return ec; + } + + // Bit 15 set means the device flagged the reading as invalid + const float temperature = (raw & 0x8000) ? NAN : raw * 0.02f - 273.15f; + ESP_LOGD(TAG, "'%s': Got temperature=%.1f°C", sensor->get_name().c_str(), temperature); + sensor->publish_state(temperature); + return ec; + }; + + const auto object_ec = publish_sensor(this->object_sensor_, MLX90614_TEMPERATURE_OBJECT_1); + const auto ambient_ec = publish_sensor(this->ambient_sensor_, MLX90614_TEMPERATURE_AMBIENT); + + if (object_ec != i2c::ERROR_OK || ambient_ec != i2c::ERROR_OK) { + this->status_set_warning(LOG_STR("Failed to read some sensors")); + } else if (this->emissivity_write_failed_) { + this->status_set_warning(LOG_STR("Failed to write emissivity")); + } else if (this->emissivity_write_attempts_ != 0) { + this->status_set_warning(LOG_STR("Failed to write emissivity, will retry")); + } else { + this->status_clear_warning(); } - uint8_t raw_object[3]; - if (this->read_register(MLX90614_TEMPERATURE_OBJECT_1, raw_object, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - uint8_t raw_ambient[3]; - if (this->read_register(MLX90614_TEMPERATURE_AMBIENT, raw_ambient, 3) != i2c::ERROR_OK) { - this->status_set_warning(); - return; - } - - float ambient = raw_ambient[1] & 0x80 ? NAN : encode_uint16(raw_ambient[1], raw_ambient[0]) * 0.02f - 273.15f; - float object = raw_object[1] & 0x80 ? NAN : encode_uint16(raw_object[1], raw_object[0]) * 0.02f - 273.15f; - - ESP_LOGD(TAG, "Got Temperature=%.1f°C Ambient=%.1f°C", object, ambient); - - if (this->ambient_sensor_ != nullptr && !std::isnan(ambient)) - this->ambient_sensor_->publish_state(ambient); - if (this->object_sensor_ != nullptr && !std::isnan(object)) - this->object_sensor_->publish_state(object); - this->status_clear_warning(); } } // namespace esphome::mlx90614 diff --git a/esphome/components/mlx90614/mlx90614.h b/esphome/components/mlx90614/mlx90614.h index 882ee45186..758792aced 100644 --- a/esphome/components/mlx90614/mlx90614.h +++ b/esphome/components/mlx90614/mlx90614.h @@ -18,13 +18,18 @@ class MLX90614Component final : public PollingComponent, public i2c::I2CDevice { void set_emissivity(float emissivity) { emissivity_ = emissivity; } protected: + void try_write_emissivity_(); bool write_emissivity_(); - bool write_bytes_(uint8_t reg, uint16_t data); + bool write_register_(uint8_t reg, uint16_t data); + i2c::ErrorCode read_register_(uint8_t reg, uint16_t &data); sensor::Sensor *ambient_sensor_{nullptr}; sensor::Sensor *object_sensor_{nullptr}; float emissivity_{NAN}; + // Remaining attempts to program the emissivity EEPROM cell, bounded to limit cell wear + uint8_t emissivity_write_attempts_{0}; + bool emissivity_write_failed_{false}; }; } // namespace esphome::mlx90614 From 54706e869c13abc0f688a91c0f326c779799b858 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:18:15 +0200 Subject: [PATCH 38/55] [deep_sleep] disable loop (#18962) --- esphome/components/deep_sleep/deep_sleep_bk72xx.cpp | 2 +- esphome/components/deep_sleep/deep_sleep_component.cpp | 3 ++- esphome/components/deep_sleep/deep_sleep_component.h | 5 +++++ esphome/components/deep_sleep/deep_sleep_esp32.cpp | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp index 2c97dc3211..a955095875 100644 --- a/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp +++ b/esphome/components/deep_sleep/deep_sleep_bk72xx.cpp @@ -44,7 +44,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGV(TAG, "Waiting for pin to switch state to enter deep sleep..."); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } } diff --git a/esphome/components/deep_sleep/deep_sleep_component.cpp b/esphome/components/deep_sleep/deep_sleep_component.cpp index 9a3e537e05..d33102bf4f 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.cpp +++ b/esphome/components/deep_sleep/deep_sleep_component.cpp @@ -17,6 +17,7 @@ void DeepSleepComponent::setup() { void DeepSleepComponent::schedule_sleep_() { this->next_enter_deep_sleep_ = false; + this->disable_loop(); const optional run_duration = get_run_duration_(); if (run_duration.has_value()) { ESP_LOGI(TAG, "Scheduling in %" PRIu32 " ms", *run_duration); @@ -45,7 +46,7 @@ void DeepSleepComponent::loop() { void DeepSleepComponent::begin_sleep(bool manual) { if (this->prevent_ && !manual) { - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return; } diff --git a/esphome/components/deep_sleep/deep_sleep_component.h b/esphome/components/deep_sleep/deep_sleep_component.h index 208f88d707..0bbca4c5c4 100644 --- a/esphome/components/deep_sleep/deep_sleep_component.h +++ b/esphome/components/deep_sleep/deep_sleep_component.h @@ -190,6 +190,11 @@ class DeepSleepComponent final : public Component { void schedule_sleep_(); bool should_teardown_(); + void defer_sleep_() { + this->next_enter_deep_sleep_ = true; + this->enable_loop(); + } + #ifdef USE_BK72XX bool pin_prevents_sleep_(WakeUpPinItem &pin_item) const; bool get_real_pin_state_(InternalGPIOPin &pin) const { return (pin.digital_read() ^ pin.is_inverted()); } diff --git a/esphome/components/deep_sleep/deep_sleep_esp32.cpp b/esphome/components/deep_sleep/deep_sleep_esp32.cpp index 3fa1a1f1ed..20297028b2 100644 --- a/esphome/components/deep_sleep/deep_sleep_esp32.cpp +++ b/esphome/components/deep_sleep/deep_sleep_esp32.cpp @@ -100,7 +100,7 @@ bool DeepSleepComponent::prepare_to_sleep_() { this->status_set_warning(); ESP_LOGW(TAG, "Waiting for wakeup pin state change"); } - this->next_enter_deep_sleep_ = true; + this->defer_sleep_(); return false; } return true; From a88ec7d90b6b19813f2e06bb012c7192b04b3c73 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:07:09 +0200 Subject: [PATCH 39/55] [logger] Flush uart before sleep in idf 6 (#18975) --- esphome/components/logger/logger_esp32.cpp | 13 +++++++++++-- sdkconfig.defaults | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index c3d777299d..8579708559 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -3,6 +3,7 @@ #include "esphome/components/esp32/crash_handler.h" #include +#include #include #include @@ -16,8 +17,10 @@ #include #endif #endif - -#include "esp_idf_version.h" +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) +#include "esp_sleep.h" +#endif #include "freertos/FreeRTOS.h" #include @@ -87,6 +90,12 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). const int min_rx_buffer_size = UART_HW_FIFO_LEN(uart_num) + 1; uart_driver_install(uart_num, min_rx_buffer_size, tx_buffer_size, 0, nullptr, 0); +#if defined(CONFIG_PM_ENABLE) && defined(CONFIG_FREERTOS_USE_TICKLESS_IDLE) && \ + (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)) + // Always flush before going to light sleep. Could be disabled for devices + // without TOP_PD or if source_clk = UART_SCLK_RTC + esp_sleep_set_console_uart_handling_mode(ESP_SLEEP_ALWAYS_FLUSH_UART); +#endif } void Logger::pre_setup() { diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 2bd702f48e..f4fe331df4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -17,6 +17,8 @@ CONFIG_ESP_TASK_WDT_INIT=y CONFIG_ESP_TASK_WDT_PANIC=y CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=n +CONFIG_FREERTOS_USE_TICKLESS_IDLE=y +CONFIG_PM_ENABLE=y # esp32_ble CONFIG_BT_ENABLED=y From 66f829c760358a291a9a97d90f9b981d8ac6a6ec Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:08:07 +0200 Subject: [PATCH 40/55] [zigbee] wake loop on defer/set_timeout (#19050) --- esphome/components/zigbee/time/zigbee_time_zephyr.cpp | 2 ++ esphome/components/zigbee/zigbee_esp32.cpp | 5 ++++- esphome/components/zigbee/zigbee_zephyr.cpp | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp index 92d238629a..3f14d0a62d 100644 --- a/esphome/components/zigbee/time/zigbee_time_zephyr.cpp +++ b/esphome/components/zigbee/time/zigbee_time_zephyr.cpp @@ -1,6 +1,7 @@ #include "zigbee_time_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) && defined(USE_TIME) #include "esphome/core/log.h" +#include "esphome/core/application.h" namespace esphome::zigbee { @@ -47,6 +48,7 @@ void ZigbeeTime::set_epoch_time(uint32_t epoch) { this->synchronize_epoch_(epoch); this->has_time_ = true; }); + App.wake_loop_threadsafe(); } void ZigbeeTime::zcl_device_cb_(zb_bufid_t bufid) { diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index cd094306f4..4f9c70da75 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -49,7 +49,8 @@ void ZigbeeComponent::factory_reset() { void ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(ezb_bdb_comm_mode_mask_t mode) { if (!esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { - global_zigbee->set_timeout("zb_init", 10, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + global_zigbee->set_timeout("zb_init", 100, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); + App.wake_loop_threadsafe(); return; } if (ezb_bdb_start_top_level_commissioning(mode) != EZB_ERR_NONE) { @@ -88,6 +89,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { global_zigbee->set_timeout("zb_init", 1000, []() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_INITIALIZATION); }); + App.wake_loop_threadsafe(); } } break; case EZB_BDB_SIGNAL_STEERING: { @@ -113,6 +115,7 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(EZB_BDB_MODE_NETWORK_STEERING); }); } + App.wake_loop_threadsafe(); } } break; case EZB_ZDO_SIGNAL_LEAVE: { diff --git a/esphome/components/zigbee/zigbee_zephyr.cpp b/esphome/components/zigbee/zigbee_zephyr.cpp index b8bb0a2036..286c83b8f5 100644 --- a/esphome/components/zigbee/zigbee_zephyr.cpp +++ b/esphome/components/zigbee/zigbee_zephyr.cpp @@ -1,10 +1,10 @@ #include "zigbee_zephyr.h" #if defined(USE_ZIGBEE) && defined(USE_NRF52) #include "esphome/core/log.h" +#include "esphome/core/application.h" #include #include #include "esphome/core/hal.h" -#include "esphome/core/wake.h" extern "C" { #include @@ -120,7 +120,7 @@ void ZigbeeComponent::zcl_device_cb(zb_bufid_t bufid) { /* Set default response value. */ p_device_cb_param->status = RET_OK; - esphome::wake_loop_threadsafe(); + App.wake_loop_threadsafe(); // endpoints are enumerated from 1 if (global_zigbee->callbacks_.size() >= endpoint) { @@ -138,6 +138,7 @@ void ZigbeeComponent::on_join_(bool factory_new) { ESP_LOGD(TAG, "Joined the network"); this->join_cb_.call(factory_new); }); + App.wake_loop_threadsafe(); } void ZigbeeComponent::on_start_() { @@ -145,6 +146,7 @@ void ZigbeeComponent::on_start_() { ESP_LOGD(TAG, "Started zigbee stack"); this->start_cb_.call(); }); + App.wake_loop_threadsafe(); } #ifdef USE_ZIGBEE_WIPE_ON_BOOT From 99241483026d54d47f8f06bdd2415e02c0cd3ebb Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:59:23 +0000 Subject: [PATCH 41/55] Bump aioesphomeapi from 46.3.0 to 46.4.0 (#19071) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfddbed00b..72c42dad32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.4.0 click==8.3.3 -aioesphomeapi==46.3.0 +aioesphomeapi==46.4.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.151.3 puremagic==2.2.0 From 280fac11e6a8b571f6859dc4f9203e470cbbf1d4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 10 Sep 2026 10:03:33 -0500 Subject: [PATCH 42/55] [serial_proxy] Add tap interface and port mode (#18955) Co-authored-by: puddly <32534428+puddly@users.noreply.github.com> --- esphome/components/api/api.proto | 37 +++- esphome/components/api/api_connection.cpp | 16 +- esphome/components/api/api_connection.h | 1 + esphome/components/api/api_pb2.cpp | 13 ++ esphome/components/api/api_pb2.h | 21 ++ esphome/components/api/api_pb2_dump.cpp | 18 ++ esphome/components/api/api_pb2_service.cpp | 11 ++ esphome/components/api/api_pb2_service.h | 3 + esphome/components/serial_proxy/__init__.py | 1 + .../components/serial_proxy/serial_proxy.cpp | 182 +++++++++++++++--- .../components/serial_proxy/serial_proxy.h | 103 +++++++++- esphome/core/defines.h | 1 + .../components/serial_proxy/serial_proxy.h | 3 + .../serial_proxy/test-tap.esp32-idf.yaml | 14 ++ 14 files changed, 394 insertions(+), 30 deletions(-) create mode 100644 tests/components/serial_proxy/test-tap.esp32-idf.yaml diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 3a0e0abea9..21972decad 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -77,6 +77,7 @@ service APIConnection { rpc serial_proxy_set_modem_pins(SerialProxySetModemPinsRequest) returns (void) {} rpc serial_proxy_get_modem_pins(SerialProxyGetModemPinsRequest) returns (void) {} rpc serial_proxy_request(SerialProxyRequest) returns (void) {} + rpc serial_proxy_set_mode(SerialProxySetModeRequest) returns (void) {} } @@ -2726,7 +2727,8 @@ enum SerialProxyParity { SERIAL_PROXY_PARITY_ODD = 2; } -// Configure UART parameters for a serial proxy instance +// Configure UART parameters for a serial proxy instance. Only the subscribed client may +// configure the port; others are refused with PORT_IN_USE (since API 1.17). message SerialProxyConfigureRequest { option (id) = 138; option (source) = SOURCE_CLIENT; @@ -2752,7 +2754,8 @@ message SerialProxyDataReceived { bytes data = 2; // Raw data received from the serial device } -// Write data to a serial device +// Write data to a serial device. Only the subscribed client may write; writes from +// others are ignored (since API 1.17). message SerialProxyWriteRequest { option (id) = 140; option (source) = SOURCE_CLIENT; @@ -2763,7 +2766,8 @@ message SerialProxyWriteRequest { bytes data = 2; // Raw data to write to the serial device } -// Set modem control pin states (RTS and DTR) +// Set modem control pin states (RTS and DTR). Only the subscribed client may set them; +// others are refused with PORT_IN_USE (since API 1.17). message SerialProxySetModemPinsRequest { option (id) = 141; option (source) = SOURCE_CLIENT; @@ -2802,6 +2806,7 @@ enum SerialProxyRequestType { // error the device answers with INVALID_ARGUMENT. SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3; // Acknowledges a SerialProxyConfigureRequest SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4; // Acknowledges a SerialProxySetModemPinsRequest + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5; // Acknowledges a SerialProxySetModeRequest (since API 1.17) } enum SerialProxyStatus { @@ -2814,7 +2819,8 @@ enum SerialProxyStatus { SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6; // Invalid instance index or parameter value } -// Generic request message for simple serial proxy operations +// Generic request message for simple serial proxy operations. FLUSH requires an active +// subscription; it is refused with PORT_IN_USE otherwise (since API 1.17). message SerialProxyRequest { option (id) = 144; option (source) = SOURCE_CLIENT; @@ -2838,6 +2844,29 @@ message SerialProxyRequestResponse { string error_message = 4; // Additional detail on failure (optional) } +// How a port treats the bytes passing through it. RAW is a plain byte pipe; PROTOCOL +// activates the port's protocol-aware tap (if one is configured), letting it observe +// traffic and inject protocol bytes such as acknowledgements. Which protocol the tap +// speaks is a property of the device configuration, discoverable from the tap +// component's own API surface. A client that is about to flash firmware selects RAW +// first, which definitively disables that injection. +enum SerialProxyMode { + SERIAL_PROXY_MODE_RAW = 0; + SERIAL_PROXY_MODE_PROTOCOL = 1; +} + +// Only the subscribed client may change the mode; any other caller -- including one that +// never subscribed -- is refused with PORT_IN_USE. PROTOCOL is refused with NOT_SUPPORTED +// when the port has no protocol-aware tap configured. +message SerialProxySetModeRequest { + option (id) = 152; + option (source) = SOURCE_CLIENT; + option (ifdef) = "USE_SERIAL_PROXY"; + + uint32 instance = 1; + SerialProxyMode mode = 2; +} + // ==================== BLUETOOTH CONNECTION PARAMS ==================== message BluetoothSetConnectionParamsRequest { option (id) = 145; diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index da4b7d7702..d910f6fc67 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1661,6 +1661,7 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { break; case enums::SERIAL_PROXY_REQUEST_TYPE_CONFIGURE: case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: // Response-only discriminators; never valid in a request ESP_LOGW(TAG, "Response-only serial proxy request type: %" PRIu32, static_cast(msg.type)); status = enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT; @@ -1673,6 +1674,19 @@ void APIConnection::on_serial_proxy_request(const SerialProxyRequest &msg) { send_serial_proxy_ack(this, msg.instance, msg.type, status); } +void APIConnection::on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg) { + auto &proxies = App.get_serial_proxies(); + if (msg.instance >= proxies.size()) { + ESP_LOGW(TAG, "Serial proxy instance %" PRIu32 " out of range", msg.instance); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + enums::SERIAL_PROXY_STATUS_INVALID_ARGUMENT); + return; + } + serial_proxy::SerialProxyResult result = proxies[msg.instance]->set_mode_from_client(this, msg.mode); + send_serial_proxy_ack(this, msg.instance, enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE, + serial_proxy_result_to_status(result)); +} + void APIConnection::send_serial_proxy_data(const SerialProxyDataReceived &msg) { if (!this->send_message(msg)) { ESP_LOGV(TAG, "Serial proxy data dropped, TCP buffer full"); @@ -1799,7 +1813,7 @@ bool APIConnection::send_hello_response_(const HelloRequest &msg) { HelloResponse resp; resp.api_version_major = 1; - resp.api_version_minor = 16; + resp.api_version_minor = 17; // Send only the version string - the client only logs this for debugging and doesn't use it otherwise resp.server_info = ESPHOME_VERSION_REF; resp.name = StringRef(App.get_name()); diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index a4c49dccf4..c19a33ca9b 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -244,6 +244,7 @@ class APIConnection final : public APIServerConnectionBase { void on_serial_proxy_set_modem_pins_request(const SerialProxySetModemPinsRequest &msg); void on_serial_proxy_get_modem_pins_request(const SerialProxyGetModemPinsRequest &msg); void on_serial_proxy_request(const SerialProxyRequest &msg); + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &msg); void send_serial_proxy_data(const SerialProxyDataReceived &msg); #endif diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 2de1f0a15c..7f162d9c15 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -4253,6 +4253,19 @@ uint32_t SerialProxyRequestResponse::calculate_size() const { size += ProtoSize::calc_length(1, this->error_message.size()); return size; } +bool SerialProxySetModeRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { + switch (field_id) { + case 1: + this->instance = value; + break; + case 2: + this->mode = static_cast(value); + break; + default: + return false; + } + return true; +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS bool BluetoothSetConnectionParamsRequest::decode_varint(uint32_t field_id, proto_varint_value_t value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 5c3429a63a..799aaa27b5 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -356,6 +356,7 @@ enum SerialProxyRequestType : uint32_t { SERIAL_PROXY_REQUEST_TYPE_FLUSH = 2, SERIAL_PROXY_REQUEST_TYPE_CONFIGURE = 3, SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS = 4, + SERIAL_PROXY_REQUEST_TYPE_SET_MODE = 5, }; enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_OK = 0, @@ -366,6 +367,10 @@ enum SerialProxyStatus : uint32_t { SERIAL_PROXY_STATUS_PORT_IN_USE = 5, SERIAL_PROXY_STATUS_INVALID_ARGUMENT = 6, }; +enum SerialProxyMode : uint32_t { + SERIAL_PROXY_MODE_RAW = 0, + SERIAL_PROXY_MODE_PROTOCOL = 1, +}; #endif } // namespace enums @@ -3403,6 +3408,22 @@ class SerialProxyRequestResponse final : public ProtoMessage { protected: }; +class SerialProxySetModeRequest final : public ProtoDecodableMessage { + public: + static constexpr uint16_t MESSAGE_TYPE = 152; + static constexpr uint8_t ESTIMATED_SIZE = 6; +#ifdef HAS_PROTO_MESSAGE_DUMP + const LogString *message_name() const override { return LOG_STR("serial_proxy_set_mode_request"); } +#endif + uint32_t instance{0}; + enums::SerialProxyMode mode{}; +#ifdef HAS_PROTO_MESSAGE_DUMP + const char *dump_to(DumpBuffer &out) const override; +#endif + + protected: + bool decode_varint(uint32_t field_id, proto_varint_value_t value) override; +}; #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS class BluetoothSetConnectionParamsRequest final : public ProtoDecodableMessage { diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index dced81ee30..bb244973a1 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -854,6 +854,8 @@ template<> const char *proto_enum_to_string(enums return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_CONFIGURE"); case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS: return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODEM_PINS"); + case enums::SERIAL_PROXY_REQUEST_TYPE_SET_MODE: + return ESPHOME_PSTR("SERIAL_PROXY_REQUEST_TYPE_SET_MODE"); default: return ESPHOME_PSTR("UNKNOWN"); } @@ -878,6 +880,16 @@ template<> const char *proto_enum_to_string(enums::Ser return ESPHOME_PSTR("UNKNOWN"); } } +template<> const char *proto_enum_to_string(enums::SerialProxyMode value) { + switch (value) { + case enums::SERIAL_PROXY_MODE_RAW: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_RAW"); + case enums::SERIAL_PROXY_MODE_PROTOCOL: + return ESPHOME_PSTR("SERIAL_PROXY_MODE_PROTOCOL"); + default: + return ESPHOME_PSTR("UNKNOWN"); + } +} #endif const char *HelloRequest::dump_to(DumpBuffer &out) const { @@ -2805,6 +2817,12 @@ const char *SerialProxyRequestResponse::dump_to(DumpBuffer &out) const { dump_field(out, ESPHOME_PSTR("error_message"), this->error_message); return out.c_str(); } +const char *SerialProxySetModeRequest::dump_to(DumpBuffer &out) const { + MessageDumpHelper helper(out, ESPHOME_PSTR("SerialProxySetModeRequest")); + dump_field(out, ESPHOME_PSTR("instance"), this->instance); + dump_field(out, ESPHOME_PSTR("mode"), static_cast(this->mode)); + return out.c_str(); +} #endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS const char *BluetoothSetConnectionParamsRequest::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/api/api_pb2_service.cpp b/esphome/components/api/api_pb2_service.cpp index 65c7b8858c..172062be63 100644 --- a/esphome/components/api/api_pb2_service.cpp +++ b/esphome/components/api/api_pb2_service.cpp @@ -712,6 +712,17 @@ void APIConnection::read_message_(uint32_t msg_size, uint32_t msg_type, const ui this->on_device_capabilities_request(); break; } +#ifdef USE_SERIAL_PROXY + case SerialProxySetModeRequest::MESSAGE_TYPE: { + SerialProxySetModeRequest msg; + msg.decode(msg_data, msg_size); +#ifdef HAS_PROTO_MESSAGE_DUMP + this->log_receive_message_(LOG_STR("on_serial_proxy_set_mode_request"), msg); +#endif + this->on_serial_proxy_set_mode_request(msg); + break; + } +#endif default: break; } diff --git a/esphome/components/api/api_pb2_service.h b/esphome/components/api/api_pb2_service.h index 6abdf7093e..a4dfd6a366 100644 --- a/esphome/components/api/api_pb2_service.h +++ b/esphome/components/api/api_pb2_service.h @@ -235,6 +235,9 @@ class APIServerConnectionBase { void on_serial_proxy_request(const SerialProxyRequest &value){}; #endif +#ifdef USE_SERIAL_PROXY + void on_serial_proxy_set_mode_request(const SerialProxySetModeRequest &value){}; +#endif #ifdef USE_BLUETOOTH_PROXY_CONNECTIONS void on_bluetooth_set_connection_params_request(const BluetoothSetConnectionParamsRequest &value){}; #endif diff --git a/esphome/components/serial_proxy/__init__.py b/esphome/components/serial_proxy/__init__.py index 4186fcf8b1..b6e780fabd 100644 --- a/esphome/components/serial_proxy/__init__.py +++ b/esphome/components/serial_proxy/__init__.py @@ -30,6 +30,7 @@ MULTI_CONF = True serial_proxy_ns = cg.esphome_ns.namespace("serial_proxy") SerialProxy = serial_proxy_ns.class_("SerialProxy", cg.Component, uart.UARTDevice) +SerialProxyTap = serial_proxy_ns.class_("SerialProxyTap") api_enums_ns = cg.esphome_ns.namespace("api").namespace("enums") SerialProxyPortType = api_enums_ns.enum("SerialProxyPortType") diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index c1c1510643..129745c1c9 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -29,26 +29,57 @@ void SerialProxy::setup() { #ifdef USE_API // instance_index_ is fixed at registration time; pre-set it so loop() only needs to update data this->outgoing_msg_.instance = this->instance_index_; +#endif +#ifdef USE_SERIAL_PROXY_TAP + // A tap sets itself up before this runs (its setup priority is higher), so it may + // already be waiting on the port -- a boot-time handshake with the device, say. Leaving + // the loop enabled is what lets that finish; without it the tap would stall until a + // client happened to subscribe. + if (this->tap_ != nullptr && this->tap_->tap_needs_port()) { + return; + } #endif // No subscriber at startup; disable loop until a client subscribes this->disable_loop(); } -void SerialProxy::loop() { -#ifdef USE_API - // Safety check — loop should only run when subscribed, but guard against races - if (this->api_connection_ == nullptr) [[unlikely]] { - this->disable_loop(); +#ifdef USE_SERIAL_PROXY_TAP +void SerialProxy::reset_mode_() { + // The mode belongs to a session, not to the port. Carrying a departed client's choice + // over to the next one would inject protocol bytes into a stream that never asked for + // them -- a firmware upload, or any client built before this request existed and so + // unable to turn it off. Guessing RAW is the safe direction: a client that wanted + // protocol handling and did not ask for it merely sends its own acknowledgements. + if (this->mode_ == api::enums::SERIAL_PROXY_MODE_RAW) { return; } + ESP_LOGD(TAG, "Session ended, returning serial proxy [%" PRIu32 "] to RAW mode", this->instance_index_); + this->mode_ = api::enums::SERIAL_PROXY_MODE_RAW; +} +#endif +void SerialProxy::loop() { +#ifdef USE_API // Detect subscriber disconnect - if (this->api_connection_->is_marked_for_removal() || !this->api_connection_->is_connection_setup() || - !api_is_connected()) { + if (this->api_connection_ != nullptr && (this->api_connection_->is_marked_for_removal() || + !this->api_connection_->is_connection_setup() || !api_is_connected())) { ESP_LOGW(TAG, "Subscriber disconnected"); this->api_connection_ = nullptr; + this->reset_mode_(); + } + + // With no subscriber there is normally nothing to do, but a tap may still need the port + // read -- it does its protocol work precisely while nobody else is listening. + if (this->api_connection_ == nullptr) [[unlikely]] { +#ifdef USE_SERIAL_PROXY_TAP + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + return; + } +#else this->disable_loop(); return; +#endif } // Read available data from UART and forward to subscribed client @@ -69,11 +100,54 @@ void __attribute__((noinline)) SerialProxy::read_and_send_(size_t available) { if (!this->read_array(buffer, to_read)) return; +#ifdef USE_SERIAL_PROXY_TAP + // Before forwarding, so a tap that answers the device (an acknowledgement, say) is not + // waiting on the network round trip to a subscriber that may not even exist. + if (this->tap_observing_()) { + this->tap_->on_device_rx(buffer, to_read); + } +#endif + + if (this->api_connection_ == nullptr) { + return; + } this->outgoing_msg_.set_data(buffer, to_read); this->api_connection_->send_serial_proxy_data(this->outgoing_msg_); } #endif +#ifdef USE_SERIAL_PROXY_TAP + +bool SerialProxy::tap_observing_() const { + if (this->tap_ == nullptr) { + return false; + } + // With no subscriber, a tap doing its own protocol work (the boot-time handshake with + // the device, say) is served regardless of mode -- nobody has chosen one yet. Once a + // subscriber holds the port, the mode alone decides, so RAW stays inert. + if (this->api_connection_ == nullptr && this->tap_->tap_needs_port()) { + return true; + } + // Otherwise the mode decides. RAW must be inert: a client that flips to RAW before + // flashing firmware is entitled to a byte pipe with nothing injecting protocol bytes + // into it, and "the tap turned out not to recognise the stream" is not good enough. + return this->mode_ == api::enums::SERIAL_PROXY_MODE_PROTOCOL; +} + +void SerialProxy::tap_pump() { +#ifdef USE_API + // Nothing would consume the bytes; leave them in the FIFO + if (!this->tap_observing_() && this->api_connection_ == nullptr) { + return; + } + const size_t available = this->available(); + if (available > 0) { + this->read_and_send_(available); + } +#endif +} +#endif + void SerialProxy::dump_config() { ESP_LOGCONFIG(TAG, "Serial Proxy [%" PRIu32 "]:\n" @@ -92,8 +166,9 @@ void SerialProxy::dump_config() { SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, uint8_t parity, uint8_t stop_bits, uint8_t data_size) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring configure request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring configure request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -159,24 +234,80 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } +SerialProxyResult SerialProxy::set_mode_from_client(api::APIConnection *api_connection, + api::enums::SerialProxyMode mode) { +#ifdef USE_API + // Only the live subscriber may change the mode, so the mode cannot outlive a session + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring mode request from client without port subscription [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; + } +#endif + // Values come from a remote client + if (mode != api::enums::SERIAL_PROXY_MODE_RAW && mode != api::enums::SERIAL_PROXY_MODE_PROTOCOL) { + ESP_LOGW(TAG, "Invalid mode: %" PRIu32, static_cast(mode)); + return SerialProxyResult::SERIAL_PROXY_RESULT_INVALID_ARGUMENT; + } + // PROTOCOL on a port with no tap would be a silent no-op; refuse so the client knows +#ifdef USE_SERIAL_PROXY_TAP + const bool has_tap = this->tap_ != nullptr; +#else + const bool has_tap = false; +#endif + if (mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL && !has_tap) { + ESP_LOGW(TAG, "No tap on serial proxy [%" PRIu32 "]; PROTOCOL mode unavailable", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; + } + ESP_LOGD(TAG, "Serial proxy [%" PRIu32 "] mode set to %s", this->instance_index_, + mode == api::enums::SERIAL_PROXY_MODE_PROTOCOL ? LOG_STR_LITERAL("PROTOCOL") : LOG_STR_LITERAL("RAW")); +#ifdef USE_SERIAL_PROXY_TAP + const bool leaving_protocol_mode = + this->mode_ != api::enums::SERIAL_PROXY_MODE_RAW && mode == api::enums::SERIAL_PROXY_MODE_RAW; + this->mode_ = mode; + + // Only for an explicit client request, not for reset_mode_() at the end of a session: + // an ordinary disconnect says nothing about the device, whereas a client deliberately + // asking for raw bytes usually precedes changing what the device is. + if (leaving_protocol_mode && this->tap_ != nullptr) { + this->tap_->on_protocol_disabled(); + } +#endif + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; +} + void SerialProxy::write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) { #ifdef USE_API - // Bytes from a client other than the live subscriber would interleave with the - // subscriber's traffic on the wire - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring write from client without port access [%" PRIu32 "]", this->instance_index_); + // Bytes from anyone but the live subscriber would interleave with the subscriber's + // traffic -- or with an active tap's -- on the wire + if (!this->is_subscriber_(api_connection)) { + if (this->api_connection_ != nullptr) { + ESP_LOGW(TAG, "Ignoring write from client that does not hold serial proxy [%" PRIu32 "]", this->instance_index_); + } else { + // A legacy client streaming writes without subscribing would flood WARN, one per + // request; writes are the only high-rate, unacknowledged operation, so keep this + // visible without drowning the log + ESP_LOGV(TAG, "Ignoring write from client without port subscription [%" PRIu32 "]", this->instance_index_); + } return; } #endif if (data == nullptr || len == 0) return; this->write_array(data, len); + +#ifdef USE_SERIAL_PROXY_TAP + // After the write, so the tap observes the same ordering the device does + if (this->tap_observing_()) { + this->tap_->on_client_tx(data, len); + } +#endif } SerialProxyResult SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { #ifdef USE_API - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring modem pin request from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring modem pin request from client without port subscription [%" PRIu32 "]", + this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -210,8 +341,8 @@ uint32_t SerialProxy::get_modem_pins() const { SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { #ifdef USE_API // Flushing stalls the port, so it gets the same ownership check as writes - if (this->port_claimed_by_other_(api_connection)) { - ESP_LOGW(TAG, "Ignoring flush from client without port access [%" PRIu32 "]", this->instance_index_); + if (!this->is_subscriber_(api_connection)) { + ESP_LOGW(TAG, "Ignoring flush from client without port subscription [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } #endif @@ -230,11 +361,6 @@ SerialProxyResult SerialProxy::flush_port(api::APIConnection *api_connection) { } #ifdef USE_API -bool SerialProxy::port_claimed_by_other_(api::APIConnection *api_connection) const { - return this->api_connection_ != nullptr && this->api_connection_ != api_connection && - this->api_connection_->is_connection_setup(); -} - SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_connection, api::enums::SerialProxyRequestType type) { switch (type) { @@ -252,6 +378,10 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_PORT_IN_USE; } ESP_LOGW(TAG, "Previous subscriber disconnected; taking over subscription"); + // End the dead client's session before starting the new one, so its mode + // cannot leak into a session that never asked for it + this->api_connection_ = nullptr; + this->reset_mode_(); } this->api_connection_ = api_connection; this->enable_loop(); @@ -264,7 +394,15 @@ SerialProxyResult SerialProxy::serial_proxy_request(api::APIConnection *api_conn return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } this->api_connection_ = nullptr; + this->reset_mode_(); +#ifdef USE_SERIAL_PROXY_TAP + // Keep the loop alive for a tap that still needs the port (mirrors loop()) + if (this->tap_ == nullptr || !this->tap_->tap_needs_port()) { + this->disable_loop(); + } +#else this->disable_loop(); +#endif ESP_LOGV(TAG, "API connection unsubscribed from serial proxy [%" PRIu32 "]", this->instance_index_); return SerialProxyResult::SERIAL_PROXY_RESULT_OK; default: diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h index a0e47ee686..e3f4264cfa 100644 --- a/esphome/components/serial_proxy/serial_proxy.h +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -26,6 +26,7 @@ class APIConnection; namespace enums { enum SerialProxyPortType : uint32_t; enum SerialProxyRequestType : uint32_t; +enum SerialProxyMode : uint32_t; } // namespace enums } // namespace esphome::api @@ -52,6 +53,36 @@ enum class SerialProxyResult : uint8_t { /// Maximum bytes to read from UART in a single loop iteration inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; +#ifdef USE_SERIAL_PROXY_TAP +/// Observes a port's traffic without owning it, and may inject bytes of its own. +/// +/// This exists so protocol-aware behaviour can be layered onto a plain byte pipe without +/// the pipe knowing anything about the protocol: the tap is compiled in only when some +/// component asks for one, so a proxy carrying an RS485 meter pays nothing for it. +/// +/// A tap is an observer, never a gatekeeper -- it cannot suppress or alter the bytes +/// flowing in either direction, so a misbehaving tap cannot corrupt the stream. +class SerialProxyTap { + public: + /// Bytes read from the device, before they are forwarded to any subscriber. + virtual void on_device_rx(const uint8_t *data, size_t len) = 0; + + /// Bytes a subscriber sent towards the device, after they have been written. + virtual void on_client_tx(const uint8_t *data, size_t len) = 0; + + /// True when the port must keep reading even with no subscriber attached, so a tap can + /// do its own protocol work while nobody is listening. Honoured only while no + /// subscriber holds the port; with one attached, the port mode alone decides. + virtual bool tap_needs_port() const = 0; + + /// A client explicitly turned protocol handling off for this port. Distinct from the + /// automatic reset when a session ends: this one means a client intends to do something + /// else with the device -- reflash it, most likely -- so anything the tap believes about + /// it should be treated as suspect. + virtual void on_protocol_disabled() = 0; +}; +#endif + class SerialProxy final : public uart::UARTDevice, public Component { public: void setup() override; @@ -77,6 +108,9 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Get the port type api::enums::SerialProxyPortType get_port_type() const { return this->port_type_; } + /// Handle a mode change requested by an API client + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode); + /// Configure UART parameters and apply them /// @param api_connection The API connection requesting the change /// @param baudrate Baud rate in bits per second @@ -121,13 +155,67 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Set the DTR GPIO pin (from YAML configuration) void set_dtr_pin(GPIOPin *pin) { this->dtr_pin_ = pin; } +#ifdef USE_SERIAL_PROXY_TAP + /// Attach a traffic observer. At most one, set once at setup time. + void set_tap(SerialProxyTap *tap) { this->tap_ = tap; } + + /// Write bytes originating from the tap rather than from a client. Bypasses the + /// subscriber ownership check, but only while the tap is being served bytes -- so a + /// port in RAW mode with a subscriber attached stays inert. Returns false when the + /// bytes were dropped for that reason. + bool write_from_tap(const uint8_t *data, size_t len) { + if (!this->tap_observing_()) { + return false; + } + this->write_array(data, len); + return true; + } + + /// Whether the tap is currently being served bytes. Can flip false with no callback + /// (a subscriber attaching in RAW mode, say), so a tap should check before starting + /// protocol work and when a reply seems overdue. + bool tap_is_observed() const { return this->tap_observing_(); } + + /// Resume reading after a tap's needs change. loop() disables itself when there is + /// neither a subscriber nor a tap that wants the port, so a tap starting fresh work + /// must ask for it back. Must be called from the main loop. + void tap_request_port() { this->enable_loop(); } + + /// Whether the underlying device is present. On a USB UART this tracks enumeration, so + /// a tap can notice the device being unplugged and plugged back in. + bool is_device_connected() const { return this->parent_->is_connected(); } + + /// Run one read-and-dispatch cycle immediately. Lets a tap make progress before the + /// main loop is running -- during setup, for instance, while a component is still + /// blocking on can_proceed(). Must not be called from on_device_rx() or + /// on_client_tx(): each nested cycle costs a 256-byte stack frame. + void tap_pump(); +#endif + protected: #ifdef USE_API - /// Read from UART and send to API client (slow path with 256-byte stack buffer) + /// Read from UART, hand the bytes to any tap, and forward them to a subscriber + /// (slow path with a 256-byte stack buffer) void read_and_send_(size_t available); - /// True when a live subscriber other than the given connection holds the port - bool port_claimed_by_other_(api::APIConnection *api_connection) const; + /// True when the given connection is the live subscriber. Every port operation + /// (write, configure, modem pins, flush, mode) requires this, so an unsubscribed + /// client can never share the wire with the subscriber or an active tap. + bool is_subscriber_(api::APIConnection *api_connection) const { return this->api_connection_ == api_connection; } +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// Return the port to RAW when a subscriber goes away, so the mode never outlives it + void reset_mode_(); +#else + /// Without a tap, PROTOCOL is refused, so the mode is fixed at RAW and there is + /// nothing to reset + void reset_mode_() {} +#endif + +#ifdef USE_SERIAL_PROXY_TAP + /// True when the tap should be shown the traffic passing through this port + bool tap_observing_() const; #endif /// Instance index for identifying this proxy in API messages @@ -147,6 +235,11 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Port type api::enums::SerialProxyPortType port_type_{}; +#ifdef USE_SERIAL_PROXY_TAP + /// How the bytes passing through are treated; zero is SERIAL_PROXY_MODE_RAW + api::enums::SerialProxyMode mode_{}; +#endif + /// Optional GPIO pins for modem control GPIOPin *rts_pin_{nullptr}; GPIOPin *dtr_pin_{nullptr}; @@ -154,6 +247,10 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Current modem pin states bool rts_state_{false}; bool dtr_state_{false}; + +#ifdef USE_SERIAL_PROXY_TAP + SerialProxyTap *tap_{nullptr}; +#endif }; } // namespace esphome::serial_proxy diff --git a/esphome/core/defines.h b/esphome/core/defines.h index eaece6d5ff..c3b16d833a 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -181,6 +181,7 @@ #define USE_SENSOR #define USE_SENSOR_FILTER #define USE_SERIAL_PROXY +#define USE_SERIAL_PROXY_TAP #define USE_SETUP_PRIORITY_OVERRIDE #define USE_STATUS_LED #define USE_STATUS_SENSOR diff --git a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h index 6fc20f3350..7da6fff017 100644 --- a/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h +++ b/tests/benchmarks/stubs/esphome/components/serial_proxy/serial_proxy.h @@ -40,6 +40,9 @@ class SerialProxy { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } void write_from_client(api::APIConnection *api_connection, const uint8_t *data, size_t len) {} + SerialProxyResult set_mode_from_client(api::APIConnection *api_connection, api::enums::SerialProxyMode mode) { + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } SerialProxyResult set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { return SerialProxyResult::SERIAL_PROXY_RESULT_OK; } diff --git a/tests/components/serial_proxy/test-tap.esp32-idf.yaml b/tests/components/serial_proxy/test-tap.esp32-idf.yaml new file mode 100644 index 0000000000..5522e53c47 --- /dev/null +++ b/tests/components/serial_proxy/test-tap.esp32-idf.yaml @@ -0,0 +1,14 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +# Compile the tap code paths; no tap is attached, so this exercises the +# null-tap branches that a normal build never defines. +esphome: + platformio_options: + build_flags: + - "-DUSE_SERIAL_PROXY_TAP" + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + serial_proxy: !include common.yaml From a807a8f9451b172abf4cb05ca2f35c609224e414 Mon Sep 17 00:00:00 2001 From: matt123p Date: Thu, 10 Sep 2026 16:11:43 +0100 Subject: [PATCH 43/55] [es7210] Fix 4 channel microphone support (#19034) --- esphome/components/es7210/es7210.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esphome/components/es7210/es7210.cpp b/esphome/components/es7210/es7210.cpp index 892b67b270..5afc22aec4 100644 --- a/esphome/components/es7210/es7210.cpp +++ b/esphome/components/es7210/es7210.cpp @@ -153,13 +153,14 @@ bool ES7210::configure_mic_gain_() { ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC2_GAIN_REG44, 0x0f, regv)); // Configure mic 3 - ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x0b, 0x00)); + // MIC3 uses the ADC3/4 and MIC3/4 clock domains (bits 2 and 4), not the MIC1/2 domains. + ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x15, 0x00)); ES7210_ERROR_CHECK(this->write_byte(ES7210_MIC34_POWER_REG4C, 0x00)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC3_GAIN_REG45, 0x10, 0x10)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC3_GAIN_REG45, 0x0f, regv)); // Configure mic 4 - ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x0b, 0x00)); + ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_CLOCK_OFF_REG01, 0x15, 0x00)); ES7210_ERROR_CHECK(this->write_byte(ES7210_MIC34_POWER_REG4C, 0x00)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC4_GAIN_REG46, 0x10, 0x10)); ES7210_ERROR_CHECK(this->es7210_update_reg_bit_(ES7210_MIC4_GAIN_REG46, 0x0f, regv)); From 7564f5ff1ace9bbe107ec79107bd6606796f3156 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Thu, 10 Sep 2026 12:06:58 -0400 Subject: [PATCH 44/55] [sendspin] Add manufacturer, model, and firmware version options (#18792) Co-authored-by: J. Nick Koston --- esphome/components/sendspin/__init__.py | 32 +++++++ esphome/components/sendspin/sendspin_hub.cpp | 16 +++- esphome/components/sendspin/sendspin_hub.h | 19 +++++ .../sendspin/config/device_info_default.yaml | 12 +++ .../sendspin/config/device_info_explicit.yaml | 18 ++++ .../sendspin/config/device_info_project.yaml | 15 ++++ .../sendspin/test_device_info.py | 83 +++++++++++++++++++ tests/components/sendspin/common-hub.yaml | 3 + 8 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/sendspin/config/device_info_default.yaml create mode 100644 tests/component_tests/sendspin/config/device_info_explicit.yaml create mode 100644 tests/component_tests/sendspin/config/device_info_project.yaml create mode 100644 tests/component_tests/sendspin/test_device_info.py diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index c1970ab132..c21047c70a 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -6,12 +6,17 @@ from esphome.components import esp32, network, psram, socket, wifi import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, + CONF_ESPHOME, CONF_FORMAT, CONF_HEIGHT, CONF_ID, + CONF_MODEL, + CONF_NAME, + CONF_PROJECT, CONF_SAMPLE_RATE, CONF_SOURCE, CONF_TASK_STACK_IN_PSRAM, + CONF_VERSION, CONF_WIDTH, ) from esphome.core import CORE, ID @@ -27,6 +32,14 @@ DOMAIN = "sendspin" CONF_DISPLAY_OFFSET = "display_offset" CONF_SENDSPIN_ID = "sendspin_id" +CONF_FIRMWARE_VERSION = "firmware_version" +CONF_MANUFACTURER = "manufacturer" + +# An empty device information string would be sent to the server as an empty value rather than +# falling back, so reject it instead of silently substituting the fallback. The 127 byte cap keeps +# the length prefix of a protobuf string field to a single byte, matching `esphome: project:`. +DEVICE_INFO_STRING = cv.All(cv.string_strict, cv.Length(min=1), cv.ByteLength(max=127)) + CONF_INITIAL_STATIC_DELAY = "initial_static_delay" CONF_FIXED_DELAY = "fixed_delay" CONF_DECODE_MEMORY = "decode_memory" @@ -198,6 +211,9 @@ CONFIG_SCHEMA = cv.All( { cv.GenerateID(): cv.declare_id(SendspinHub), cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, + cv.Optional(CONF_MANUFACTURER): DEVICE_INFO_STRING, + cv.Optional(CONF_MODEL): DEVICE_INFO_STRING, + cv.Optional(CONF_FIRMWARE_VERSION): DEVICE_INFO_STRING, } ), cv.only_on_esp32, @@ -248,6 +264,22 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_task_stack_in_psram(True)) psram.request_external_task_stack() + # Device information for the server's client/hello message. Falls back to the project + # information, which is written as `manufacturer.model`. Anything still unset keeps the + # default the hub itself applies: the ESPHome name and version. + project = CORE.config[CONF_ESPHOME].get(CONF_PROJECT, {}) + project_manufacturer, _, project_model = project.get(CONF_NAME, "").partition(".") + for value, setter in ( + (config.get(CONF_MANUFACTURER) or project_manufacturer, var.set_manufacturer), + (config.get(CONF_MODEL) or project_model, var.set_model), + ( + config.get(CONF_FIRMWARE_VERSION) or project.get(CONF_VERSION), + var.set_firmware_version, + ), + ): + if value: + cg.add(setter(value)) + # sendspin-cpp library esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.7.2") diff --git a/esphome/components/sendspin/sendspin_hub.cpp b/esphome/components/sendspin/sendspin_hub.cpp index 028491284a..2cb2b90995 100644 --- a/esphome/components/sendspin/sendspin_hub.cpp +++ b/esphome/components/sendspin/sendspin_hub.cpp @@ -76,8 +76,12 @@ void SendspinHub::dump_config() { ESP_LOGCONFIG(TAG, "Sendspin Hub:\n" " Client ID: %s\n" + " Manufacturer: %s\n" + " Model: %s\n" + " Firmware version: %s\n" " Task stack in PSRAM: %s", - get_client_id_into_buffer(mac_buf), YESNO(this->task_stack_in_psram_)); + get_client_id_into_buffer(mac_buf), this->manufacturer_, this->get_product_name_(), + this->firmware_version_, YESNO(this->task_stack_in_psram_)); #ifdef USE_SENDSPIN_ARTWORK // Slot indices come from the order the image platform entries were declared, so the log is the @@ -127,15 +131,19 @@ const char *SendspinHub::get_client_id_into_buffer(std::spanmodel_ != nullptr ? this->model_ : App.get_name().c_str(); +} + sendspin::SendspinClientConfig SendspinHub::build_client_config_() { sendspin::SendspinClientConfig config; char mac_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; config.client_id = SendspinHub::get_client_id_into_buffer(mac_buf); config.name = App.get_friendly_name(); - config.product_name = App.get_name(); - config.manufacturer = "ESPHome"; - config.software_version = ESPHOME_VERSION; + config.product_name = this->get_product_name_(); + config.manufacturer = this->manufacturer_; + config.software_version = this->firmware_version_; config.httpd_psram_stack = this->task_stack_in_psram_; return config; diff --git a/esphome/components/sendspin/sendspin_hub.h b/esphome/components/sendspin/sendspin_hub.h index 7c50c3eb80..c66c7db3cc 100644 --- a/esphome/components/sendspin/sendspin_hub.h +++ b/esphome/components/sendspin/sendspin_hub.h @@ -8,6 +8,7 @@ #include "esphome/core/component.h" #include "esphome/core/helpers.h" #include "esphome/core/preferences.h" +#include "esphome/core/version.h" #include #include @@ -125,6 +126,15 @@ class SendspinHub final : public Component, void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + /// @brief Sets the device information reported to the server in the `client/hello` message. + /// + /// Each takes a pointer to a string literal emitted by codegen, so it must stay valid for the + /// lifetime of the hub. Only called for values the configuration overrides; anything left alone + /// keeps the default described on the member below. + void set_manufacturer(const char *manufacturer) { this->manufacturer_ = manufacturer; } + void set_model(const char *model) { this->model_ = model; } + void set_firmware_version(const char *firmware_version) { this->firmware_version_ = firmware_version; } + // --- Sendspin role specific methods --- #ifdef USE_SENDSPIN_ARTWORK @@ -187,6 +197,9 @@ class SendspinHub final : public Component, /// @brief Builds the SendspinClientConfig from ESPHome configuration and platform info. sendspin::SendspinClientConfig build_client_config_(); + /// @brief Returns the product name reported to the server: the configured model, or the device name. + const char *get_product_name_() const; + /// @brief Writes the active network interface's MAC into @p buf and returns its data pointer. /// Uses the ethernet MAC if ethernet is configured, otherwise the base MAC (used by wifi). static const char *get_client_id_into_buffer(std::span buf); @@ -268,6 +281,12 @@ class SendspinHub final : public Component, CallbackManager group_update_callbacks_{}; bool task_stack_in_psram_{false}; + + // Device information sent in the `client/hello` message. Defaults apply when neither the + // sendspin configuration nor the project information supplies a value. + const char *manufacturer_{"ESPHome"}; + const char *model_{nullptr}; // nullptr reports the device name instead + const char *firmware_version_{ESPHOME_VERSION}; }; /// @brief Base class for all sendspin subcomponents. diff --git a/tests/component_tests/sendspin/config/device_info_default.yaml b/tests/component_tests/sendspin/config/device_info_default.yaml new file mode 100644 index 0000000000..669b2e99bc --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_default.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: diff --git a/tests/component_tests/sendspin/config/device_info_explicit.yaml b/tests/component_tests/sendspin/config/device_info_explicit.yaml new file mode 100644 index 0000000000..c3fec3ead4 --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_explicit.yaml @@ -0,0 +1,18 @@ +esphome: + name: test + project: + name: project_manufacturer.project_model + version: 9.9.9 + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: + manufacturer: Explicit Manufacturer + model: Explicit Model + firmware_version: 1.2.3 diff --git a/tests/component_tests/sendspin/config/device_info_project.yaml b/tests/component_tests/sendspin/config/device_info_project.yaml new file mode 100644 index 0000000000..395b2889fc --- /dev/null +++ b/tests/component_tests/sendspin/config/device_info_project.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + project: + name: project_manufacturer.project_model + version: 9.9.9 + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ap: + +sendspin: diff --git a/tests/component_tests/sendspin/test_device_info.py b/tests/component_tests/sendspin/test_device_info.py new file mode 100644 index 0000000000..833dd398b4 --- /dev/null +++ b/tests/component_tests/sendspin/test_device_info.py @@ -0,0 +1,83 @@ +"""Tests for the device information the sendspin hub reports to the server.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components.sendspin import ( + CONF_FIRMWARE_VERSION, + CONF_MANUFACTURER, + CONFIG_SCHEMA, +) +from esphome.const import CONF_MODEL, PlatformFramework +from tests.component_tests.types import SetCoreConfigCallable + + +def test_explicit_device_info_wins_over_project( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Configured values take precedence over the project information.""" + main_cpp = generate_main(component_config_path("device_info_explicit.yaml")) + + assert 'set_manufacturer("Explicit Manufacturer")' in main_cpp + assert 'set_model("Explicit Model")' in main_cpp + assert 'set_firmware_version("1.2.3")' in main_cpp + + +def test_project_supplies_device_info( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Without configured values, the project name splits into manufacturer and model.""" + main_cpp = generate_main(component_config_path("device_info_project.yaml")) + + assert 'set_manufacturer("project_manufacturer")' in main_cpp + assert 'set_model("project_model")' in main_cpp + assert 'set_firmware_version("9.9.9")' in main_cpp + + +def test_no_device_info_leaves_hub_defaults( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """With neither source, nothing is emitted and the hub keeps its own defaults.""" + main_cpp = generate_main(component_config_path("device_info_default.yaml")) + + assert "set_manufacturer(" not in main_cpp + assert "set_model(" not in main_cpp + assert "set_firmware_version(" not in main_cpp + + +@pytest.mark.parametrize( + "conf_key", [CONF_MANUFACTURER, CONF_MODEL, CONF_FIRMWARE_VERSION] +) +def test_empty_device_info_rejected( + set_core_config: SetCoreConfigCallable, conf_key: str +) -> None: + """An empty string would be sent to the server as an empty value, so it is not accepted.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: ""}) + + +@pytest.mark.parametrize( + "conf_key", [CONF_MANUFACTURER, CONF_MODEL, CONF_FIRMWARE_VERSION] +) +def test_device_info_capped_at_127_bytes( + set_core_config: SetCoreConfigCallable, conf_key: str +) -> None: + """The cap is in bytes so the protobuf length prefix stays a single byte.""" + set_core_config(PlatformFramework.ESP32_IDF) + + CONFIG_SCHEMA({conf_key: "a" * 127}) + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: "a" * 128}) + # 64 two-byte characters is 128 bytes. + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA({conf_key: "é" * 64}) diff --git a/tests/components/sendspin/common-hub.yaml b/tests/components/sendspin/common-hub.yaml index 7a6a9ffd4f..bd6747ee07 100644 --- a/tests/components/sendspin/common-hub.yaml +++ b/tests/components/sendspin/common-hub.yaml @@ -4,3 +4,6 @@ psram: sendspin: id: sendspin_hub_id task_stack_in_psram: true + manufacturer: Test Manufacturer + model: Test Model + firmware_version: 1.2.3 From 3e3822e5541f3562fae64b29837b79b1027af674 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:22:30 +0000 Subject: [PATCH 45/55] Bump bundled esphome-device-builder to 1.14.6 (#19072) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ac84ee4689..cfa47fbdad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 RUN \ platformio settings set enable_telemetry No \ From f66ef23256f467a572517f6ee87956f5f527fa60 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 03:20:30 -0500 Subject: [PATCH 46/55] [core] Support set_internal() during setup, log error after setup (#19069) --- esphome/core/entity_base.cpp | 9 ++++ esphome/core/entity_base.h | 27 ++++++++---- .../fixtures/set_internal_at_boot.yaml | 34 +++++++++++++++ .../integration/test_set_internal_at_boot.py | 41 +++++++++++++++++++ 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/set_internal_at_boot.yaml create mode 100644 tests/integration/test_set_internal_at_boot.py diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 21a5fc3706..dc27c1e56a 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -56,6 +56,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, ui this->flags_.entity_category = (entity_fields >> ENTITY_FIELD_ENTITY_CATEGORY_SHIFT) & 0x3; } +void EntityBase::set_internal(bool internal) { + // Remove the after-setup path in 2027.3.0 and ignore the call instead. + if (App.is_setup_complete()) { + ESP_LOGE(TAG, "'%s': set_internal() after setup is undefined behavior, stops working in 2027.3.0", + this->get_name().c_str()); + } + this->flags_.internal = internal; +} + // Weak default lookup functions — overridden by generated code in main.cpp __attribute__((weak)) const char *entity_device_class_lookup(uint8_t) { return ""; } __attribute__((weak)) const char *entity_uom_lookup(uint8_t) { return ""; } diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index f38e30bf52..8796e9f067 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -88,13 +88,26 @@ class EntityBase { // Get whether this Entity should be hidden outside ESPHome bool is_internal() const { return this->flags_.internal; } - // Deprecated: Calling set_internal() at runtime is undefined behavior. Components and clients - // are NOT notified of the change, the flag may have already been read during setup, and there - // is NO guarantee any consumer will observe the new value. Use the 'internal:' YAML key instead. - ESPDEPRECATED("set_internal() is undefined behavior at runtime — components and Home Assistant are NOT " - "notified. Use the 'internal:' YAML key instead. Will be removed in 2027.3.0.", - "2026.3.0") - void set_internal(bool internal) { this->flags_.internal = internal; } + // Set whether this Entity should be hidden outside ESPHome. Prefer the 'internal:' YAML key + // whenever possible: it is guaranteed and has none of the limitations below. Use this only when + // the decision can only be made at boot. Must be called before MQTT and the API read the flag: + // from on_boot at the default priority, or a setup() that runs above setup_priority::AFTER_WIFI. + // If the answer comes from a device handshake, hold setup with can_proceed() until it arrives. + // Calls after setup finishes are undefined behavior: the flag is still written and an error is + // logged, and from 2027.3.0 the call will be ignored. + // + // Known limitations. Not bugs, so no issue reports please; a PR that removes one with no RAM + // or performance cost would be considered. + // - No consumer is notified of a change, so the flag can only be decided once per boot. + // - The guard is coarse: a call from a priority below AFTER_WIFI (an on_boot with a low priority, + // or a setup() at LATE) still passes, but the API camera listener is already registered, MQTT + // (AFTER_CONNECTION) has cached the flag, and an API client that connected while setup was + // stalled on a slow component has already listed the entities, so they keep the old value. + // - Un-hiding an entity declared 'internal: true' in YAML skips the duplicate name check that + // codegen runs for exposed entities, so a name collision can surface at runtime. Entities with + // only an 'id:' are forced internal and use the id as their name. + // - Zigbee codegen skips YAML internal entities entirely, so un-hiding cannot add them to Zigbee. + void set_internal(bool internal); // Check if this object is declared to be disabled by default. // That means that when the device gets added to Home Assistant (or other clients) it should diff --git a/tests/integration/fixtures/set_internal_at_boot.yaml b/tests/integration/fixtures/set_internal_at_boot.yaml new file mode 100644 index 0000000000..b3007e9dbd --- /dev/null +++ b/tests/integration/fixtures/set_internal_at_boot.yaml @@ -0,0 +1,34 @@ +esphome: + name: set-internal-at-boot + on_boot: + then: + - lambda: |- + id(hidden_at_boot).set_internal(true); + id(shown_at_boot).set_internal(false); + +host: + +api: + actions: + - action: set_internal_late + then: + - lambda: id(untouched).set_internal(true); + +logger: + +sensor: + - platform: template + name: "Hidden At Boot" + id: hidden_at_boot + lambda: return 1.0; + + - platform: template + name: "Shown At Boot" + id: shown_at_boot + internal: true + lambda: return 2.0; + + - platform: template + name: "Untouched" + id: untouched + lambda: return 3.0; diff --git a/tests/integration/test_set_internal_at_boot.py b/tests/integration/test_set_internal_at_boot.py new file mode 100644 index 0000000000..68b0bd1080 --- /dev/null +++ b/tests/integration/test_set_internal_at_boot.py @@ -0,0 +1,41 @@ +"""Integration test for set_internal() called during and after setup.""" + +from __future__ import annotations + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_set_internal_at_boot( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """set_internal() in on_boot changes API exposure, later calls log an error.""" + waiter = LineWaiter() + + async with ( + run_compiled(yaml_config, line_callback=waiter.callback), + api_client_connected() as client, + ): + entities, services = await client.list_entities_services() + names = {entity.name for entity in entities} + + assert "Hidden At Boot" not in names + assert "Shown At Boot" in names + assert "Untouched" in names + + late = next(s for s in services if s.name == "set_internal_late") + await client.execute_service(late, {}) + await waiter.wait_for( + "'Untouched'", + "set_internal() after setup is undefined behavior", + timeout=5.0, + ) + + # Still written during the deprecation window, ignored from 2027.3.0 + entities, _ = await client.list_entities_services() + assert "Untouched" not in {entity.name for entity in entities} From 380938177c1cc0599f4df97f1368adb96f48f07a Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Fri, 11 Sep 2026 03:25:52 -0500 Subject: [PATCH 47/55] [uart] Add apply_settings_live() for in-place ESP-IDF reconfiguration (#19087) Co-authored-by: Claude Fable 5.1 --- .../uart/uart_component_esp_idf.cpp | 129 +++++++++++++----- .../components/uart/uart_component_esp_idf.h | 36 +++++ 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/esphome/components/uart/uart_component_esp_idf.cpp b/esphome/components/uart/uart_component_esp_idf.cpp index bbeb86bcdb..e5d5fbc983 100644 --- a/esphome/components/uart/uart_component_esp_idf.cpp +++ b/esphome/components/uart/uart_component_esp_idf.cpp @@ -160,6 +160,7 @@ void IDFUARTComponent::load_settings(bool dump_config) { this->mark_failed(); return; } + this->last_good_framing_ = this->framing_(); int8_t tx = this->tx_pin_ != nullptr ? this->tx_pin_->get_pin() : -1; int8_t rx = this->rx_pin_ != nullptr ? this->rx_pin_->get_pin() : -1; @@ -189,18 +190,9 @@ void IDFUARTComponent::load_settings(bool dump_config) { setup_pin_if_needed(this->tx_pin_); } - uint32_t invert = 0; - if (this->tx_pin_ != nullptr && this->tx_pin_->is_inverted()) { - invert |= UART_SIGNAL_TXD_INV; - } - if (this->rx_pin_ != nullptr && this->rx_pin_->is_inverted()) { - invert |= UART_SIGNAL_RXD_INV; - } - if (this->flow_control_pin_ != nullptr && this->flow_control_pin_->is_inverted()) { - invert |= UART_SIGNAL_RTS_INV; - } - - err = uart_set_line_inverse(this->uart_num_, invert); + // Must precede uart_set_pin() so an inverted TX line never shows the wrong idle + // level; apply_line_settings_() repeats it later for the reset registers. + err = uart_set_line_inverse(this->uart_num_, this->line_inversion_mask_()); if (err != ESP_OK) { ESP_LOGW(TAG, "uart_set_line_inverse failed: %s", esp_err_to_name(err)); this->mark_failed(); @@ -214,25 +206,7 @@ void IDFUARTComponent::load_settings(bool dump_config) { return; } - err = uart_set_rx_full_threshold(this->uart_num_, this->rx_full_threshold_); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_rx_full_threshold failed: %s", esp_err_to_name(err)); - this->mark_failed(); - return; - } - - err = uart_set_rx_timeout(this->uart_num_, this->rx_timeout_); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_rx_timeout failed: %s", esp_err_to_name(err)); - this->mark_failed(); - return; - } - - // Per ESP-IDF docs, uart_set_mode() must be called only after uart_driver_install(). - auto mode = this->flow_control_pin_ != nullptr ? UART_MODE_RS485_HALF_DUPLEX : UART_MODE_UART; - err = uart_set_mode(this->uart_num_, mode); - if (err != ESP_OK) { - ESP_LOGW(TAG, "uart_set_mode failed: %s", esp_err_to_name(err)); + if (this->apply_line_settings_() != ESP_OK) { this->mark_failed(); return; } @@ -250,6 +224,99 @@ void IDFUARTComponent::load_settings(bool dump_config) { } } +uint32_t IDFUARTComponent::line_inversion_mask_() { + uint32_t invert = 0; + if (this->tx_pin_ != nullptr && this->tx_pin_->is_inverted()) { + invert |= UART_SIGNAL_TXD_INV; + } + if (this->rx_pin_ != nullptr && this->rx_pin_->is_inverted()) { + invert |= UART_SIGNAL_RXD_INV; + } + if (this->flow_control_pin_ != nullptr && this->flow_control_pin_->is_inverted()) { + invert |= UART_SIGNAL_RTS_INV; + } + return invert; +} + +esp_err_t IDFUARTComponent::apply_line_settings_() { + // uart_param_config() resets these; call after every use of it. + esp_err_t err = uart_set_line_inverse(this->uart_num_, this->line_inversion_mask_()); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_line_inverse failed: %s", esp_err_to_name(err)); + return err; + } + + err = uart_set_rx_full_threshold(this->uart_num_, this->rx_full_threshold_); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_rx_full_threshold failed: %s", esp_err_to_name(err)); + return err; + } + + err = uart_set_rx_timeout(this->uart_num_, this->rx_timeout_); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_rx_timeout failed: %s", esp_err_to_name(err)); + return err; + } + + // Per ESP-IDF docs, uart_set_mode() must be called only after uart_driver_install(). + auto mode = this->flow_control_pin_ != nullptr ? UART_MODE_RS485_HALF_DUPLEX : UART_MODE_UART; + err = uart_set_mode(this->uart_num_, mode); + if (err != ESP_OK) { + ESP_LOGW(TAG, "uart_set_mode failed: %s", esp_err_to_name(err)); + return err; + } + + return ESP_OK; +} + +void IDFUARTComponent::set_framing_(const Framing &framing) { + this->baud_rate_ = framing.baud_rate; + this->data_bits_ = framing.data_bits; + this->stop_bits_ = framing.stop_bits; + this->parity_ = framing.parity; + this->rx_full_threshold_ = framing.rx_full_threshold; +} + +esp_err_t IDFUARTComponent::apply_settings_live() { + if (this->is_failed()) { + return ESP_ERR_INVALID_STATE; + } + // No driver yet: nothing to reconfigure in place. + if (!uart_is_driver_installed(this->uart_num_)) { + this->load_settings(false); + return this->is_failed() ? ESP_FAIL : ESP_OK; + } + // Keeps the driver ring buffers; flushes both hardware FIFOs (in-flight bytes lost). + uart_config_t uart_config = this->get_config_(); + esp_err_t err = uart_param_config(this->uart_num_, &uart_config); + if (err != ESP_OK) { + // Failure leaves the registers reset; put back the last accepted framing so the + // getters still describe the hardware. + if (this->last_good_framing_.baud_rate == 0) { + ESP_LOGE(TAG, "uart_param_config (live) failed: %s; no previous framing to restore", esp_err_to_name(err)); + this->mark_failed(); + return err; + } + ESP_LOGW(TAG, "uart_param_config (live) failed: %s; restoring %" PRIu32 " baud", esp_err_to_name(err), + this->last_good_framing_.baud_rate); + this->set_framing_(this->last_good_framing_); + uart_config = this->get_config_(); + esp_err_t restore_err = uart_param_config(this->uart_num_, &uart_config); + if (restore_err != ESP_OK) { + ESP_LOGE(TAG, "UART left unconfigured after failed live reconfigure: %s", esp_err_to_name(restore_err)); + this->mark_failed(); + return err; + } + // Previous framing is live again; report the refusal (line-setting errors log). + this->apply_line_settings_(); + return err; + } + this->last_good_framing_ = this->framing_(); + // The new framing is live; a line-setting failure here only logs. + this->apply_line_settings_(); + return ESP_OK; +} + void IDFUARTComponent::dump_config() { ESP_LOGCONFIG(TAG, "UART Bus %u:", this->uart_num_); LOG_PIN(" TX Pin: ", this->tx_pin_); diff --git a/esphome/components/uart/uart_component_esp_idf.h b/esphome/components/uart/uart_component_esp_idf.h index a761d80f04..d9297bfa34 100644 --- a/esphome/components/uart/uart_component_esp_idf.h +++ b/esphome/components/uart/uart_component_esp_idf.h @@ -52,13 +52,49 @@ class IDFUARTComponent final : public UARTComponent, public Component { void load_settings(bool dump_config) override; using UARTComponent::load_settings; // also bring in the no-arg overload for convenience + /** + * Apply the current framing (baud rate, parity, data/stop bits) to the installed + * driver in place, without the delete/reinstall of load_settings(). Tasks blocked in + * the driver survive and the ring buffers are kept, but both hardware FIFOs are + * flushed: a frame in flight reaches the peer truncated and bytes not yet out of the + * RX FIFO are dropped. No lock is taken: quiesce writers first if that matters. + * rx_full_threshold is not rescaled (call set_rx_full_threshold_ms() first if it + * should follow the baud rate); a rollback restores the value from the last accepted + * configuration, undoing a standalone set_rx_full_threshold() made since. Without an + * installed driver this is a full load_settings(false) instead. + * + * @return ESP_OK once the new framing is live (a line-setting error after that only + * logs). On rejection (unreachable baud rate) the previous framing is restored and + * the driver's error returned; if the restore fails too the component is marked + * failed. ESP_ERR_INVALID_STATE if already failed; ESP_FAIL if the fallback + * load_settings() fails. + */ + esp_err_t apply_settings_live(); + void on_shutdown() override; protected: void check_logger_conflict() override; + uint32_t line_inversion_mask_(); + // Re-applies what uart_param_config() resets: inversion, RX threshold/timeout, mode. + esp_err_t apply_line_settings_(); uart_port_t uart_num_{UART_NUM_MAX}; uart_config_t get_config_(); + struct Framing { + uint32_t baud_rate; + uint8_t data_bits; + uint8_t stop_bits; + UARTParityOptions parity; + size_t rx_full_threshold; // sized for the baud rate, so rolled back with it + }; + Framing framing_() const { + return {this->baud_rate_, this->data_bits_, this->stop_bits_, this->parity_, this->rx_full_threshold_}; + } + void set_framing_(const Framing &framing); + // Last framing the driver accepted; baud_rate 0 means none yet. + Framing last_good_framing_{}; + bool has_peek_{false}; uint8_t peek_byte_; uint32_t flush_timeout_ms_{0}; ///< 0 means wait indefinitely (portMAX_DELAY). From 6a21ab4ea705cb4c885cb2590683949874d9b931 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:30:51 +1200 Subject: [PATCH 48/55] [esp32] Trim mbedTLS to client-only defaults and stub vasprintf on the C6 (#19088) --- esphome/components/esp32/__init__.py | 118 ++++++++++++++++++ esphome/components/esp32/vasprintf_stubs.cpp | 53 ++++++++ esphome/components/openthread/__init__.py | 10 ++ esphome/components/wifi/__init__.py | 7 ++ esphome/core/defines.h | 1 + .../esp32/config/mbedtls_tls_default.yaml | 14 +++ .../esp32/config/mbedtls_tls_openthread.yaml | 19 +++ .../esp32/config/mbedtls_tls_opt_out.yaml | 17 +++ .../config/mbedtls_tls_user_sdkconfig.yaml | 17 +++ .../esp32/config/mbedtls_tls_wifi_eap.yaml | 17 +++ .../esp32/config/vasprintf_stub_c6.yaml | 7 ++ .../config/vasprintf_stub_c6_full_printf.yaml | 9 ++ tests/component_tests/esp32/test_esp32.py | 99 +++++++++++++++ tests/components/esp32/test.esp32-idf.yaml | 2 + .../http_request/test.esp32-c6-idf.yaml | 4 + 15 files changed, 394 insertions(+) create mode 100644 esphome/components/esp32/vasprintf_stubs.cpp create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_default.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml create mode 100644 tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml create mode 100644 tests/component_tests/esp32/config/vasprintf_stub_c6.yaml create mode 100644 tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml create mode 100644 tests/components/http_request/test.esp32-c6-idf.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 3f5a34bc73..d027c9a1c6 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -189,6 +189,13 @@ PSRAM_XIP_VARIANTS = { VARIANT_ESP32S31, } +# Variants whose ROM exports a full-format vsnprintf but no vasprintf +# (esp32c6.rom.newlib-normal.ld). There, the newlib printf engine is only +# linked because esp_http_client calls vasprintf; see vasprintf_stubs.cpp. +# The other variants either export both (classic ESP32, nano-format only) or +# neither, so the engine is already in the image and the wrap saves nothing. +ROM_VSNPRINTF_WITHOUT_VASPRINTF_VARIANTS = {VARIANT_ESP32C6} + # NVS encryption (HMAC peripheral scheme) is only available on variants that # expose the HMAC peripheral (SOC_HMAC_SUPPORTED in soc_caps.h). The original # ESP32 and ESP32-C2 do not have it. New variants with an HMAC peripheral @@ -1732,6 +1739,8 @@ CONF_DISABLE_USB_SERIAL_JTAG_SECONDARY = "disable_usb_serial_jtag_secondary" CONF_DISABLE_DEV_NULL_VFS = "disable_dev_null_vfs" CONF_DISABLE_MBEDTLS_PEER_CERT = "disable_mbedtls_peer_cert" CONF_DISABLE_MBEDTLS_PKCS7 = "disable_mbedtls_pkcs7" +CONF_DISABLE_MBEDTLS_TLS_SERVER = "disable_mbedtls_tls_server" +CONF_DISABLE_MBEDTLS_TLS_EXTRAS = "disable_mbedtls_tls_extras" CONF_DISABLE_REGI2C_IN_IRAM = "disable_regi2c_in_iram" CONF_DISABLE_FATFS = "disable_fatfs" CONF_ADC_ONESHOT_IN_IRAM = "adc_oneshot_in_iram" @@ -1746,6 +1755,8 @@ KEY_VFS_TERMIOS_REQUIRED = "vfs_termios_required" KEY_USB_SERIAL_JTAG_SECONDARY_REQUIRED = "usb_serial_jtag_secondary_required" KEY_MBEDTLS_PEER_CERT_REQUIRED = "mbedtls_peer_cert_required" KEY_MBEDTLS_PKCS7_REQUIRED = "mbedtls_pkcs7_required" +KEY_MBEDTLS_TLS_SERVER_REQUIRED = "mbedtls_tls_server_required" +KEY_MBEDTLS_TLS_EXTRAS_REQUIRED = "mbedtls_tls_extras_required" KEY_FATFS_REQUIRED = "fatfs_required" KEY_MBEDTLS_SHA512_REQUIRED = "mbedtls_sha512_required" KEY_ADC_ONESHOT_IRAM_REQUIRED = "adc_oneshot_iram_required" @@ -1830,6 +1841,30 @@ def require_mbedtls_pkcs7() -> None: CORE.data[KEY_ESP32][KEY_MBEDTLS_PKCS7_REQUIRED] = True +def require_mbedtls_tls_server() -> None: + """Mark that the mbedTLS server-side TLS/DTLS handshake is required. + + Call this from components that accept TLS connections (OpenThread's DTLS + commissioner does). This prevents CONFIG_MBEDTLS_TLS_CLIENT_ONLY from + being selected. + """ + CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_SERVER_REQUIRED] = True + + +def require_mbedtls_tls_extras(options: Iterable[str] | None = None) -> None: + """Mark TLS features disabled by ``disable_mbedtls_tls_extras`` as required. + + ``options`` names the entries of ``MBEDTLS_TLS_EXTRA_OPTIONS`` to keep; + omit it to keep all of them. Call this from components that need AES-CCM, + deterministic ECDSA signing, static RSA/ECDH key exchange, TLS + renegotiation or session tickets, or that run a TLS client against + servers ESPHome cannot vet (wpa_supplicant's EAP client). A user-supplied + sdkconfig_options value is never overridden either. + """ + required = CORE.data[KEY_ESP32].setdefault(KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, set()) + required.update(MBEDTLS_TLS_EXTRA_OPTIONS if options is None else options) + + def require_mbedtls_sha512() -> None: """Mark that mbedTLS SHA-384/SHA-512 support is required by a component. @@ -1987,6 +2022,8 @@ FRAMEWORK_SCHEMA = cv.Schema( cv.Optional(CONF_DISABLE_DEV_NULL_VFS, default=True): cv.boolean, cv.Optional(CONF_DISABLE_MBEDTLS_PEER_CERT, default=True): cv.boolean, cv.Optional(CONF_DISABLE_MBEDTLS_PKCS7, default=True): cv.boolean, + cv.Optional(CONF_DISABLE_MBEDTLS_TLS_SERVER, default=True): cv.boolean, + cv.Optional(CONF_DISABLE_MBEDTLS_TLS_EXTRAS, default=True): cv.boolean, cv.Optional(CONF_DISABLE_REGI2C_IN_IRAM, default=True): cv.boolean, cv.Optional(CONF_ADC_ONESHOT_IN_IRAM, default=False): cv.boolean, cv.Optional(CONF_DISABLE_FATFS, default=True): cv.boolean, @@ -2302,6 +2339,69 @@ async def _reconcile_certificate_bundle_sdkconfig() -> None: set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) +# TLS features an HTTPS/MQTT client talking to a modern server never +# negotiates. Static RSA and static ECDH key exchange have no forward secrecy +# and are gone in TLS 1.3, renegotiation is deprecated, esp-tls never enables +# session tickets, AES-CCM ciphersuites are not offered by web servers, and +# deterministic ECDSA only matters when signing with a private key. Together +# they cost ~10 KB of flash whenever TLS is linked (http_request, mqtt). +# wpa_supplicant's EAP client is a second TLS client that talks to RADIUS +# servers ESPHome cannot vet, and a failed EAP handshake leaves the device +# off the network, so the wifi component re-enables all of these when eap is +# configured. +# The EC public key parsing extras stay enabled: they decide whether a peer +# certificate with a compressed point or explicit curve parameters parses, +# which no component can know ahead of time. +MBEDTLS_TLS_EXTRA_OPTIONS = ( + "CONFIG_MBEDTLS_KEY_EXCHANGE_RSA", + "CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA", + "CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_RSA", + "CONFIG_MBEDTLS_SSL_RENEGOTIATION", + "CONFIG_MBEDTLS_CLIENT_SSL_SESSION_TICKETS", + "CONFIG_MBEDTLS_SERVER_SSL_SESSION_TICKETS", + "CONFIG_MBEDTLS_CCM_C", + "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC", +) + +# Members of the mbedTLS "TLS Protocol Role" Kconfig choice. Setting one +# member is only valid when the user has not already chosen another. +MBEDTLS_TLS_ROLE_OPTIONS = ( + "CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", + "CONFIG_MBEDTLS_TLS_SERVER_ONLY", + "CONFIG_MBEDTLS_TLS_CLIENT_ONLY", + "CONFIG_MBEDTLS_TLS_DISABLED", +) + + +@coroutine_with_priority(CoroPriority.FINAL) +async def _reconcile_mbedtls_tls_sdkconfig( + disable_tls_server: bool, disable_tls_extras: bool +) -> None: + """Trim mbedTLS to what a TLS client needs unless a component asked otherwise. + + Runs at FINAL priority so every require_mbedtls_tls_server() and + require_mbedtls_tls_extras() call has happened. Only the server-side + handshake (~7 KB) is a separate option; nothing in ESPHome accepts TLS + connections, but OpenThread's DTLS commissioner does. A user-supplied + sdkconfig_options value always wins; for the TLS role choice, any member + the user set leaves the whole choice alone so the pair cannot conflict. + """ + data = CORE.data[KEY_ESP32] + sdkconfig = data[KEY_SDKCONFIG_OPTIONS] + if ( + disable_tls_server + and not data.get(KEY_MBEDTLS_TLS_SERVER_REQUIRED, False) + and not any(option in sdkconfig for option in MBEDTLS_TLS_ROLE_OPTIONS) + ): + add_idf_sdkconfig_option("CONFIG_MBEDTLS_TLS_CLIENT_ONLY", True) + add_idf_sdkconfig_option("CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", False) + if disable_tls_extras: + required = data.get(KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, set()) + for option in MBEDTLS_TLS_EXTRA_OPTIONS: + if option not in required: + set_idf_sdkconfig_default(option, False) + + @coroutine_with_priority(CoroPriority.FINAL) async def _reconcile_network_sdkconfig() -> None: """Reconcile WiFi/Ethernet/Bluetooth/coexistence sdkconfig flags. @@ -2566,6 +2666,17 @@ async def to_code(config): else: for symbol in ("vprintf", "printf", "fprintf", "vfprintf"): cg.add_build_flag(f"-Wl,--wrap={symbol}") + # esp_http_client calls vasprintf, which on the ESP32-C6 is the only + # reference to newlib's full printf engine (~20 KB: _svfprintf_r, + # _dtoa_r and their helpers); every other caller resolves to the + # ROM. See vasprintf_stubs.cpp. The --undefined flag is needed + # because libsrc.a is scanned before the IDF libraries that + # reference the symbol, so the stub would otherwise never be pulled + # from the archive. + if variant in ROM_VSNPRINTF_WITHOUT_VASPRINTF_VARIANTS: + cg.add_define("USE_ESP32_VASPRINTF_STUB") + cg.add_build_flag("-Wl,--wrap=vasprintf") + cg.add_build_flag("-Wl,--undefined=__wrap_vasprintf") else: cg.add_build_flag("-DUSE_ARDUINO") cg.add_build_flag("-DUSE_ESP32_FRAMEWORK_ARDUINO") @@ -2991,6 +3102,13 @@ async def to_code(config): # FINAL priority: runs after every require_certificate_bundle() call CORE.add_job(_reconcile_certificate_bundle_sdkconfig) + # FINAL priority: runs after every require_mbedtls_tls_*() call + CORE.add_job( + _reconcile_mbedtls_tls_sdkconfig, + advanced[CONF_DISABLE_MBEDTLS_TLS_SERVER], + advanced[CONF_DISABLE_MBEDTLS_TLS_EXTRAS], + ) + # FINAL: require_*() calls can come from to_code at or below this priority, so an # inline read would be iteration-order-dependent; reconcile once after every job ran. CORE.add_job( diff --git a/esphome/components/esp32/vasprintf_stubs.cpp b/esphome/components/esp32/vasprintf_stubs.cpp new file mode 100644 index 0000000000..308a58ebda --- /dev/null +++ b/esphome/components/esp32/vasprintf_stubs.cpp @@ -0,0 +1,53 @@ +/* + * Linker wrap stub for vasprintf() on variants whose ROM exports a + * full-format vsnprintf() but no vasprintf() (ESP32-C6, newlib only). + * + * On those chips every snprintf/vsnprintf call in the image resolves to + * the ROM, so the newlib printf engine (_svfprintf_r, _dtoa_r and their + * helpers, ~20 KB) is not linked at all until something references a + * printf-family function the ROM lacks. esp_http_client does exactly that + * through vasprintf() in its header and auth helpers, so adding + * http_request to a build costs the whole engine on top of the HTTP and + * TLS code itself. + * + * This stub reimplements vasprintf() on top of the ROM vsnprintf(), which + * keeps the engine out of the image. It is only compiled in when codegen + * defines USE_ESP32_VASPRINTF_STUB, which is gated on the variant's ROM + * linker script and on the same newlib condition as printf_stubs.cpp. + */ + +#include "esphome/core/defines.h" + +#if defined(USE_ESP_IDF) && defined(USE_ESP32_VASPRINTF_STUB) + +#include +#include +#include + +namespace esphome::esp32 {} + +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +int __wrap_vasprintf(char **strp, const char *fmt, va_list ap) { + va_list ap_copy; + va_copy(ap_copy, ap); + int len = vsnprintf(nullptr, 0, fmt, ap_copy); + va_end(ap_copy); + if (len < 0) { + return len; + } + // vasprintf's contract is a malloc'd buffer the caller releases with free() + char *buf = static_cast(malloc(static_cast(len) + 1)); // NOLINT(cppcoreguidelines-no-malloc) + if (buf == nullptr) { + return -1; + } + vsnprintf(buf, static_cast(len) + 1, fmt, ap); + *strp = buf; + return len; +} + +} // extern "C" +// NOLINTEND(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) + +#endif // USE_ESP_IDF && USE_ESP32_VASPRINTF_STUB diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index ab69f5d9ae..a71151f3ff 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -13,6 +13,8 @@ from esphome.components.esp32 import ( get_esp32_variant, include_builtin_idf_component, only_on_variant, + require_mbedtls_tls_extras, + require_mbedtls_tls_server, require_vfs_select, ) from esphome.components.mdns import MDNSComponent, enable_mdns_storage @@ -109,6 +111,14 @@ def set_sdkconfig_options(config: ConfigType) -> None: add_idf_sdkconfig_option("CONFIG_OPENTHREAD_ENABLED", True) + # OpenThread's DTLS commissioner is a TLS server, and its crypto platform + # uses AES-CCM and deterministic ECDSA directly. Keep the esp32 component + # from trimming them out of mbedTLS. + require_mbedtls_tls_server() + require_mbedtls_tls_extras( + ("CONFIG_MBEDTLS_CCM_C", "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC") + ) + if not config.get(CONF_TLV): if pan_id := config.get(CONF_PAN_ID): add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_PANID", pan_id) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 1691dcc293..58803a8cdf 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -12,6 +12,7 @@ from esphome.components.esp32 import ( get_esp32_variant, only_on_variant, request_wifi, + require_mbedtls_tls_extras, ) from esphome.components.network import ( add_use_address, @@ -658,6 +659,12 @@ async def to_code(config): # Disable Enterprise WiFi support if no EAP is configured if CORE.is_esp32: add_idf_sdkconfig_option("CONFIG_ESP_WIFI_ENTERPRISE_SUPPORT", has_eap) + if has_eap: + # wpa_supplicant's EAP client negotiates with whatever the RADIUS + # server offers, and a failed handshake leaves the device off the + # network, so keep every mbedTLS client feature the esp32 platform + # would otherwise trim. + require_mbedtls_tls_extras() # Only define USE_WIFI_MANUAL_IP if any AP uses manual IP if has_manual_ip: diff --git a/esphome/core/defines.h b/esphome/core/defines.h index c3b16d833a..9144e65576 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -298,6 +298,7 @@ // ESP32-specific feature flags #ifdef USE_ESP32 #define USE_ESP32_CRASH_HANDLER +#define USE_ESP32_VASPRINTF_STUB #define USE_ESP32_INTERNAL_GPIO #define USE_MQTT_IDF_ENQUEUE #define USE_ESPHOME_TASK_LOG_BUFFER diff --git a/tests/component_tests/esp32/config/mbedtls_tls_default.yaml b/tests/component_tests/esp32/config/mbedtls_tls_default.yaml new file mode 100644 index 0000000000..b29e5de2bd --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_default.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml b/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml new file mode 100644 index 0000000000..62ca893d2c --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_openthread.yaml @@ -0,0 +1,19 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf + +network: + enable_ipv6: true + +openthread: + channel: 13 + network_name: OpenThread-8f28 + network_key: 0xdfd34f0f05cad978ec4e32b0413038ff + pan_id: 0x8f28 + ext_pan_id: 0xd63e8e3e495ebbc3 + pskc: 0xc23a76e98f1a6483639b1ac1271e2e27 + mesh_local_prefix: fd53:145f:ed22:ad81::/64 diff --git a/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml b/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml new file mode 100644 index 0000000000..e675848391 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_opt_out.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + advanced: + disable_mbedtls_tls_server: false + disable_mbedtls_tls_extras: false + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml b/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml new file mode 100644 index 0000000000..44ff047a48 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_user_sdkconfig.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT: y + CONFIG_MBEDTLS_CCM_C: y + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml b/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml new file mode 100644 index 0000000000..6c78e06265 --- /dev/null +++ b/tests/component_tests/esp32/config/mbedtls_tls_wifi_eap.yaml @@ -0,0 +1,17 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + eap: + identity: "user@example.org" + username: "user" + password: "secret" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml b/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml new file mode 100644 index 0000000000..8fa28e7c0f --- /dev/null +++ b/tests/component_tests/esp32/config/vasprintf_stub_c6.yaml @@ -0,0 +1,7 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf diff --git a/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml b/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml new file mode 100644 index 0000000000..075c3913b5 --- /dev/null +++ b/tests/component_tests/esp32/config/vasprintf_stub_c6_full_printf.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf + advanced: + enable_full_printf: true diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 759020c732..2dd2a50c83 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -11,9 +11,12 @@ import pytest from esphome.components.esp32 import ( KEY_FATFS_REQUIRED, + KEY_MBEDTLS_TLS_EXTRAS_REQUIRED, + KEY_MBEDTLS_TLS_SERVER_REQUIRED, KEY_VFS_DIR_REQUIRED, KEY_VFS_SELECT_REQUIRED, KEY_VFS_TERMIOS_REQUIRED, + MBEDTLS_TLS_EXTRA_OPTIONS, VARIANT_ESP32, VARIANTS, NetworkSdkconfigData, @@ -1339,3 +1342,99 @@ def test_esp32_s31_gpio_validation( with caplog.at_level("WARNING"): validate_supports(pin) assert "GPIO36 is a strapping PIN" in caplog.text + + +_TLS_SERVER_OPTIONS = ( + "CONFIG_MBEDTLS_TLS_CLIENT_ONLY", + "CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT", +) + + +@pytest.mark.parametrize( + ("config_file", "server", "extras"), + [ + pytest.param("mbedtls_tls_default.yaml", (True, False), False, id="default"), + pytest.param("mbedtls_tls_opt_out.yaml", (None, None), None, id="opt_out"), + pytest.param("mbedtls_tls_wifi_eap.yaml", (True, False), None, id="wifi_eap"), + ], +) +def test_mbedtls_tls_trim_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + server: tuple[bool | None, bool | None], + extras: bool | None, +) -> None: + """Client-only TLS and the unused-feature trims apply unless opted out or required.""" + generate_main(component_config_path(config_file)) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _TLS_SERVER_OPTIONS) == server + assert {sdkconfig.get(name) for name in MBEDTLS_TLS_EXTRA_OPTIONS} == {extras} + + +_OPENTHREAD_EXTRAS = {"CONFIG_MBEDTLS_CCM_C", "CONFIG_MBEDTLS_ECDSA_DETERMINISTIC"} + + +def test_mbedtls_tls_openthread_keeps_only_what_it_uses( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The OpenThread config keeps the DTLS server, CCM and deterministic ECDSA; the rest is trimmed.""" + generate_main(component_config_path("mbedtls_tls_openthread.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _TLS_SERVER_OPTIONS) == (None, None) + for name in MBEDTLS_TLS_EXTRA_OPTIONS: + assert sdkconfig.get(name) is (None if name in _OPENTHREAD_EXTRAS else False) + + +def test_mbedtls_tls_user_sdkconfig_wins( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A user-set TLS role member leaves the whole choice alone; other user values are kept.""" + generate_main(component_config_path("mbedtls_tls_user_sdkconfig.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_MBEDTLS_TLS_CLIENT_ONLY") is None + role = sdkconfig["CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT"] + assert isinstance(role, RawSdkconfigValue) and role.value == "y" + ccm = sdkconfig["CONFIG_MBEDTLS_CCM_C"] + assert isinstance(ccm, RawSdkconfigValue) and ccm.value == "y" + assert { + sdkconfig.get(name) + for name in MBEDTLS_TLS_EXTRA_OPTIONS + if name != "CONFIG_MBEDTLS_CCM_C" + } == {False} + + +def test_mbedtls_tls_openthread_requires_server_and_extras( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The OpenThread hooks mark the DTLS server and CCM/deterministic ECDSA as required.""" + generate_main(component_config_path("mbedtls_tls_openthread.yaml")) + assert CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_SERVER_REQUIRED] is True + assert CORE.data[KEY_ESP32][KEY_MBEDTLS_TLS_EXTRAS_REQUIRED] == _OPENTHREAD_EXTRAS + + +_VASPRINTF_STUB_FLAGS = {"-Wl,--wrap=vasprintf", "-Wl,--undefined=__wrap_vasprintf"} + + +@pytest.mark.parametrize( + ("config_file", "expected"), + [ + pytest.param("vasprintf_stub_c6.yaml", True, id="c6"), + pytest.param("vasprintf_stub_c6_full_printf.yaml", False, id="c6_full_printf"), + pytest.param("exclusion_reincludes.yaml", False, id="esp32"), + ], +) +def test_vasprintf_stub_only_on_rom_vsnprintf_variants( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + expected: bool, +) -> None: + """The vasprintf wrap is emitted only where the ROM lacks vasprintf but has vsnprintf.""" + generate_main(component_config_path(config_file)) + assert (CORE.build_flags >= _VASPRINTF_STUB_FLAGS) is expected + defines = {define.name for define in CORE.defines} + assert ("USE_ESP32_VASPRINTF_STUB" in defines) is expected diff --git a/tests/components/esp32/test.esp32-idf.yaml b/tests/components/esp32/test.esp32-idf.yaml index 523e614e24..7f31fe59c6 100644 --- a/tests/components/esp32/test.esp32-idf.yaml +++ b/tests/components/esp32/test.esp32-idf.yaml @@ -17,6 +17,8 @@ esp32: disable_dev_null_vfs: true disable_mbedtls_peer_cert: true disable_mbedtls_pkcs7: true + disable_mbedtls_tls_server: true + disable_mbedtls_tls_extras: true disable_regi2c_in_iram: true disable_fatfs: true sram1_as_iram: true diff --git a/tests/components/http_request/test.esp32-c6-idf.yaml b/tests/components/http_request/test.esp32-c6-idf.yaml new file mode 100644 index 0000000000..ee2f5aa59b --- /dev/null +++ b/tests/components/http_request/test.esp32-c6-idf.yaml @@ -0,0 +1,4 @@ +substitutions: + verify_ssl: "true" + +<<: !include common.yaml From 37e9b2b7af3ae84358bf59c9270462d551ec7644 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 14:12:40 -0500 Subject: [PATCH 49/55] [remote_base] Make protocol methods non-virtual and size receiver lists from codegen (#19084) --- AGENTS.md | 3 + esphome/components/coolix/climate.py | 3 +- esphome/components/infrared/infrared.cpp | 5 - esphome/components/infrared/infrared.h | 3 +- esphome/components/ir_rf_proxy/infrared.py | 10 +- .../components/ir_rf_proxy/ir_rf_proxy.cpp | 4 - esphome/components/ir_rf_proxy/ir_rf_proxy.h | 3 +- .../components/ir_rf_proxy/radio_frequency.py | 10 +- esphome/components/midea/climate.py | 3 +- esphome/components/midea_ir/climate.py | 6 +- esphome/components/remote_base/__init__.py | 109 ++++++++++++++++-- .../remote_base/abbwelcome_protocol.h | 6 +- .../components/remote_base/aeha_protocol.h | 6 +- .../components/remote_base/beo4_protocol.h | 6 +- .../remote_base/brennenstuhl_protocol.h | 6 +- .../components/remote_base/byronsx_protocol.h | 6 +- .../remote_base/canalsat_protocol.h | 6 +- .../components/remote_base/coolix_protocol.h | 6 +- .../components/remote_base/dish_protocol.h | 6 +- .../components/remote_base/dooya_protocol.h | 6 +- .../components/remote_base/drayton_protocol.h | 6 +- .../components/remote_base/dyson_protocol.h | 6 +- .../components/remote_base/gobox_protocol.h | 6 +- .../components/remote_base/haier_protocol.h | 6 +- esphome/components/remote_base/jvc_protocol.h | 6 +- .../components/remote_base/keeloq_protocol.h | 6 +- esphome/components/remote_base/lg_protocol.h | 6 +- .../remote_base/magiquest_protocol.h | 6 +- .../components/remote_base/midea_protocol.h | 6 +- .../components/remote_base/mirage_protocol.h | 6 +- esphome/components/remote_base/nec_protocol.h | 6 +- .../components/remote_base/nexa_protocol.h | 6 +- .../remote_base/panasonic_protocol.h | 6 +- .../components/remote_base/pioneer_protocol.h | 6 +- .../components/remote_base/pronto_protocol.h | 6 +- esphome/components/remote_base/rc5_protocol.h | 6 +- esphome/components/remote_base/rc6_protocol.h | 6 +- .../remote_base/rc_switch_protocol.cpp | 38 +++--- .../remote_base/rc_switch_protocol.h | 35 +++++- .../components/remote_base/remote_base.cpp | 39 +++++-- esphome/components/remote_base/remote_base.h | 74 ++++++++---- .../components/remote_base/roomba_protocol.h | 6 +- .../remote_base/samsung36_protocol.h | 6 +- .../components/remote_base/samsung_protocol.h | 6 +- .../components/remote_base/sony_protocol.h | 6 +- .../remote_base/symphony_protocol.h | 6 +- .../remote_base/toshiba_ac_protocol.h | 6 +- .../components/remote_base/toto_protocol.h | 6 +- .../components/remote_receiver/__init__.py | 4 +- esphome/components/toshiba/climate.py | 3 +- esphome/core/defines.h | 37 ++++++ esphome/cpp_helpers.py | 38 ++++-- .../remote_receiver/__init__.py | 0 .../remote_receiver/config/receiver_bare.yaml | 9 ++ .../config/receiver_with_dumpers.yaml | 24 ++++ .../config/receiver_with_proxies.yaml | 22 ++++ .../remote_receiver/test_slot_counts.py | 91 +++++++++++++++ .../remote_receiver/bare-common.yaml | 6 + .../remote_receiver/test-bare.esp32-idf.yaml | 5 + tests/unit_tests/test_cpp_helpers.py | 25 ++++ 60 files changed, 606 insertions(+), 201 deletions(-) create mode 100644 tests/component_tests/remote_receiver/__init__.py create mode 100644 tests/component_tests/remote_receiver/config/receiver_bare.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml create mode 100644 tests/component_tests/remote_receiver/test_slot_counts.py create mode 100644 tests/components/remote_receiver/bare-common.yaml create mode 100644 tests/components/remote_receiver/test-bare.esp32-idf.yaml diff --git a/AGENTS.md b/AGENTS.md index 98bdd58ec5..8db3cd3d62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -629,6 +629,9 @@ file does, and it is the authority when they disagree. The most useful starting _request_listener_slot() cg.add(hub.register_listener(var)) ``` + When several instances each own a list declared at the same size (one per hub of a + `MULTI_CONF` component), pass the owning object as the key, `_request_listener_slot(str(hub))`; + the define is then the largest count any one key requested instead of the total. ```cpp #ifdef MY_COMPONENT_LISTENER_COUNT void register_listener(MyComponentListener *listener); diff --git a/esphome/components/coolix/climate.py b/esphome/components/coolix/climate.py index 3eb8dbe2f4..fcca8b89db 100644 --- a/esphome/components/coolix/climate.py +++ b/esphome/components/coolix/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base from esphome.types import ConfigType AUTO_LOAD = ["climate_ir"] @@ -12,4 +12,5 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(CoolixClimate) async def to_code(config: ConfigType) -> None: + remote_base.request_protocol("coolix") # used from C++ await climate_ir.new_climate_ir(config) diff --git a/esphome/components/infrared/infrared.cpp b/esphome/components/infrared/infrared.cpp index 5a909738c6..83039a5a9b 100644 --- a/esphome/components/infrared/infrared.cpp +++ b/esphome/components/infrared/infrared.cpp @@ -59,11 +59,6 @@ void Infrared::setup() { // Set up traits based on configuration this->traits_.set_supports_transmitter(this->has_transmitter()); this->traits_.set_supports_receiver(this->has_receiver()); - - // Register as listener for received IR data - if (this->receiver_ != nullptr) { - this->receiver_->register_listener(this); - } } void Infrared::dump_config() { diff --git a/esphome/components/infrared/infrared.h b/esphome/components/infrared/infrared.h index b6863e37ce..afbde57be2 100644 --- a/esphome/components/infrared/infrared.h +++ b/esphome/components/infrared/infrared.h @@ -119,7 +119,8 @@ class Infrared : public Component, public EntityBase, public remote_base::Remote void dump_config() override; float get_setup_priority() const override { return setup_priority::AFTER_CONNECTION; } - /// Set the remote receiver component + /// Set the remote receiver component; the listener registration happens from codegen, see + /// remote_base.attach_receiver void set_receiver(remote_base::RemoteReceiverBase *receiver) { this->receiver_ = receiver; } /// Set the remote transmitter component void set_transmitter(remote_base::RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } diff --git a/esphome/components/ir_rf_proxy/infrared.py b/esphome/components/ir_rf_proxy/infrared.py index 3218889721..288bd91673 100644 --- a/esphome/components/ir_rf_proxy/infrared.py +++ b/esphome/components/ir_rf_proxy/infrared.py @@ -3,7 +3,12 @@ from typing import Any import esphome.codegen as cg -from esphome.components import infrared, remote_receiver, remote_transmitter +from esphome.components import ( + infrared, + remote_base, + remote_receiver, + remote_transmitter, +) from esphome.components.const import CONF_RECEIVER_FREQUENCY import esphome.config_validation as cv from esphome.const import CONF_CARRIER_DUTY_PERCENT, CONF_FREQUENCY @@ -82,8 +87,7 @@ async def to_code(config: dict[str, Any]) -> None: # Link receiver if specified if CONF_REMOTE_RECEIVER_ID in config: - receiver = await cg.get_variable(config[CONF_REMOTE_RECEIVER_ID]) - cg.add(var.set_receiver(receiver)) + await remote_base.attach_receiver(var, config, CONF_REMOTE_RECEIVER_ID) # Set receiver demodulation frequency if specified (metadata only, no hardware effect) if CONF_RECEIVER_FREQUENCY in config: diff --git a/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp b/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp index c13c6198cb..ceb4c9a67c 100644 --- a/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp +++ b/esphome/components/ir_rf_proxy/ir_rf_proxy.cpp @@ -97,10 +97,6 @@ void RfProxy::setup() { // remote_transmitter/receiver always uses OOK (on-off keying) this->traits_.add_supported_modulation(radio_frequency::RadioFrequencyModulation::RADIO_FREQUENCY_MODULATION_OOK); - - if (this->receiver_ != nullptr) { - this->receiver_->register_listener(this); - } } void RfProxy::dump_config() { diff --git a/esphome/components/ir_rf_proxy/ir_rf_proxy.h b/esphome/components/ir_rf_proxy/ir_rf_proxy.h index 5fc683354b..1aa4394fe8 100644 --- a/esphome/components/ir_rf_proxy/ir_rf_proxy.h +++ b/esphome/components/ir_rf_proxy/ir_rf_proxy.h @@ -56,7 +56,8 @@ class RfProxy final : public radio_frequency::RadioFrequency { /// Set the remote transmitter component void set_transmitter(remote_base::RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } - /// Set the remote receiver component + /// Set the remote receiver component; the listener registration happens from codegen, see + /// remote_base.attach_receiver void set_receiver(remote_base::RemoteReceiverBase *receiver) { this->receiver_ = receiver; } /// Set the fixed carrier frequency in Hz (metadata: advertised via traits, does not tune hardware) diff --git a/esphome/components/ir_rf_proxy/radio_frequency.py b/esphome/components/ir_rf_proxy/radio_frequency.py index a243909837..28b8fd5953 100644 --- a/esphome/components/ir_rf_proxy/radio_frequency.py +++ b/esphome/components/ir_rf_proxy/radio_frequency.py @@ -1,7 +1,12 @@ """Radio Frequency platform implementation using remote_base (remote_transmitter/receiver).""" import esphome.codegen as cg -from esphome.components import radio_frequency, remote_receiver, remote_transmitter +from esphome.components import ( + radio_frequency, + remote_base, + remote_receiver, + remote_transmitter, +) import esphome.config_validation as cv from esphome.const import CONF_CARRIER_DUTY_PERCENT, CONF_FREQUENCY import esphome.final_validate as fv @@ -66,5 +71,4 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_transmitter(transmitter)) if CONF_REMOTE_RECEIVER_ID in config: - receiver = await cg.get_variable(config[CONF_REMOTE_RECEIVER_ID]) - cg.add(var.set_receiver(receiver)) + await remote_base.attach_receiver(var, config, CONF_REMOTE_RECEIVER_ID) diff --git a/esphome/components/midea/climate.py b/esphome/components/midea/climate.py index 0e03bca233..07ad02d3af 100644 --- a/esphome/components/midea/climate.py +++ b/esphome/components/midea/climate.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import climate, remote_transmitter, sensor, uart +from esphome.components import climate, remote_base, remote_transmitter, sensor, uart from esphome.components.climate import ClimateMode, ClimatePreset, ClimateSwingMode from esphome.components.remote_base import CONF_TRANSMITTER_ID import esphome.config_validation as cv @@ -280,6 +280,7 @@ async def to_code(config): cg.add(var.set_response_timeout(config[CONF_TIMEOUT].total_milliseconds)) cg.add(var.set_request_attempts(config[CONF_NUM_ATTEMPTS])) if CONF_TRANSMITTER_ID in config: + remote_base.request_protocol("midea") # ir_transmitter.h uses it from C++ cg.add_define("USE_REMOTE_TRANSMITTER") transmitter_ = await cg.get_variable(config[CONF_TRANSMITTER_ID]) cg.add(var.set_transmitter(transmitter_)) diff --git a/esphome/components/midea_ir/climate.py b/esphome/components/midea_ir/climate.py index 84bfeab0d4..e1b2b56ada 100644 --- a/esphome/components/midea_ir/climate.py +++ b/esphome/components/midea_ir/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base import esphome.config_validation as cv from esphome.const import CONF_USE_FAHRENHEIT from esphome.types import ConfigType @@ -19,5 +19,9 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(MideaIR).extend( async def to_code(config: ConfigType) -> None: + # midea_ir uses MideaProtocol from C++ and auto-loads coolix, whose coolix.cpp uses + # CoolixProtocol even when no coolix climate is configured + remote_base.request_protocol("midea") + remote_base.request_protocol("coolix") var = await climate_ir.new_climate_ir(config) cg.add(var.set_fahrenheit(config[CONF_USE_FAHRENHEIT])) diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py index 19b8549f75..27b6eb9fc8 100644 --- a/esphome/components/remote_base/__init__.py +++ b/esphome/components/remote_base/__init__.py @@ -1,6 +1,11 @@ +from collections.abc import Callable +from pathlib import Path +from typing import Any + from esphome import automation import esphome.codegen as cg from esphome.components import binary_sensor +from esphome.config_helpers import filter_source_files_from_defines import esphome.config_validation as cv from esphome.const import ( CONF_ADDRESS, @@ -40,11 +45,14 @@ from esphome.const import ( CONF_ZERO, ) from esphome.core import ID, coroutine +from esphome.cpp_generator import MockObj from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType from esphome.util import Registry, SimpleRegistry AUTO_LOAD = ["binary_sensor"] + CONF_RECEIVER_ID = "receiver_id" CONF_TRANSMITTER_ID = "transmitter_id" CONF_FIRST = "first" @@ -90,9 +98,42 @@ REMOTE_TRANSMITTABLE_SCHEMA = cv.Schema( ) -async def register_listener(var, config): +# Listener and dumper lists are StaticVectors sized from these counts, so every registration +# must go through add_listener / add_dumper. Every receiver's list gets the same capacity, so +# the slots are keyed by receiver and the define is the largest count any one receiver needs. +LISTENER_COUNT_DEFINE = "REMOTE_BASE_LISTENER_COUNT" +DUMPER_COUNT_DEFINE = "REMOTE_BASE_DUMPER_COUNT" + + +_request_listener_slot = cg.slot_counter(LISTENER_COUNT_DEFINE) +_request_dumper_slot = cg.slot_counter(DUMPER_COUNT_DEFINE) + + +def add_listener(receiver: MockObj, listener: MockObj) -> None: + _request_listener_slot(str(receiver)) + cg.add(receiver.register_listener(listener)) + + +def add_dumper(receiver: MockObj, dumper: MockObj) -> None: + _request_dumper_slot(str(receiver)) + cg.add(receiver.register_dumper(dumper)) + + +async def register_listener(var: MockObj, config: ConfigType) -> None: receiver = await cg.get_variable(config[CONF_RECEIVER_ID]) - cg.add(receiver.register_listener(var)) + add_listener(receiver, var) + + +async def attach_receiver( + var: MockObj, config: ConfigType, key: str = CONF_RECEIVER_ID +) -> None: + """Link the configured receiver to an entity and register the entity as its listener. + + The C++ set_receiver() no longer registers the listener; the slot for it is counted here. + """ + receiver = await cg.get_variable(config[key]) + cg.add(var.set_receiver(receiver)) + add_listener(receiver, var) async def register_transmittable(var, config): @@ -100,8 +141,53 @@ async def register_transmittable(var, config): cg.add(var.set_transmitter(transmitter_)) -def register_binary_sensor(name, type, schema): - return BINARY_SENSOR_REGISTRY.register(name, type, schema) +# Registry names that share a protocol source file +def _protocol_stem(name: str) -> str: + if name.startswith("rc_switch"): + return "rc_switch" + if name == "canalsatld": + return "canalsat" + return name + + +def protocol_define(name: str) -> str: + return f"USE_REMOTE_PROTOCOL_{_protocol_stem(name).upper()}" + + +_PROTOCOL_STEMS = sorted( + path.name.removesuffix("_protocol.cpp") + for path in Path(__file__).parent.glob("*_protocol.cpp") +) + + +def request_protocol(name: str) -> None: + """Keep a protocol's source file in the build; components using it from C++ must call this.""" + if _protocol_stem(name) not in _PROTOCOL_STEMS: + raise ValueError( + f"Unknown remote protocol {name!r}; expected one of {', '.join(_PROTOCOL_STEMS)}" + ) + cg.add_define(protocol_define(name)) + + +# Only the protocol sources a configuration uses are compiled +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {f"{stem}_protocol.cpp": protocol_define(stem) for stem in _PROTOCOL_STEMS} +) + + +def register_binary_sensor( + name: str, type: MockObj, schema: cv.Schema | dict +) -> Callable[[Callable[[MockObj, ConfigType], Any]], Callable]: + registerer = BINARY_SENSOR_REGISTRY.register(name, type, schema) + + def decorator(func: Callable[[MockObj, ConfigType], Any]) -> Callable: + async def new_func(var: MockObj, config: ConfigType) -> None: + request_protocol(name) + await coroutine(func)(var, config) + + return registerer(new_func) + + return decorator def register_trigger(name, type, data_type): @@ -114,6 +200,7 @@ def register_trigger(name, type, data_type): def decorator(func): async def new_func(config): + request_protocol(name) var = cg.new_Pvariable(config[CONF_TRIGGER_ID]) await coroutine(func)(var, config) await automation.build_automation(var, [(data_type, "x")], config) @@ -131,6 +218,7 @@ def register_dumper(name, type, schema=None): def decorator(func): async def new_func(config, dumper_id): + request_protocol(name) var = cg.new_Pvariable(dumper_id) await coroutine(func)(var, config) return var @@ -171,6 +259,7 @@ def register_action(name, type_, schema): def decorator(func): async def new_func(config, action_id, template_arg, args): + request_protocol(name) var = cg.new_Pvariable(action_id, template_arg) await register_transmittable(var, config) if CONF_REPEAT in config: @@ -213,7 +302,13 @@ DUMPER_REGISTRY = Registry() def validate_dumpers(value): if isinstance(value, str) and value.lower() == "all": return validate_dumpers(list(DUMPER_REGISTRY.keys())) - return cv.validate_registry("dumper", DUMPER_REGISTRY)(value) + entries = cv.validate_registry("dumper", DUMPER_REGISTRY)(value) + # a dumper listed twice would register twice; the receiver holds one secondary dumper + return list( + { + next(k for k in entry if k in DUMPER_REGISTRY): entry for entry in entries + }.values() + ) def validate_triggers(base_schema): @@ -1439,7 +1534,7 @@ def validate_rc_switch_raw_code(value): def build_rc_switch_protocol(config): if isinstance(config, int): - return rc_switch_protocols[config] + return rc_switch_protocol(config) pl = config[CONF_PULSE_LENGTH] return RCSwitchBase( config[CONF_SYNC][0] * pl, @@ -1526,7 +1621,7 @@ RC_SWITCH_TRANSMITTER = cv.Schema( } ) -rc_switch_protocols = ns.RC_SWITCH_PROTOCOLS +rc_switch_protocol = ns.rc_switch_protocol RCSwitchData = ns.struct("RCSwitchData") RCSwitchBase = ns.class_("RCSwitchBase") RCSwitchTrigger = ns.class_("RCSwitchTrigger", RemoteReceiverTrigger) diff --git a/esphome/components/remote_base/abbwelcome_protocol.h b/esphome/components/remote_base/abbwelcome_protocol.h index 7ff32923be..a309c124ee 100644 --- a/esphome/components/remote_base/abbwelcome_protocol.h +++ b/esphome/components/remote_base/abbwelcome_protocol.h @@ -191,9 +191,9 @@ class ABBWelcomeData { class ABBWelcomeProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ABBWelcomeData &src) override; - optional decode(RemoteReceiveData src) override; - void dump(const ABBWelcomeData &data) override; + void encode(RemoteTransmitData *dst, const ABBWelcomeData &src); + optional decode(RemoteReceiveData src); + void dump(const ABBWelcomeData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t data) const; diff --git a/esphome/components/remote_base/aeha_protocol.h b/esphome/components/remote_base/aeha_protocol.h index 3f4e98bd43..98a5501155 100644 --- a/esphome/components/remote_base/aeha_protocol.h +++ b/esphome/components/remote_base/aeha_protocol.h @@ -15,9 +15,9 @@ struct AEHAData { class AEHAProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const AEHAData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const AEHAData &data) override; + void encode(RemoteTransmitData *dst, const AEHAData &data); + optional decode(RemoteReceiveData src); + void dump(const AEHAData &data); private: std::string format_data_(const std::vector &data); diff --git a/esphome/components/remote_base/beo4_protocol.h b/esphome/components/remote_base/beo4_protocol.h index 30b99dbeb7..ed9d6aa671 100644 --- a/esphome/components/remote_base/beo4_protocol.h +++ b/esphome/components/remote_base/beo4_protocol.h @@ -16,9 +16,9 @@ struct Beo4Data { class Beo4Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const Beo4Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const Beo4Data &data) override; + void encode(RemoteTransmitData *dst, const Beo4Data &data); + optional decode(RemoteReceiveData src); + void dump(const Beo4Data &data); }; DECLARE_REMOTE_PROTOCOL(Beo4) diff --git a/esphome/components/remote_base/brennenstuhl_protocol.h b/esphome/components/remote_base/brennenstuhl_protocol.h index 1d5b621714..bfea463b7d 100644 --- a/esphome/components/remote_base/brennenstuhl_protocol.h +++ b/esphome/components/remote_base/brennenstuhl_protocol.h @@ -13,9 +13,9 @@ struct BrennenstuhlData { class BrennenstuhlProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const BrennenstuhlData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const BrennenstuhlData &data) override; + void encode(RemoteTransmitData *dst, const BrennenstuhlData &data); + optional decode(RemoteReceiveData src); + void dump(const BrennenstuhlData &data); }; DECLARE_REMOTE_PROTOCOL(Brennenstuhl) diff --git a/esphome/components/remote_base/byronsx_protocol.h b/esphome/components/remote_base/byronsx_protocol.h index 674fa99ea1..c71390c267 100644 --- a/esphome/components/remote_base/byronsx_protocol.h +++ b/esphome/components/remote_base/byronsx_protocol.h @@ -21,9 +21,9 @@ struct ByronSXData { class ByronSXProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ByronSXData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ByronSXData &data) override; + void encode(RemoteTransmitData *dst, const ByronSXData &data); + optional decode(RemoteReceiveData src); + void dump(const ByronSXData &data); }; DECLARE_REMOTE_PROTOCOL(ByronSX) diff --git a/esphome/components/remote_base/canalsat_protocol.h b/esphome/components/remote_base/canalsat_protocol.h index 5ba9115ea8..09bead18b3 100644 --- a/esphome/components/remote_base/canalsat_protocol.h +++ b/esphome/components/remote_base/canalsat_protocol.h @@ -19,9 +19,9 @@ struct CanalSatLDData : public CanalSatData {}; class CanalSatBaseProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const CanalSatData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const CanalSatData &data) override; + void encode(RemoteTransmitData *dst, const CanalSatData &data); + optional decode(RemoteReceiveData src); + void dump(const CanalSatData &data); protected: uint16_t frequency_; diff --git a/esphome/components/remote_base/coolix_protocol.h b/esphome/components/remote_base/coolix_protocol.h index d9441e8417..29a306ce29 100644 --- a/esphome/components/remote_base/coolix_protocol.h +++ b/esphome/components/remote_base/coolix_protocol.h @@ -21,9 +21,9 @@ struct CoolixData { class CoolixProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const CoolixData &data) override; - optional decode(RemoteReceiveData data) override; - void dump(const CoolixData &data) override; + void encode(RemoteTransmitData *dst, const CoolixData &data); + optional decode(RemoteReceiveData data); + void dump(const CoolixData &data); }; DECLARE_REMOTE_PROTOCOL(Coolix) diff --git a/esphome/components/remote_base/dish_protocol.h b/esphome/components/remote_base/dish_protocol.h index c89f4e78e1..f319b55f43 100644 --- a/esphome/components/remote_base/dish_protocol.h +++ b/esphome/components/remote_base/dish_protocol.h @@ -13,9 +13,9 @@ struct DishData { class DishProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DishData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DishData &data) override; + void encode(RemoteTransmitData *dst, const DishData &data); + optional decode(RemoteReceiveData src); + void dump(const DishData &data); }; DECLARE_REMOTE_PROTOCOL(Dish) diff --git a/esphome/components/remote_base/dooya_protocol.h b/esphome/components/remote_base/dooya_protocol.h index 148c7c17bc..954c3cf1d3 100644 --- a/esphome/components/remote_base/dooya_protocol.h +++ b/esphome/components/remote_base/dooya_protocol.h @@ -20,9 +20,9 @@ struct DooyaData { class DooyaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DooyaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DooyaData &data) override; + void encode(RemoteTransmitData *dst, const DooyaData &data); + optional decode(RemoteReceiveData src); + void dump(const DooyaData &data); }; DECLARE_REMOTE_PROTOCOL(Dooya) diff --git a/esphome/components/remote_base/drayton_protocol.h b/esphome/components/remote_base/drayton_protocol.h index 693a1bbe85..4e879f0f75 100644 --- a/esphome/components/remote_base/drayton_protocol.h +++ b/esphome/components/remote_base/drayton_protocol.h @@ -19,9 +19,9 @@ struct DraytonData { class DraytonProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DraytonData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DraytonData &data) override; + void encode(RemoteTransmitData *dst, const DraytonData &data); + optional decode(RemoteReceiveData src); + void dump(const DraytonData &data); }; DECLARE_REMOTE_PROTOCOL(Drayton) diff --git a/esphome/components/remote_base/dyson_protocol.h b/esphome/components/remote_base/dyson_protocol.h index 3473a489b2..663e50fb4b 100644 --- a/esphome/components/remote_base/dyson_protocol.h +++ b/esphome/components/remote_base/dyson_protocol.h @@ -21,9 +21,9 @@ struct DysonData { class DysonProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const DysonData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const DysonData &data) override; + void encode(RemoteTransmitData *dst, const DysonData &data); + optional decode(RemoteReceiveData src); + void dump(const DysonData &data); }; DECLARE_REMOTE_PROTOCOL(Dyson) diff --git a/esphome/components/remote_base/gobox_protocol.h b/esphome/components/remote_base/gobox_protocol.h index f6b278771e..0c8797af70 100644 --- a/esphome/components/remote_base/gobox_protocol.h +++ b/esphome/components/remote_base/gobox_protocol.h @@ -31,9 +31,9 @@ class GoboxProtocol : public RemoteProtocol { void dump_timings_(const RawTimings &timings) const; public: - void encode(RemoteTransmitData *dst, const GoboxData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const GoboxData &data) override; + void encode(RemoteTransmitData *dst, const GoboxData &data); + optional decode(RemoteReceiveData src); + void dump(const GoboxData &data); }; DECLARE_REMOTE_PROTOCOL(Gobox) diff --git a/esphome/components/remote_base/haier_protocol.h b/esphome/components/remote_base/haier_protocol.h index 9c45ba1a63..e1fd60411f 100644 --- a/esphome/components/remote_base/haier_protocol.h +++ b/esphome/components/remote_base/haier_protocol.h @@ -13,9 +13,9 @@ struct HaierData { class HaierProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const HaierData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const HaierData &data) override; + void encode(RemoteTransmitData *dst, const HaierData &data); + optional decode(RemoteReceiveData src); + void dump(const HaierData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t item); diff --git a/esphome/components/remote_base/jvc_protocol.h b/esphome/components/remote_base/jvc_protocol.h index f6e2548dea..5911664fc3 100644 --- a/esphome/components/remote_base/jvc_protocol.h +++ b/esphome/components/remote_base/jvc_protocol.h @@ -14,9 +14,9 @@ struct JVCData { class JVCProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const JVCData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const JVCData &data) override; + void encode(RemoteTransmitData *dst, const JVCData &data); + optional decode(RemoteReceiveData src); + void dump(const JVCData &data); }; DECLARE_REMOTE_PROTOCOL(JVC) diff --git a/esphome/components/remote_base/keeloq_protocol.h b/esphome/components/remote_base/keeloq_protocol.h index 432313b87b..335fbd164b 100644 --- a/esphome/components/remote_base/keeloq_protocol.h +++ b/esphome/components/remote_base/keeloq_protocol.h @@ -24,9 +24,9 @@ struct KeeloqData { class KeeloqProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const KeeloqData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const KeeloqData &data) override; + void encode(RemoteTransmitData *dst, const KeeloqData &data); + optional decode(RemoteReceiveData src); + void dump(const KeeloqData &data); }; DECLARE_REMOTE_PROTOCOL(Keeloq) diff --git a/esphome/components/remote_base/lg_protocol.h b/esphome/components/remote_base/lg_protocol.h index 9715974995..91dfbadb0c 100644 --- a/esphome/components/remote_base/lg_protocol.h +++ b/esphome/components/remote_base/lg_protocol.h @@ -16,9 +16,9 @@ struct LGData { class LGProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const LGData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const LGData &data) override; + void encode(RemoteTransmitData *dst, const LGData &data); + optional decode(RemoteReceiveData src); + void dump(const LGData &data); }; DECLARE_REMOTE_PROTOCOL(LG) diff --git a/esphome/components/remote_base/magiquest_protocol.h b/esphome/components/remote_base/magiquest_protocol.h index 18662ec759..f0d2410fe2 100644 --- a/esphome/components/remote_base/magiquest_protocol.h +++ b/esphome/components/remote_base/magiquest_protocol.h @@ -27,9 +27,9 @@ struct MagiQuestData { class MagiQuestProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MagiQuestData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const MagiQuestData &data) override; + void encode(RemoteTransmitData *dst, const MagiQuestData &data); + optional decode(RemoteReceiveData src); + void dump(const MagiQuestData &data); }; DECLARE_REMOTE_PROTOCOL(MagiQuest) diff --git a/esphome/components/remote_base/midea_protocol.h b/esphome/components/remote_base/midea_protocol.h index 47bad6826f..85bbef1cb1 100644 --- a/esphome/components/remote_base/midea_protocol.h +++ b/esphome/components/remote_base/midea_protocol.h @@ -67,9 +67,9 @@ class MideaData { class MideaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MideaData &src) override; - optional decode(RemoteReceiveData src) override; - void dump(const MideaData &data) override; + void encode(RemoteTransmitData *dst, const MideaData &src); + optional decode(RemoteReceiveData src); + void dump(const MideaData &data); }; DECLARE_REMOTE_PROTOCOL(Midea) diff --git a/esphome/components/remote_base/mirage_protocol.h b/esphome/components/remote_base/mirage_protocol.h index c967e72f13..a37fb93f4f 100644 --- a/esphome/components/remote_base/mirage_protocol.h +++ b/esphome/components/remote_base/mirage_protocol.h @@ -13,9 +13,9 @@ struct MirageData { class MirageProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const MirageData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const MirageData &data) override; + void encode(RemoteTransmitData *dst, const MirageData &data); + optional decode(RemoteReceiveData src); + void dump(const MirageData &data); protected: void encode_byte_(RemoteTransmitData *dst, uint8_t item); diff --git a/esphome/components/remote_base/nec_protocol.h b/esphome/components/remote_base/nec_protocol.h index 7b310e8ba5..1337f7a8b3 100644 --- a/esphome/components/remote_base/nec_protocol.h +++ b/esphome/components/remote_base/nec_protocol.h @@ -14,9 +14,9 @@ struct NECData { class NECProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const NECData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const NECData &data) override; + void encode(RemoteTransmitData *dst, const NECData &data); + optional decode(RemoteReceiveData src); + void dump(const NECData &data); }; DECLARE_REMOTE_PROTOCOL(NEC) diff --git a/esphome/components/remote_base/nexa_protocol.h b/esphome/components/remote_base/nexa_protocol.h index ebcd2a2c11..ebf85387b0 100644 --- a/esphome/components/remote_base/nexa_protocol.h +++ b/esphome/components/remote_base/nexa_protocol.h @@ -24,9 +24,9 @@ class NexaProtocol : public RemoteProtocol { void zero(RemoteTransmitData *dst) const; void sync(RemoteTransmitData *dst) const; - void encode(RemoteTransmitData *dst, const NexaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const NexaData &data) override; + void encode(RemoteTransmitData *dst, const NexaData &data); + optional decode(RemoteReceiveData src); + void dump(const NexaData &data); }; DECLARE_REMOTE_PROTOCOL(Nexa) diff --git a/esphome/components/remote_base/panasonic_protocol.h b/esphome/components/remote_base/panasonic_protocol.h index d13c0f2798..84df3c08b7 100644 --- a/esphome/components/remote_base/panasonic_protocol.h +++ b/esphome/components/remote_base/panasonic_protocol.h @@ -16,9 +16,9 @@ struct PanasonicData { class PanasonicProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const PanasonicData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const PanasonicData &data) override; + void encode(RemoteTransmitData *dst, const PanasonicData &data); + optional decode(RemoteReceiveData src); + void dump(const PanasonicData &data); }; DECLARE_REMOTE_PROTOCOL(Panasonic) diff --git a/esphome/components/remote_base/pioneer_protocol.h b/esphome/components/remote_base/pioneer_protocol.h index 514ab67501..d02bd3451f 100644 --- a/esphome/components/remote_base/pioneer_protocol.h +++ b/esphome/components/remote_base/pioneer_protocol.h @@ -13,9 +13,9 @@ struct PioneerData { class PioneerProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const PioneerData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const PioneerData &data) override; + void encode(RemoteTransmitData *dst, const PioneerData &data); + optional decode(RemoteReceiveData src); + void dump(const PioneerData &data); }; DECLARE_REMOTE_PROTOCOL(Pioneer) diff --git a/esphome/components/remote_base/pronto_protocol.h b/esphome/components/remote_base/pronto_protocol.h index f4f6b2144d..bfd04c5cd9 100644 --- a/esphome/components/remote_base/pronto_protocol.h +++ b/esphome/components/remote_base/pronto_protocol.h @@ -30,9 +30,9 @@ class ProntoProtocol : public RemoteProtocol { std::string compensate_and_dump_sequence_(const RawTimings &data, uint16_t timebase); public: - void encode(RemoteTransmitData *dst, const ProntoData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ProntoData &data) override; + void encode(RemoteTransmitData *dst, const ProntoData &data); + optional decode(RemoteReceiveData src); + void dump(const ProntoData &data); }; DECLARE_REMOTE_PROTOCOL(Pronto) diff --git a/esphome/components/remote_base/rc5_protocol.h b/esphome/components/remote_base/rc5_protocol.h index dbb89e41c6..f6f0f33c6e 100644 --- a/esphome/components/remote_base/rc5_protocol.h +++ b/esphome/components/remote_base/rc5_protocol.h @@ -14,9 +14,9 @@ struct RC5Data { class RC5Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RC5Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RC5Data &data) override; + void encode(RemoteTransmitData *dst, const RC5Data &data); + optional decode(RemoteReceiveData src); + void dump(const RC5Data &data); }; DECLARE_REMOTE_PROTOCOL(RC5) diff --git a/esphome/components/remote_base/rc6_protocol.h b/esphome/components/remote_base/rc6_protocol.h index fda9d98ecb..c4a2e8529b 100644 --- a/esphome/components/remote_base/rc6_protocol.h +++ b/esphome/components/remote_base/rc6_protocol.h @@ -15,9 +15,9 @@ struct RC6Data { class RC6Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RC6Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RC6Data &data) override; + void encode(RemoteTransmitData *dst, const RC6Data &data); + optional decode(RemoteReceiveData src); + void dump(const RC6Data &data); }; DECLARE_REMOTE_PROTOCOL(RC6) diff --git a/esphome/components/remote_base/rc_switch_protocol.cpp b/esphome/components/remote_base/rc_switch_protocol.cpp index 612558ca1c..de16c55cb0 100644 --- a/esphome/components/remote_base/rc_switch_protocol.cpp +++ b/esphome/components/remote_base/rc_switch_protocol.cpp @@ -1,29 +1,21 @@ #include "rc_switch_protocol.h" + +#include +#include "esphome/core/hal.h" #include "esphome/core/log.h" namespace esphome::remote_base { static const char *const TAG = "remote.rc_switch"; -const RCSwitchBase RC_SWITCH_PROTOCOLS[9] = {RCSwitchBase(0, 0, 0, 0, 0, 0, false), - RCSwitchBase(350, 10850, 350, 1050, 1050, 350, false), - RCSwitchBase(650, 6500, 650, 1300, 1300, 650, false), - RCSwitchBase(3000, 7100, 400, 1100, 900, 600, false), - RCSwitchBase(380, 2280, 380, 1140, 1140, 380, false), - RCSwitchBase(3000, 7000, 500, 1000, 1000, 500, false), - RCSwitchBase(10350, 450, 450, 900, 900, 450, true), - RCSwitchBase(300, 9300, 150, 900, 900, 150, false), - RCSwitchBase(250, 2500, 250, 1250, 250, 250, false)}; - -RCSwitchBase::RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, - uint32_t one_high, uint32_t one_low, bool inverted) - : sync_high_(sync_high), - sync_low_(sync_low), - zero_high_(zero_high), - zero_low_(zero_low), - one_high_(one_high), - one_low_(one_low), - inverted_(inverted) {} +RCSwitchBase rc_switch_protocol(uint8_t index) { + RCSwitchBase protocol; + // entry 0 is the all-zero protocol, so an out of range index from a lambda transmits nothing + if (index >= std::size(RC_SWITCH_PROTOCOLS)) + index = 0; + progmem_memcpy(&protocol, &RC_SWITCH_PROTOCOLS[index], sizeof(protocol)); + return protocol; +} void RCSwitchBase::one(RemoteTransmitData *dst) const { if (!this->inverted_) { @@ -133,11 +125,11 @@ bool RCSwitchBase::decode(RemoteReceiveData &src, uint64_t *out_data, uint8_t *o optional RCSwitchBase::decode(RemoteReceiveData &src) const { RCSwitchData out; uint8_t out_nbits; - for (uint8_t i = 1; i <= 8; i++) { + for (size_t i = 1; i < std::size(RC_SWITCH_PROTOCOLS); i++) { src.reset(); const RCSwitchBase *protocol = &RC_SWITCH_PROTOCOLS[i]; if (protocol->decode(src, &out.code, &out_nbits) && out_nbits >= 3) { - out.protocol = i; + out.protocol = static_cast(i); return out; } } @@ -246,7 +238,7 @@ bool RCSwitchRawReceiver::matches(RemoteReceiveData src) { return decoded_nbits == this->nbits_ && (decoded_code & this->mask_) == (this->code_ & this->mask_); } bool RCSwitchDumper::dump(RemoteReceiveData src) { - for (uint8_t i = 1; i <= 8; i++) { + for (size_t i = 1; i < std::size(RC_SWITCH_PROTOCOLS); i++) { src.reset(); uint64_t out_data; uint8_t out_nbits; @@ -257,7 +249,7 @@ bool RCSwitchDumper::dump(RemoteReceiveData src) { buffer[j] = (out_data & ((uint64_t) 1 << (out_nbits - j - 1))) ? '1' : '0'; buffer[out_nbits] = '\0'; - ESP_LOGI(TAG, "Received RCSwitch Raw: protocol=%u data='%s'", i, buffer); + ESP_LOGI(TAG, "Received RCSwitch Raw: protocol=%u data='%s'", static_cast(i), buffer); // only send first decoded protocol return true; diff --git a/esphome/components/remote_base/rc_switch_protocol.h b/esphome/components/remote_base/rc_switch_protocol.h index 3224c04fb2..9ccea4d15a 100644 --- a/esphome/components/remote_base/rc_switch_protocol.h +++ b/esphome/components/remote_base/rc_switch_protocol.h @@ -16,9 +16,16 @@ class RCSwitchBase { public: using ProtocolData = RCSwitchData; - RCSwitchBase() = default; - RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, uint32_t one_high, - uint32_t one_low, bool inverted); + constexpr RCSwitchBase() = default; + constexpr RCSwitchBase(uint32_t sync_high, uint32_t sync_low, uint32_t zero_high, uint32_t zero_low, + uint32_t one_high, uint32_t one_low, bool inverted) + : sync_high_(sync_high), + sync_low_(sync_low), + zero_high_(zero_high), + zero_low_(zero_low), + one_high_(one_high), + one_low_(one_low), + inverted_(inverted) {} void one(RemoteTransmitData *dst) const; @@ -58,10 +65,28 @@ class RCSwitchBase { uint32_t zero_low_{}; uint32_t one_high_{}; uint32_t one_low_{}; - bool inverted_{}; + uint32_t inverted_{}; // bool widened so every field is a word: the table is read from flash }; -extern const RCSwitchBase RC_SWITCH_PROTOCOLS[9]; +// Constant-initialized and kept in flash on every platform. The decoder reads entries in place +// through a pointer, which ESP8266 only allows while every field is a whole word; copies out of +// the table go through rc_switch_protocol() +static_assert(sizeof(RCSwitchBase) == 7 * sizeof(uint32_t), "RCSwitchBase must stay word-only for flash reads"); +inline constexpr RCSwitchBase RC_SWITCH_PROTOCOLS[] PROGMEM = { + {0, 0, 0, 0, 0, 0, false}, + {350, 10850, 350, 1050, 1050, 350, false}, + {650, 6500, 650, 1300, 1300, 650, false}, + {3000, 7100, 400, 1100, 900, 600, false}, + {380, 2280, 380, 1140, 1140, 380, false}, + {3000, 7000, 500, 1000, 1000, 500, false}, + {10350, 450, 450, 900, 900, 450, true}, + {300, 9300, 150, 900, 900, 150, false}, + {250, 2500, 250, 1250, 250, 250, false}, +}; + +/// RAM copy of RC_SWITCH_PROTOCOLS[index] (0 when out of range) for the transmit actions and the dumper, made with +/// progmem_memcpy so no byte load ever touches the flash table on ESP8266 +RCSwitchBase rc_switch_protocol(uint8_t index); uint64_t decode_binary_string(const std::string &data); diff --git a/esphome/components/remote_base/remote_base.cpp b/esphome/components/remote_base/remote_base.cpp index 4d9bc55f21..5d1bba16b6 100644 --- a/esphome/components/remote_base/remote_base.cpp +++ b/esphome/components/remote_base/remote_base.cpp @@ -99,29 +99,48 @@ bool RemoteReceiverBinarySensorBase::on_receive(RemoteReceiveData src) { /* RemoteReceiverBase */ +// Slots are counted at code generation; a registration from C++ setup() has none +#ifdef REMOTE_BASE_LISTENER_COUNT +void RemoteReceiverBase::register_listener(RemoteReceiverListener *listener) { + if (this->listeners_.size() == REMOTE_BASE_LISTENER_COUNT) { + ESP_LOGE(TAG, "No %s slot: register it from to_code() with remote_base.add_%s", LOG_STR_LITERAL("listener"), + LOG_STR_LITERAL("listener")); + return; + } + this->listeners_.push_back(listener); +} +#endif + +#ifdef REMOTE_BASE_DUMPER_COUNT void RemoteReceiverBase::register_dumper(RemoteReceiverDumperBase *dumper) { if (dumper->is_secondary()) { - this->secondary_dumpers_.push_back(dumper); - } else { + if (this->secondary_dumper_ == nullptr) { + this->secondary_dumper_ = dumper; + return; + } + } else if (this->dumpers_.size() != REMOTE_BASE_DUMPER_COUNT) { this->dumpers_.push_back(dumper); + return; } + ESP_LOGE(TAG, "No %s slot: register it from to_code() with remote_base.add_%s", LOG_STR_LITERAL("dumper"), + LOG_STR_LITERAL("dumper")); } +#endif -void RemoteReceiverBase::call_listeners_() { +void RemoteReceiverBase::call_listeners_dumpers_() { +#ifdef REMOTE_BASE_LISTENER_COUNT for (auto *listener : this->listeners_) listener->on_receive(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); -} - -void RemoteReceiverBase::call_dumpers_() { +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT bool success = false; for (auto *dumper : this->dumpers_) { if (dumper->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_))) success = true; } - if (!success) { - for (auto *dumper : this->secondary_dumpers_) - dumper->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); - } + if (!success && this->secondary_dumper_ != nullptr) + this->secondary_dumper_->dump(RemoteReceiveData(this->temp_, this->tolerance_, this->tolerance_mode_)); +#endif } void RemoteReceiverBinarySensorBase::dump_config() { LOG_BINARY_SENSOR("", "Remote Receiver Binary Sensor", this); } diff --git a/esphome/components/remote_base/remote_base.h b/esphome/components/remote_base/remote_base.h index 4e2ed4b71c..67e5799bca 100644 --- a/esphome/components/remote_base/remote_base.h +++ b/esphome/components/remote_base/remote_base.h @@ -1,12 +1,14 @@ +#pragma once + +#include #include #include -#pragma once - #include "esphome/components/binary_sensor/binary_sensor.h" #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" namespace esphome::remote_base { @@ -141,6 +143,22 @@ class RemoteRMTChannel { #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 +// Protocol shapes, checked where a protocol is used so a missing method fails at the use site +// instead of deep inside a template body. Receive-only protocols such as RCSwitchBase decode +// without encoding. +template +concept RemoteProtocolDecoder = requires(T proto, RemoteReceiveData src) { + { proto.decode(src) } -> std::same_as>; +}; +template +concept RemoteProtocolDumper = RemoteProtocolDecoder && requires(T proto, const typename T::ProtocolData &data) { + proto.dump(data); +}; +template +concept RemoteProtocolEncoder = requires(T proto, RemoteTransmitData *dst, const typename T::ProtocolData &data) { + proto.encode(dst, data); +}; + class RemoteTransmitterBase : public RemoteComponentBase { public: RemoteTransmitterBase(InternalGPIOPin *pin) : RemoteComponentBase(pin) {} @@ -162,8 +180,8 @@ class RemoteTransmitterBase : public RemoteComponentBase { this->temp_.reset(); return TransmitCall(this); } - template - void transmit(const Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { + template + void transmit(const typename Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { auto call = this->transmit(); Protocol().encode(call.get_data(), data); call.set_send_times(send_times); @@ -194,24 +212,37 @@ class RemoteReceiverDumperBase { class RemoteReceiverBase : public RemoteComponentBase { public: RemoteReceiverBase(InternalGPIOPin *pin) : RemoteComponentBase(pin) {} - void register_listener(RemoteReceiverListener *listener) { this->listeners_.push_back(listener); } + // Slots are counted at code generation; without one the call fails at compile time with the same message + // the runtime check logs +#ifdef REMOTE_BASE_LISTENER_COUNT + void register_listener(RemoteReceiverListener *listener); +#else + template void register_listener(T *) { + static_assert(sizeof(T) == 0, "No listener slot: register it from to_code() with remote_base.add_listener"); + } +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT void register_dumper(RemoteReceiverDumperBase *dumper); +#else + template void register_dumper(T *) { + static_assert(sizeof(T) == 0, "No dumper slot: register it from to_code() with remote_base.add_dumper"); + } +#endif void set_tolerance(uint32_t tolerance, ToleranceMode tolerance_mode) { this->tolerance_ = tolerance; this->tolerance_mode_ = tolerance_mode; } protected: - void call_listeners_(); - void call_dumpers_(); - void call_listeners_dumpers_() { - this->call_listeners_(); - this->call_dumpers_(); - } + void call_listeners_dumpers_(); - std::vector listeners_; - std::vector dumpers_; - std::vector secondary_dumpers_; +#ifdef REMOTE_BASE_LISTENER_COUNT + StaticVector listeners_; +#endif +#ifdef REMOTE_BASE_DUMPER_COUNT + StaticVector dumpers_; + RemoteReceiverDumperBase *secondary_dumper_{nullptr}; // runs only when no primary dumper matched +#endif RawTimings temp_; uint32_t tolerance_{25}; ToleranceMode tolerance_mode_{TOLERANCE_MODE_PERCENTAGE}; @@ -229,15 +260,14 @@ class RemoteReceiverBinarySensorBase : public binary_sensor::BinarySensorInitial /* TEMPLATES */ +// Protocols are used only through their concrete type (see the RemoteProtocol* concepts); encode/decode/dump +// stay non-virtual so unused ones link out template class RemoteProtocol { public: using ProtocolData = T; - virtual void encode(RemoteTransmitData *dst, const ProtocolData &data) = 0; - virtual optional decode(RemoteReceiveData src) = 0; - virtual void dump(const ProtocolData &data) = 0; }; -template class RemoteReceiverBinarySensor : public RemoteReceiverBinarySensorBase { +template class RemoteReceiverBinarySensor : public RemoteReceiverBinarySensorBase { public: RemoteReceiverBinarySensor() : RemoteReceiverBinarySensorBase() {} @@ -255,7 +285,7 @@ template class RemoteReceiverBinarySensor : public RemoteReceiverBin T::ProtocolData data_; }; -template +template class RemoteReceiverTrigger final : public Trigger, public RemoteReceiverListener { protected: bool on_receive(RemoteReceiveData src) override { @@ -276,8 +306,8 @@ class RemoteTransmittable { void set_transmitter(RemoteTransmitterBase *transmitter) { this->transmitter_ = transmitter; } protected: - template - void transmit_(const Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { + template + void transmit_(const typename Protocol::ProtocolData &data, uint32_t send_times = 1, uint32_t send_wait = 0) { this->transmitter_->transmit(data, send_times, send_wait); } RemoteTransmitterBase *transmitter_; @@ -298,7 +328,7 @@ template class RemoteTransmitterActionBase : public RemoteTransm virtual void encode(RemoteTransmitData *dst, Ts... x) = 0; }; -template class RemoteReceiverDumper : public RemoteReceiverDumperBase { +template class RemoteReceiverDumper : public RemoteReceiverDumperBase { public: bool dump(RemoteReceiveData src) override { auto proto = T(); diff --git a/esphome/components/remote_base/roomba_protocol.h b/esphome/components/remote_base/roomba_protocol.h index 3582dac398..8db025f812 100644 --- a/esphome/components/remote_base/roomba_protocol.h +++ b/esphome/components/remote_base/roomba_protocol.h @@ -12,9 +12,9 @@ struct RoombaData { class RoombaProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const RoombaData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const RoombaData &data) override; + void encode(RemoteTransmitData *dst, const RoombaData &data); + optional decode(RemoteReceiveData src); + void dump(const RoombaData &data); }; DECLARE_REMOTE_PROTOCOL(Roomba) diff --git a/esphome/components/remote_base/samsung36_protocol.h b/esphome/components/remote_base/samsung36_protocol.h index 4f15d906e7..df4e1af8d8 100644 --- a/esphome/components/remote_base/samsung36_protocol.h +++ b/esphome/components/remote_base/samsung36_protocol.h @@ -16,9 +16,9 @@ struct Samsung36Data { class Samsung36Protocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const Samsung36Data &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const Samsung36Data &data) override; + void encode(RemoteTransmitData *dst, const Samsung36Data &data); + optional decode(RemoteReceiveData src); + void dump(const Samsung36Data &data); }; DECLARE_REMOTE_PROTOCOL(Samsung36) diff --git a/esphome/components/remote_base/samsung_protocol.h b/esphome/components/remote_base/samsung_protocol.h index bb234d681d..dfa22ff85c 100644 --- a/esphome/components/remote_base/samsung_protocol.h +++ b/esphome/components/remote_base/samsung_protocol.h @@ -14,9 +14,9 @@ struct SamsungData { class SamsungProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SamsungData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SamsungData &data) override; + void encode(RemoteTransmitData *dst, const SamsungData &data); + optional decode(RemoteReceiveData src); + void dump(const SamsungData &data); }; DECLARE_REMOTE_PROTOCOL(Samsung) diff --git a/esphome/components/remote_base/sony_protocol.h b/esphome/components/remote_base/sony_protocol.h index eb873e8b7d..f83b2908b6 100644 --- a/esphome/components/remote_base/sony_protocol.h +++ b/esphome/components/remote_base/sony_protocol.h @@ -16,9 +16,9 @@ struct SonyData { class SonyProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SonyData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SonyData &data) override; + void encode(RemoteTransmitData *dst, const SonyData &data); + optional decode(RemoteReceiveData src); + void dump(const SonyData &data); }; DECLARE_REMOTE_PROTOCOL(Sony) diff --git a/esphome/components/remote_base/symphony_protocol.h b/esphome/components/remote_base/symphony_protocol.h index 7caf5eab86..40a5c2daec 100644 --- a/esphome/components/remote_base/symphony_protocol.h +++ b/esphome/components/remote_base/symphony_protocol.h @@ -17,9 +17,9 @@ struct SymphonyData { class SymphonyProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const SymphonyData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const SymphonyData &data) override; + void encode(RemoteTransmitData *dst, const SymphonyData &data); + optional decode(RemoteReceiveData src); + void dump(const SymphonyData &data); }; DECLARE_REMOTE_PROTOCOL(Symphony) diff --git a/esphome/components/remote_base/toshiba_ac_protocol.h b/esphome/components/remote_base/toshiba_ac_protocol.h index 8a853005ac..35d5af314c 100644 --- a/esphome/components/remote_base/toshiba_ac_protocol.h +++ b/esphome/components/remote_base/toshiba_ac_protocol.h @@ -14,9 +14,9 @@ struct ToshibaAcData { class ToshibaAcProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const ToshibaAcData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const ToshibaAcData &data) override; + void encode(RemoteTransmitData *dst, const ToshibaAcData &data); + optional decode(RemoteReceiveData src); + void dump(const ToshibaAcData &data); }; DECLARE_REMOTE_PROTOCOL(ToshibaAc) diff --git a/esphome/components/remote_base/toto_protocol.h b/esphome/components/remote_base/toto_protocol.h index 285c9f2125..8e965a5c73 100644 --- a/esphome/components/remote_base/toto_protocol.h +++ b/esphome/components/remote_base/toto_protocol.h @@ -16,9 +16,9 @@ struct TotoData { class TotoProtocol : public RemoteProtocol { public: - void encode(RemoteTransmitData *dst, const TotoData &data) override; - optional decode(RemoteReceiveData src) override; - void dump(const TotoData &data) override; + void encode(RemoteTransmitData *dst, const TotoData &data); + optional decode(RemoteReceiveData src); + void dump(const TotoData &data); }; DECLARE_REMOTE_PROTOCOL(Toto) diff --git a/esphome/components/remote_receiver/__init__.py b/esphome/components/remote_receiver/__init__.py index 6e8c73d331..b2fd87165e 100644 --- a/esphome/components/remote_receiver/__init__.py +++ b/esphome/components/remote_receiver/__init__.py @@ -221,11 +221,11 @@ async def to_code(config: ConfigType) -> None: dumpers = await remote_base.build_dumpers(config[CONF_DUMP]) for dumper in dumpers: - cg.add(var.register_dumper(dumper)) + remote_base.add_dumper(var, dumper) triggers = await remote_base.build_triggers(config) for trigger in triggers: - cg.add(var.register_listener(trigger)) + remote_base.add_listener(var, trigger) await cg.register_component(var, config) cg.add( diff --git a/esphome/components/toshiba/climate.py b/esphome/components/toshiba/climate.py index 3b1e7352f9..e5f8544f2f 100644 --- a/esphome/components/toshiba/climate.py +++ b/esphome/components/toshiba/climate.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import climate_ir +from esphome.components import climate_ir, remote_base import esphome.config_validation as cv from esphome.const import CONF_MODEL from esphome.types import ConfigType @@ -26,5 +26,6 @@ CONFIG_SCHEMA = climate_ir.climate_ir_with_receiver_schema(ToshibaClimate).exten async def to_code(config: ConfigType) -> None: + remote_base.request_protocol("toshiba_ac") # used from C++ var = await climate_ir.new_climate_ir(config) cg.add(var.set_model(config[CONF_MODEL])) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9144e65576..6b9b9eda43 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -137,6 +137,43 @@ #define MICRONOVA_LISTENER_COUNT 1 #define USE_MICRONOVA_WRITER #define MK2PVROUTER_LISTENER_COUNT 1 +#define REMOTE_BASE_DUMPER_COUNT 1 +#define REMOTE_BASE_LISTENER_COUNT 1 +#define USE_REMOTE_PROTOCOL_ABBWELCOME +#define USE_REMOTE_PROTOCOL_AEHA +#define USE_REMOTE_PROTOCOL_BEO4 +#define USE_REMOTE_PROTOCOL_BRENNENSTUHL +#define USE_REMOTE_PROTOCOL_BYRONSX +#define USE_REMOTE_PROTOCOL_CANALSAT +#define USE_REMOTE_PROTOCOL_COOLIX +#define USE_REMOTE_PROTOCOL_DISH +#define USE_REMOTE_PROTOCOL_DOOYA +#define USE_REMOTE_PROTOCOL_DRAYTON +#define USE_REMOTE_PROTOCOL_DYSON +#define USE_REMOTE_PROTOCOL_GOBOX +#define USE_REMOTE_PROTOCOL_HAIER +#define USE_REMOTE_PROTOCOL_JVC +#define USE_REMOTE_PROTOCOL_KEELOQ +#define USE_REMOTE_PROTOCOL_LG +#define USE_REMOTE_PROTOCOL_MAGIQUEST +#define USE_REMOTE_PROTOCOL_MIDEA +#define USE_REMOTE_PROTOCOL_MIRAGE +#define USE_REMOTE_PROTOCOL_NEC +#define USE_REMOTE_PROTOCOL_NEXA +#define USE_REMOTE_PROTOCOL_PANASONIC +#define USE_REMOTE_PROTOCOL_PIONEER +#define USE_REMOTE_PROTOCOL_PRONTO +#define USE_REMOTE_PROTOCOL_RAW +#define USE_REMOTE_PROTOCOL_RC5 +#define USE_REMOTE_PROTOCOL_RC6 +#define USE_REMOTE_PROTOCOL_RC_SWITCH +#define USE_REMOTE_PROTOCOL_ROOMBA +#define USE_REMOTE_PROTOCOL_SAMSUNG +#define USE_REMOTE_PROTOCOL_SAMSUNG36 +#define USE_REMOTE_PROTOCOL_SONY +#define USE_REMOTE_PROTOCOL_SYMPHONY +#define USE_REMOTE_PROTOCOL_TOSHIBA_AC +#define USE_REMOTE_PROTOCOL_TOTO #define SERIAL_PROXY_COUNT 2 #define SNTP_SERVER_COUNT 3 #define USE_MEDIA_PLAYER diff --git a/esphome/cpp_helpers.py b/esphome/cpp_helpers.py index 53b59cb124..fc44d27f47 100644 --- a/esphome/cpp_helpers.py +++ b/esphome/cpp_helpers.py @@ -1,4 +1,4 @@ -from collections.abc import Callable +from collections.abc import Callable, Hashable from dataclasses import dataclass, field import logging @@ -142,9 +142,10 @@ _SLOT_COUNTER_DOMAIN = "slot_counter" @dataclass class _SlotCounterState: - """Per-run slot counter state: requested counts and already-emitted defines.""" + """Per-run slot counter state: requested counts per define and key, and + already-emitted defines.""" - counts: dict[str, int] = field(default_factory=dict) + counts: dict[str, dict[Hashable, int]] = field(default_factory=dict) emitted: set[str] = field(default_factory=set) @@ -156,11 +157,13 @@ def _get_slot_counter_state() -> _SlotCounterState: def get_slot_count(define: str) -> int: - """Number of slots requested so far for `define`.""" - return _get_slot_counter_state().counts.get(define, 0) + """Value `define` would be emitted with so far: the largest count requested + under any one key, which is the plain request count when no key is used.""" + counts = _get_slot_counter_state().counts.get(define) + return max(counts.values()) if counts else 0 -def slot_counter(define: str) -> Callable[[], None]: +def slot_counter(define: str) -> Callable[..., None]: """Create a request_slot function for codegen-sized storage. The pattern behind a StaticVector listener array: a consumer's to_code @@ -169,6 +172,11 @@ def slot_counter(define: str) -> Callable[[], None]: emitted with the requested count. No requests, no define: the guarded storage and its registration method compile out entirely. + When several objects each declare the storage at the same size (one list + per receiver, per hub, ...) the caller passes the owning object as `key` + and the define becomes the largest count any one key requested, not the + total. Requests without a key share one count. + The counts live in a table under CORE.data, which clears between runs. A request arriving after the define was already emitted raises instead of silently undercounting: the define would keep the stale smaller value and @@ -179,10 +187,10 @@ def slot_counter(define: str) -> Callable[[], None]: async def emit_job() -> None: state = _get_slot_counter_state() state.emitted.add(define) - # Scheduled only by the first request, so the count is always >= 1 here. - add_define(define, state.counts[define]) + # Scheduled only by the first request, so there is at least one count here. + add_define(define, max(state.counts[define].values())) - def request_slot() -> None: + def request_slot(key: Hashable = None) -> None: state = _get_slot_counter_state() if define in state.emitted: raise ValueError( @@ -190,10 +198,16 @@ def slot_counter(define: str) -> Callable[[], None]: f"define was emitted; request slots from to_code, not from a " f"job running after FINAL emission" ) - counts = state.counts - counts[define] = (count := counts.get(define, 0) + 1) - if count == 1: + counts = state.counts.get(define) + if counts is None: + counts = state.counts[define] = {} CORE.add_job(emit_job) + elif (key is None) != (None in counts): + # a keyed and an unkeyed request would compare buckets instead of adding up + raise ValueError( + f"slot_counter('{define}'): every request must use a key, or none of them" + ) + counts[key] = counts.get(key, 0) + 1 return request_slot diff --git a/tests/component_tests/remote_receiver/__init__.py b/tests/component_tests/remote_receiver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/remote_receiver/config/receiver_bare.yaml b/tests/component_tests/remote_receiver/config/receiver_bare.yaml new file mode 100644 index 0000000000..b474194801 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_bare.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml b/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml new file mode 100644 index 0000000000..32c1b07f57 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_with_dumpers.yaml @@ -0,0 +1,24 @@ +esphome: + name: test + +esp32: + board: esp32dev + +logger: + +remote_receiver: + - id: rcvr + pin: GPIO4 + dump: + - nec + - rc_switch + on_nec: + then: + - logger.log: nec + +binary_sensor: + - platform: remote_receiver + name: Remote Input + nec: + address: 0x1234 + command: 0x5678 diff --git a/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml b/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml new file mode 100644 index 0000000000..c443a842f2 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_with_proxies.yaml @@ -0,0 +1,22 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr_ir + pin: GPIO4 + - id: rcvr_rf + pin: GPIO5 + +infrared: + - platform: ir_rf_proxy + name: IR Receiver + remote_receiver_id: rcvr_ir + +radio_frequency: + - platform: ir_rf_proxy + name: RF Receiver + frequency: 433.92MHz + remote_receiver_id: rcvr_rf diff --git a/tests/component_tests/remote_receiver/test_slot_counts.py b/tests/component_tests/remote_receiver/test_slot_counts.py new file mode 100644 index 0000000000..f381a64092 --- /dev/null +++ b/tests/component_tests/remote_receiver/test_slot_counts.py @@ -0,0 +1,91 @@ +"""Listener and dumper StaticVector sizes come from codegen slot counts.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.automation import ACTION_REGISTRY +from esphome.components import remote_base +import esphome.config_validation as cv + +from ..helpers import get_define_value + + +def test_dumper_and_listener_counts( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_dumpers.yaml")) + # nec and rc_switch dumpers + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") == "2" + # on_nec trigger plus the remote_receiver binary sensor + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") == "2" + + +def test_bare_receiver_emits_no_counts( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_bare.yaml")) + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") is None + + +def test_proxy_receivers_count_as_listeners( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_proxies.yaml")) + # one proxy entity listens on each of the two receivers; every receiver's list gets the + # capacity of the busiest one, so this is the largest per receiver count, not the sum + assert get_define_value("REMOTE_BASE_LISTENER_COUNT") == "1" + assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None + + +def test_only_used_protocol_sources_are_compiled( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + generate_main(component_config_path("receiver_with_dumpers.yaml")) + excluded = set(remote_base.FILTER_SOURCE_FILES()) + assert "nec_protocol.cpp" not in excluded + assert "rc_switch_protocol.cpp" not in excluded + assert "sony_protocol.cpp" in excluded + assert "remote_base.cpp" not in excluded + + +def test_every_registry_name_maps_to_a_protocol_source() -> None: + """A registry name must resolve to a source file or request_protocol rejects it.""" + names = ( + set(remote_base.BINARY_SENSOR_REGISTRY) + | set(remote_base.DUMPER_REGISTRY) + | {key.removeprefix("on_") for key in remote_base.TRIGGER_REGISTRY} + | { + key.removeprefix("remote_transmitter.transmit_") + for key in ACTION_REGISTRY + if key.startswith("remote_transmitter.transmit_") + } + ) + assert len(names) > 40 + for name in names: + assert remote_base._protocol_stem(name) in remote_base._PROTOCOL_STEMS, name + + +def test_request_protocol_rejects_unknown_names() -> None: + """A misspelled protocol would otherwise surface only as a link error.""" + with pytest.raises(ValueError, match="Unknown remote protocol 'toshiba'"): + remote_base.request_protocol("toshiba") + + +def test_dump_list_is_deduplicated_across_forms() -> None: + dumpers = remote_base.validate_dumpers(["raw", {"raw": None}, "nec", "nec"]) + assert [ + next(k for k in entry if k in remote_base.DUMPER_REGISTRY) for entry in dumpers + ] == ["raw", "nec"] + + +@pytest.mark.parametrize("bad", [["nec", None], [5]]) +def test_dump_list_rejects_invalid_entries_with_a_validation_error(bad: list) -> None: + with pytest.raises(cv.Invalid): + remote_base.validate_dumpers(bad) diff --git a/tests/components/remote_receiver/bare-common.yaml b/tests/components/remote_receiver/bare-common.yaml new file mode 100644 index 0000000000..c100c5c2da --- /dev/null +++ b/tests/components/remote_receiver/bare-common.yaml @@ -0,0 +1,6 @@ +# A receiver with no dumpers and no listeners compiles both lists out. +# Only built while remote_receiver is tested in isolation: the counts are global defines, +# so this variant cannot be merged with configs that register any. +remote_receiver: + - id: rcvr_bare + pin: ${pin} diff --git a/tests/components/remote_receiver/test-bare.esp32-idf.yaml b/tests/components/remote_receiver/test-bare.esp32-idf.yaml new file mode 100644 index 0000000000..152853b65f --- /dev/null +++ b/tests/components/remote_receiver/test-bare.esp32-idf.yaml @@ -0,0 +1,5 @@ +substitutions: + pin: GPIO2 + +packages: + bare: !include bare-common.yaml diff --git a/tests/unit_tests/test_cpp_helpers.py b/tests/unit_tests/test_cpp_helpers.py index 1c0e0d0a93..725c1daebb 100644 --- a/tests/unit_tests/test_cpp_helpers.py +++ b/tests/unit_tests/test_cpp_helpers.py @@ -187,6 +187,31 @@ def test_slot_counter_emits_requested_count() -> None: assert _define_value("TEST_SLOT_COUNT") == "2" +def test_slot_counter_keyed_emits_largest_count() -> None: + """Keyed requests size storage every key declares at the same capacity: + the define is the busiest key's count, not the total over all keys.""" + request = ch.slot_counter("TEST_SLOT_COUNT_KEYED") + request("rx_a") + request("rx_a") + request("rx_a") + request("rx_b") + assert ch.get_slot_count("TEST_SLOT_COUNT_KEYED") == 3 + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT_KEYED") == "3" + + +def test_slot_counter_rejects_mixed_keyed_and_unkeyed_requests() -> None: + """A keyed and an unkeyed request for one define cannot be sized together.""" + request = ch.slot_counter("TEST_SLOT_COUNT_MIXED") + request("rx_a") + with pytest.raises(ValueError, match="TEST_SLOT_COUNT_MIXED"): + request() + unkeyed = ch.slot_counter("TEST_SLOT_COUNT_MIXED_2") + unkeyed() + with pytest.raises(ValueError, match="TEST_SLOT_COUNT_MIXED_2"): + unkeyed("rx_a") + + def test_slot_counter_without_requests_emits_nothing() -> None: """No requests, no job, no define — the guarded storage compiles out.""" ch.slot_counter("TEST_SLOT_COUNT_UNUSED") From 2578f17dc8705b0a18ce4a67f6de4bad8b2c6101 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:25:41 -0500 Subject: [PATCH 50/55] Bump bundled esphome-device-builder to 1.14.7 (#19096) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cfa47fbdad..6f500dbe6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.6 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.7 RUN \ platformio settings set enable_telemetry No \ From 2b71d5496d1c415e335a637c0839259d2ba7f39a Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:45:02 -0400 Subject: [PATCH 51/55] [const] Centralize definition of `CONF_MANUFACTURER` (#19098) --- esphome/components/const/__init__.py | 1 + esphome/components/esp32_ble_server/__init__.py | 2 +- esphome/components/sendspin/__init__.py | 2 +- tests/component_tests/sendspin/test_device_info.py | 7 ++----- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 49a625e3f1..256ab5c0a3 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -30,6 +30,7 @@ CONF_KEYS = "keys" CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" +CONF_MANUFACTURER = "manufacturer" CONF_NOX_INDEX = "nox_index" CONF_ON_PACKET = "on_packet" CONF_ON_RECEIVE = "on_receive" diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index d8095cd702..118ae06e42 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -3,6 +3,7 @@ import encodings from esphome import automation import esphome.codegen as cg from esphome.components import esp32_ble +from esphome.components.const import CONF_MANUFACTURER from esphome.components.esp32 import request_bluetooth from esphome.components.esp32_ble import BTLoggers, bt_uuid import esphome.config_validation as cv @@ -41,7 +42,6 @@ CONF_DESCRIPTORS = "descriptors" CONF_ENDIANNESS = "endianness" CONF_FIRMWARE_VERSION = "firmware_version" CONF_INDICATE = "indicate" -CONF_MANUFACTURER = "manufacturer" CONF_MANUFACTURER_DATA = "manufacturer_data" CONF_MAX_CLIENTS = "max_clients" CONF_ON_WRITE = "on_write" diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index c21047c70a..fda4d4f954 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -3,6 +3,7 @@ from dataclasses import dataclass, field from esphome import automation import esphome.codegen as cg from esphome.components import esp32, network, psram, socket, wifi +from esphome.components.const import CONF_MANUFACTURER import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -33,7 +34,6 @@ CONF_DISPLAY_OFFSET = "display_offset" CONF_SENDSPIN_ID = "sendspin_id" CONF_FIRMWARE_VERSION = "firmware_version" -CONF_MANUFACTURER = "manufacturer" # An empty device information string would be sent to the server as an empty value rather than # falling back, so reject it instead of silently substituting the fallback. The 127 byte cap keeps diff --git a/tests/component_tests/sendspin/test_device_info.py b/tests/component_tests/sendspin/test_device_info.py index 833dd398b4..61c10676da 100644 --- a/tests/component_tests/sendspin/test_device_info.py +++ b/tests/component_tests/sendspin/test_device_info.py @@ -8,11 +8,8 @@ from pathlib import Path import pytest from esphome import config_validation as cv -from esphome.components.sendspin import ( - CONF_FIRMWARE_VERSION, - CONF_MANUFACTURER, - CONFIG_SCHEMA, -) +from esphome.components.const import CONF_MANUFACTURER +from esphome.components.sendspin import CONF_FIRMWARE_VERSION, CONFIG_SCHEMA from esphome.const import CONF_MODEL, PlatformFramework from tests.component_tests.types import SetCoreConfigCallable From 9f9df85aa00bbac9d4c0db6905e538a80f279834 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:04:48 +0000 Subject: [PATCH 52/55] Bump aioesphomeapi from 46.4.0 to 46.4.1 (#19104) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72c42dad32..c73887a39d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.4.0 click==8.3.3 -aioesphomeapi==46.4.0 +aioesphomeapi==46.4.1 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.151.3 puremagic==2.2.0 From ff9b2a1c83edb7b542fa04f93407b86a5c04bcde Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 11 Sep 2026 18:20:40 -0500 Subject: [PATCH 53/55] [remote_receiver] Size the RMT ring buffer from receive symbols by default (#19100) --- .../components/remote_receiver/__init__.py | 14 ++++--- .../remote_receiver/remote_receiver.h | 4 +- .../remote_receiver/remote_receiver_rmt.cpp | 38 +++++++++++-------- .../config/receiver_buffer_size.yaml | 10 +++++ .../config/receiver_esp32_c2.yaml | 12 ++++++ .../config/receiver_esp8266.yaml | 9 +++++ .../remote_receiver/test_buffer_size.py | 28 ++++++++++++++ .../remote_receiver/test_slot_counts.py | 4 +- 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml create mode 100644 tests/component_tests/remote_receiver/config/receiver_esp8266.yaml create mode 100644 tests/component_tests/remote_receiver/test_buffer_size.py diff --git a/esphome/components/remote_receiver/__init__.py b/esphome/components/remote_receiver/__init__.py index b2fd87165e..6eaecf7ab0 100644 --- a/esphome/components/remote_receiver/__init__.py +++ b/esphome/components/remote_receiver/__init__.py @@ -114,15 +114,18 @@ CONFIG_SCHEMA = remote_base.validate_triggers( cv.Optional(CONF_TOLERANCE, default="25%"): validate_tolerance, cv.SplitDefault( CONF_BUFFER_SIZE, - esp32="10000b", - esp32_c2="1000b", - esp32_c61="1000b", + esp32=cv.UNDEFINED, + # the pulse ring needs a size; only RMT targets size themselves in setup() + **{ + f"esp32_{variant.removeprefix('ESP32').lower()}": "1000b" + for variant in esp32_rmt.VARIANTS_NO_RMT + }, esp8266="1000b", bk72xx="1000b", ln882x="1000b", rtl87xx="1000b", rp2="1000b", - ): cv.validate_bytes, + ): cv.All(cv.validate_bytes, cv.int_range(min=64)), cv.Optional(CONF_FILTER, default="50us"): cv.All( cv.positive_time_period_microseconds, cv.Range(max=TimePeriod(microseconds=4294967295)), @@ -233,7 +236,8 @@ async def to_code(config: ConfigType) -> None: config[CONF_TOLERANCE][CONF_VALUE], config[CONF_TOLERANCE][CONF_TYPE] ) ) - cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) + if CONF_BUFFER_SIZE in config: + cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) cg.add(var.set_filter_us(config[CONF_FILTER])) cg.add(var.set_idle_us(config[CONF_IDLE])) diff --git a/esphome/components/remote_receiver/remote_receiver.h b/esphome/components/remote_receiver/remote_receiver.h index f9ec054fe3..e59a8b2557 100644 --- a/esphome/components/remote_receiver/remote_receiver.h +++ b/esphome/components/remote_receiver/remote_receiver.h @@ -47,7 +47,7 @@ struct RemoteReceiverComponentStore { /// The position last read from volatile uint32_t buffer_read{0}; bool overflow{false}; - uint32_t buffer_size{1000}; + uint32_t buffer_size{0}; uint32_t receive_size{0}; uint32_t filter_symbols{0}; esp_err_t error{ESP_OK}; @@ -101,7 +101,7 @@ class RemoteReceiverComponent final : public remote_base::RemoteReceiverBase, HighFrequencyLoopRequester high_freq_; #endif - uint32_t buffer_size_{}; + uint32_t buffer_size_{}; // 0 on RMT targets: sized from receive_symbols in setup() uint32_t filter_us_{10}; uint32_t idle_us_{10000}; }; diff --git a/esphome/components/remote_receiver/remote_receiver_rmt.cpp b/esphome/components/remote_receiver/remote_receiver_rmt.cpp index 632ca9763a..4eebbbb16f 100644 --- a/esphome/components/remote_receiver/remote_receiver_rmt.cpp +++ b/esphome/components/remote_receiver/remote_receiver_rmt.cpp @@ -10,6 +10,7 @@ namespace esphome::remote_receiver { static const char *const TAG = "remote_receiver"; +static constexpr uint32_t DEFAULT_BUFFER_SLOTS = 4; static bool IRAM_ATTR HOT rmt_callback(rmt_channel_handle_t channel, const rmt_rx_done_event_data_t *event, void *arg) { RemoteReceiverComponentStore *store = (RemoteReceiverComponentStore *) arg; @@ -104,7 +105,11 @@ void RemoteReceiverComponent::setup() { this->store_.config.signal_range_max_ns = this->idle_us_ * 1000; this->store_.filter_symbols = this->filter_symbols_; this->store_.receive_size = this->receive_symbols_ * sizeof(rmt_symbol_word_t); - this->store_.buffer_size = std::max((event_size + this->store_.receive_size) * 2, this->buffer_size_); + // one slot per pending rmt_receive; two are the floor (one filling while one is decoded), and + // the default of four covers a few frames queued across a stalled loop pass + const uint32_t slot_size = event_size + this->store_.receive_size; + this->store_.buffer_size = + this->buffer_size_ != 0 ? std::max(slot_size * 2, this->buffer_size_) : slot_size * DEFAULT_BUFFER_SLOTS; this->store_.buffer = new uint8_t[this->store_.buffer_size]; error = rmt_receive(this->channel_, (uint8_t *) this->store_.buffer + event_size, this->store_.receive_size, &this->store_.config); @@ -117,20 +122,23 @@ void RemoteReceiverComponent::setup() { } void RemoteReceiverComponent::dump_config() { - ESP_LOGCONFIG(TAG, - "Remote Receiver:\n" - " Clock resolution: %" PRIu32 " hz\n" - " RMT symbols: %" PRIu32 "\n" - " Filter symbols: %" PRIu32 "\n" - " Receive symbols: %" PRIu32 "\n" - " Tolerance: %" PRIu32 "%s\n" - " Carrier frequency: %" PRIu32 " hz\n" - " Carrier duty: %u%%\n" - " Filter out pulses shorter than: %" PRIu32 " us\n" - " Signal is done after %" PRIu32 " us of no changes", - this->clock_resolution_, this->rmt_symbols_, this->filter_symbols_, this->receive_symbols_, - this->tolerance_, (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? " us" : "%", - this->carrier_frequency_, this->carrier_duty_percent_, this->filter_us_, this->idle_us_); + ESP_LOGCONFIG( + TAG, + "Remote Receiver:\n" + " Clock resolution: %" PRIu32 " hz\n" + " RMT symbols: %" PRIu32 "\n" + " Filter symbols: %" PRIu32 "\n" + " Receive symbols: %" PRIu32 "\n" + " Buffer size: %" PRIu32 " bytes\n" + " Tolerance: %" PRIu32 "%s\n" + " Carrier frequency: %" PRIu32 " hz\n" + " Carrier duty: %u%%\n" + " Filter out pulses shorter than: %" PRIu32 " us\n" + " Signal is done after %" PRIu32 " us of no changes", + this->clock_resolution_, this->rmt_symbols_, this->filter_symbols_, this->receive_symbols_, + this->store_.buffer_size, this->tolerance_, + (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? LOG_STR_LITERAL(" us") : LOG_STR_LITERAL("%"), + this->carrier_frequency_, this->carrier_duty_percent_, this->filter_us_, this->idle_us_); LOG_PIN(" Pin: ", this->pin_); if (this->is_failed()) { ESP_LOGE(TAG, "Configuring RMT driver failed: %s (%s)", esp_err_to_name(this->error_code_), diff --git a/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml b/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml new file mode 100644 index 0000000000..0b334954eb --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_buffer_size.yaml @@ -0,0 +1,10 @@ +esphome: + name: test + +esp32: + board: esp32dev + +remote_receiver: + - id: rcvr + pin: GPIO4 + buffer_size: 2kb diff --git a/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml b/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml new file mode 100644 index 0000000000..c4497fefd8 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_esp32_c2.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: esp32-c2-devkitm-1 + variant: esp32c2 + framework: + type: esp-idf + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml b/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml new file mode 100644 index 0000000000..f22d00d630 --- /dev/null +++ b/tests/component_tests/remote_receiver/config/receiver_esp8266.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +remote_receiver: + - id: rcvr + pin: GPIO4 diff --git a/tests/component_tests/remote_receiver/test_buffer_size.py b/tests/component_tests/remote_receiver/test_buffer_size.py new file mode 100644 index 0000000000..9bfd12d9f5 --- /dev/null +++ b/tests/component_tests/remote_receiver/test_buffer_size.py @@ -0,0 +1,28 @@ +"""buffer_size reaches the receiver when set, and always on the pulse ring targets.""" + +from collections.abc import Callable +from pathlib import Path + + +def test_explicit_buffer_size_is_passed_through( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_buffer_size.yaml")) + assert "rcvr->set_buffer_size(2000);" in main_cpp + + +def test_pulse_ring_target_keeps_a_default( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_esp8266.yaml")) + assert "rcvr->set_buffer_size(1000);" in main_cpp + + +def test_esp32_variant_without_rmt_keeps_a_default( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("receiver_esp32_c2.yaml")) + assert "rcvr->set_buffer_size(1000);" in main_cpp diff --git a/tests/component_tests/remote_receiver/test_slot_counts.py b/tests/component_tests/remote_receiver/test_slot_counts.py index f381a64092..4d69e6d923 100644 --- a/tests/component_tests/remote_receiver/test_slot_counts.py +++ b/tests/component_tests/remote_receiver/test_slot_counts.py @@ -27,7 +27,9 @@ def test_bare_receiver_emits_no_counts( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], ) -> None: - generate_main(component_config_path("receiver_bare.yaml")) + main_cpp = generate_main(component_config_path("receiver_bare.yaml")) + # the RMT ring is sized in setup() unless buffer_size is set + assert "set_buffer_size" not in main_cpp assert get_define_value("REMOTE_BASE_DUMPER_COUNT") is None assert get_define_value("REMOTE_BASE_LISTENER_COUNT") is None From eecea15f4f714af7dd7278cd2433ee3c800db92f Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:42:08 +1000 Subject: [PATCH 54/55] [core][lvgl] Migrate codegen helpers from LVGL to core code (#19105) Co-authored-by: Claude Sonnet 5 --- esphome/components/lvgl/automation.py | 3 +- esphome/components/lvgl/defines.py | 43 +---------- esphome/components/lvgl/lv_validation.py | 4 +- esphome/components/lvgl/widgets/__init__.py | 3 +- esphome/cpp_generator.py | 39 ++++++++++ tests/unit_tests/test_cpp_generator.py | 79 +++++++++++++++++++++ 6 files changed, 122 insertions(+), 49 deletions(-) diff --git a/esphome/components/lvgl/automation.py b/esphome/components/lvgl/automation.py index a62f466413..c23a36c389 100644 --- a/esphome/components/lvgl/automation.py +++ b/esphome/components/lvgl/automation.py @@ -14,7 +14,7 @@ from esphome.const import ( CONF_TIMEOUT, ) from esphome.core import Lambda -from esphome.cpp_generator import TemplateArguments, get_variable +from esphome.cpp_generator import StaticCastExpression, TemplateArguments, get_variable from esphome.cpp_types import nullptr from .defines import ( @@ -30,7 +30,6 @@ from .defines import ( CONF_SHOW_SNOW, CONF_TOP_LAYER, PARTS, - StaticCastExpression, add_warning, get_focused_widgets, get_options, diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 1eee8041f9..73fc58736b 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -10,12 +10,7 @@ from typing import Any from esphome import codegen as cg, config_validation as cv from esphome.const import CONF_ITEMS from esphome.core import CORE, ID, Lambda -from esphome.cpp_generator import ( - CallExpression, - LambdaExpression, - MockObj, - MockObjClass, -) +from esphome.cpp_generator import MockObj, StaticCastExpression, call_lambda from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor from esphome.types import Expression, SafeExpType @@ -157,17 +152,6 @@ def get_refreshed_widgets() -> set: return _get_data(KEY_REFRESHED_WIDGETS, set()) -class StaticCastExpression(Expression): - __slots__ = ("type", "exp") - - def __init__(self, type: Any, exp: SafeExpType): - self.type = str(type) - self.exp = cg.safe_exp(exp) - - def __str__(self): - return f"static_cast<{self.type}>({self.exp})" - - def add_define(macro: str, value="1"): lv_defines = get_defines() value = str(value) @@ -192,31 +176,6 @@ def addr(arg) -> MockObj: return MockObj(f"&{arg}") -def call_lambda(lamb: LambdaExpression) -> Expression: - """ - Given a lambda, either reduce to a simple expression or call it, possibly with parameters - from the surrounding context - :param lamb: - :return: - """ - expr = lamb.content.strip() - if expr.startswith("return") and expr.endswith(";"): - # Convert a lambda returning a simple expression to just that expression - expr = cg.RawExpression(expr[6:-1].strip()) - # Don't cast if the return type is a class - if isinstance(lamb.return_type, MockObjClass): - return expr - return StaticCastExpression(lamb.return_type, expr) - # If lambda has parameters, call it with their names - # Parameter names come from hardcoded component code (like "x", "it", "event") - # not from user input, so they're safe to use directly - if lamb.parameters and lamb.parameters.parameters: - return CallExpression( - lamb, *[MockObj(x.id) for x in lamb.parameters.parameters] - ) - return CallExpression(lamb) - - class LValidator: """ A validator for a particular type used in LVGL. Usable in configs as a validator, also diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index 42352b9602..6f86e49e51 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -16,7 +16,7 @@ from esphome.const import ( CONF_VALUE, ) from esphome.core import CORE, ID, Lambda -from esphome.cpp_generator import MockObj +from esphome.cpp_generator import MockObj, StaticCastExpression, call_lambda from esphome.cpp_types import ESPTime, int32, uint32 from esphome.helpers import cpp_string_escape from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -33,9 +33,7 @@ from .defines import ( LV_FONTS, LValidator, LvConstant, - StaticCastExpression, add_lv_use, - call_lambda, get_esphome_fonts_used, get_lv_fonts_used, get_lv_images_used, diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index c9099e3c3a..a524fe761f 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -16,7 +16,7 @@ from esphome.const import ( ) from esphome.core import ID, EsphomeError, TimePeriod from esphome.coroutine import FakeAwaitable -from esphome.cpp_generator import MockObj +from esphome.cpp_generator import MockObj, call_lambda from esphome.schema_extractors import EnableSchemaExtraction from esphome.types import Expression @@ -42,7 +42,6 @@ from ..defines import ( STATES, LValidator, add_lv_use, - call_lambda, get_styles_used, get_theme_widget_map, get_widget_map, diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index e6b8c0de42..173002438a 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -1187,3 +1187,42 @@ class MockObjClass(MockObj): def __repr__(self): return f"MockObjClass<{str(self.base)}, parents={self._parents}>" + + +class StaticCastExpression(Expression): + __slots__ = ("type", "exp") + + def __init__(self, type: Any, exp: SafeExpType): + self.type = str(type) + self.exp = safe_exp(exp) + + def __str__(self): + return f"static_cast<{self.type}>({self.exp})" + + +def call_lambda(lamb: LambdaExpression) -> Expression: + """ + Given a lambda, either reduce to a simple expression or call it, possibly with parameters + from the surrounding context. + This is for use only with value-returning lambdas, used in places where the value of a lambda call is needed. + :param lamb: The LambdaExpression to call or reduce + :return: An Expression representing the result of calling the lambda or reducing it to a simple expression + """ + # Developer error if this is called with a lambda that doesn't have a return type + assert lamb.return_type is not None, "Lambda must have a return type to be called" + expr = lamb.content.strip() + if re.match(r"^return\b", expr) and expr.endswith(";"): + # Convert a lambda returning a simple expression to just that expression + expr = RawExpression(expr[6:-1].strip()) + # Don't cast if the return type is a class + if isinstance(lamb.return_type, MockObjClass): + return expr + return StaticCastExpression(lamb.return_type, expr) + # If lambda has parameters, call it with their names + # Parameter names come from hardcoded component code (like "x", "it", "event") + # not from user input, so they're safe to use directly + if lamb.parameters and lamb.parameters.parameters: + return CallExpression( + lamb, *[MockObj(x.id) for x in lamb.parameters.parameters] + ) + return CallExpression(lamb) diff --git a/tests/unit_tests/test_cpp_generator.py b/tests/unit_tests/test_cpp_generator.py index 81ae586e23..052513ce97 100644 --- a/tests/unit_tests/test_cpp_generator.py +++ b/tests/unit_tests/test_cpp_generator.py @@ -85,6 +85,15 @@ class TestCallExpression: assert actual == 'my_function(1, "2", false)' +class TestStaticCastExpression: + def test_str(self): + target = cg.StaticCastExpression(ct.bool_, 42) + + actual = str(target) + + assert actual == "static_cast(42)" + + class TestStructInitializer: def test_str(self): target = cg.StructInitializer( @@ -229,6 +238,76 @@ class TestLambdaExpression: ) +class TestCallLambda: + """Tests for the call_lambda() function.""" + + def test_call_lambda__return_expression_casts_to_return_type(self): + """A lambda body that is just a return statement reduces to the + expression, cast to the lambda's return type.""" + lamb = cg.LambdaExpression(("return foo + 1;",), (), "", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.StaticCastExpression) + assert str(result) == "static_cast(foo + 1)" + + def test_call_lambda__return_expression_with_class_return_type_no_cast(self): + """A class return type is not cast, since static_cast doesn't apply + to arbitrary class types.""" + mock_class = cg.MockObjClass("foo::Bar", parents=()) + lamb = cg.LambdaExpression(("return get_bar();",), (), "", mock_class) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.RawExpression) + assert str(result) == "get_bar()" + + def test_call_lambda__no_return_with_parameters_calls_with_names(self): + """A multi-statement lambda with parameters is called with the + parameter names as arguments.""" + lamb = cg.LambdaExpression( + ("do_something(x, y);",), ((int, "x"), (float, "y")), "=", ct.bool_ + ) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == ( + "[=](int32_t x, float y) -> bool {\n do_something(x, y);\n}(x, y)" + ) + + def test_call_lambda__no_return_type_raises(self): + """Calling a lambda with no declared return type is a developer + error: call_lambda is only for value-returning lambdas.""" + lamb = cg.LambdaExpression(("do_something();",), (), "=") + + with pytest.raises(AssertionError): + cg.call_lambda(lamb) + + def test_call_lambda__identifier_starting_with_return_is_not_a_return_statement( + self, + ): + """A body that merely starts with the substring "return" (e.g. a call + to a function named returnValue()) must not be mistaken for a return + statement -- the match requires a word boundary after "return".""" + lamb = cg.LambdaExpression(("returnValue();",), (), "=", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == "[=]() -> bool {\n returnValue();\n}()" + + def test_call_lambda__no_return_no_parameters_calls_with_no_args(self): + """A multi-statement lambda without parameters is called with no + arguments.""" + lamb = cg.LambdaExpression(("do_something();",), (), "", ct.bool_) + + result = cg.call_lambda(lamb) + + assert isinstance(result, cg.CallExpression) + assert str(result) == "[]() -> bool {\n do_something();\n}()" + + class TestLiterals: @pytest.mark.parametrize( "target, expected", From ebb9037ea1bf802334299b7c38b26dac352d028a Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Fri, 11 Sep 2026 22:08:23 -0500 Subject: [PATCH 55/55] [bridge] New component and `cdc_acm_uart` platform (#11689) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 Co-authored-by: J. Nick Koston --- CODEOWNERS | 3 + esphome/components/bridge/__init__.py | 4 + esphome/components/cdc_acm_uart/__init__.py | 1 + .../cdc_acm_uart/bridge/__init__.py | 114 +++++ .../bridge/cdc_acm_uart_bridge.cpp | 468 ++++++++++++++++++ .../cdc_acm_uart/bridge/cdc_acm_uart_bridge.h | 117 +++++ esphome/components/usb_cdc_acm/usb_cdc_acm.h | 34 ++ .../usb_cdc_acm/usb_cdc_acm_esp32.cpp | 26 +- script/analyze_component_buses.py | 1 + .../component_tests/cdc_acm_uart/__init__.py | 0 .../component_tests/cdc_acm_uart/test_init.py | 154 ++++++ tests/component_tests/conftest.py | 11 +- tests/component_tests/types.py | 3 +- tests/components/cdc_acm_uart/common.yaml | 18 + .../components/cdc_acm_uart/common_dual.yaml | 12 + .../cdc_acm_uart/test.esp32-p4-idf.yaml | 15 + .../cdc_acm_uart/test.esp32-s2-idf.yaml | 14 + .../cdc_acm_uart/test.esp32-s3-idf.yaml | 17 + 18 files changed, 983 insertions(+), 29 deletions(-) create mode 100644 esphome/components/bridge/__init__.py create mode 100644 esphome/components/cdc_acm_uart/__init__.py create mode 100644 esphome/components/cdc_acm_uart/bridge/__init__.py create mode 100644 esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp create mode 100644 esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h create mode 100644 tests/component_tests/cdc_acm_uart/__init__.py create mode 100644 tests/component_tests/cdc_acm_uart/test_init.py create mode 100644 tests/components/cdc_acm_uart/common.yaml create mode 100644 tests/components/cdc_acm_uart/common_dual.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml create mode 100644 tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index f91bc00ae5..246a210c7c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -100,6 +100,7 @@ esphome/components/bmp581_i2c/* @danielkent-net @kahrendt esphome/components/bmp581_spi/* @danielkent-net @kahrendt esphome/components/bp1658cj/* @Cossid esphome/components/bp5758d/* @Cossid +esphome/components/bridge/* @kbx81 esphome/components/bthome_mithermometer/* @nagyrobi esphome/components/button/* @esphome/core esphome/components/bytebuffer/* @clydebarrow @@ -111,6 +112,8 @@ esphome/components/captive_portal/* @esphome/core esphome/components/cc1101/* @gabest11 @lygris esphome/components/ccs811/* @habbie esphome/components/cd74hc4067/* @asoehlke +esphome/components/cdc_acm_uart/* @kbx81 +esphome/components/cdc_acm_uart/bridge/* @kbx81 esphome/components/ch422g/* @clydebarrow @jesterret esphome/components/ch423/* @dwmw2 esphome/components/chsc6x/* @kkosik20 diff --git a/esphome/components/bridge/__init__.py b/esphome/components/bridge/__init__.py new file mode 100644 index 0000000000..49811b0181 --- /dev/null +++ b/esphome/components/bridge/__init__.py @@ -0,0 +1,4 @@ +CODEOWNERS = ["@kbx81"] +DOMAIN = "bridge" + +IS_PLATFORM_COMPONENT = True diff --git a/esphome/components/cdc_acm_uart/__init__.py b/esphome/components/cdc_acm_uart/__init__.py new file mode 100644 index 0000000000..516af84856 --- /dev/null +++ b/esphome/components/cdc_acm_uart/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@kbx81"] diff --git a/esphome/components/cdc_acm_uart/bridge/__init__.py b/esphome/components/cdc_acm_uart/bridge/__init__.py new file mode 100644 index 0000000000..cee048df5d --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/__init__.py @@ -0,0 +1,114 @@ +from esphome import pins +import esphome.codegen as cg +from esphome.components import esp32, uart, usb_cdc_acm +from esphome.components.bridge import DOMAIN as BRIDGE_DOMAIN +from esphome.components.esp32 import VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3 +import esphome.config_validation as cv +from esphome.const import CONF_DEBUG, CONF_ID, CONF_UART_ID +import esphome.final_validate as fv +from esphome.types import ConfigType + +CODEOWNERS = ["@kbx81"] +DEPENDENCIES = ["tinyusb", "uart", "usb_cdc_acm"] + +CONF_DTR_PIN = "dtr_pin" +CONF_RTS_PIN = "rts_pin" +CONF_USB_CDC_ACM_ID = "usb_cdc_acm_id" + +cdc_acm_uart_ns = cg.esphome_ns.namespace("cdc_acm_uart") +CDCACMUARTBridge = cdc_acm_uart_ns.class_("CDCACMUARTBridge", cg.Component) + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(CDCACMUARTBridge), + cv.Required(CONF_UART_ID): cv.use_id(uart.IDFUARTComponent), + cv.Required(CONF_USB_CDC_ACM_ID): cv.use_id(usb_cdc_acm.USBCDCACMInstance), + cv.Optional(CONF_DTR_PIN): pins.gpio_output_pin_schema, + cv.Optional(CONF_RTS_PIN): pins.gpio_output_pin_schema, + } + ).extend(cv.COMPONENT_SCHEMA), + # Narrower than usb_cdc_acm's variant list on purpose: S31/H4 untested on + # hardware; extend once verified. + esp32.only_on_variant( + supported=[VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3], + ), +) + + +def _subtree_references_uart(node: object, uart_id: str) -> bool: + """Return True if any dict in the subtree has a uart_id entry naming this bus.""" + if isinstance(node, dict): + return any( + (key == CONF_UART_ID and str(value) == uart_id) + or _subtree_references_uart(value, uart_id) + for key, value in node.items() + ) + if isinstance(node, list): + return any(_subtree_references_uart(item, uart_id) for item in node) + return False + + +def _reject_debug(uart_conf: ConfigType) -> ConfigType: + # The worker tasks use the IDF driver directly, so the uart debugger never sees + # bridge traffic and its dummy_receiver would drain RX bytes on the main loop. + if CONF_DEBUG in uart_conf: + raise cv.Invalid( + "A bridged UART cannot use 'debug'; the bridge bypasses the UART " + "component's read/write path.", + [CONF_DEBUG], + ) + return uart_conf + + +def _final_validate(config: ConfigType) -> ConfigType: + full_config = fv.full_config.get() + # Bridges of any platform must own their interfaces exclusively; shared ring + # buffers and overwritten callbacks would corrupt both streams silently. The + # seen-set is keyed on the bridge domain so future platforms share it. + # Other components bind either interface through the same uart_id key (the CDC + # instance is itself a uart::UARTComponent) and would race the worker tasks. + # Bare `id:` references (a uart.write action) cannot be distinguished; not caught. + data = full_config.data.setdefault(BRIDGE_DOMAIN, {}) + for conf_key, label in ( + (CONF_UART_ID, "UART"), + (CONF_USB_CDC_ACM_ID, "USB CDC-ACM interface"), + ): + owned_id = str(config[conf_key]) + used = data.setdefault(conf_key, set()) + if owned_id in used: + raise cv.Invalid( + f"The {label} '{owned_id}' is already bridged by another 'bridge' " + f"instance; each bridge requires its own {label}.", + [conf_key], + ) + used.add(owned_id) + for domain, domain_conf in full_config.items(): + if domain == BRIDGE_DOMAIN: + continue + if _subtree_references_uart(domain_conf, owned_id): + raise cv.Invalid( + f"The {label} '{owned_id}' is also used by '{domain}'; a bridge " + f"requires exclusive use of its {label}.", + [conf_key], + ) + + fv.id_declaration_match_schema(_reject_debug)(config[CONF_UART_ID]) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + uart_component = await cg.get_variable(config[CONF_UART_ID]) + usb_cdc = await cg.get_variable(config[CONF_USB_CDC_ACM_ID]) + var = cg.new_Pvariable(config[CONF_ID], uart_component, usb_cdc) + await cg.register_component(var, config) + + if dtr_pin_config := config.get(CONF_DTR_PIN): + dtr_pin = await cg.gpio_pin_expression(dtr_pin_config) + cg.add(var.set_dtr_pin(dtr_pin)) + if rts_pin_config := config.get(CONF_RTS_PIN): + rts_pin = await cg.gpio_pin_expression(rts_pin_config) + cg.add(var.set_rts_pin(rts_pin)) diff --git a/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp new file mode 100644 index 0000000000..042688bfe6 --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.cpp @@ -0,0 +1,468 @@ +#if defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) +#include "cdc_acm_uart_bridge.h" +#include "esphome/core/application.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/ringbuf.h" +#include "driver/uart.h" +#include "soc/soc_caps.h" + +namespace esphome::cdc_acm_uart { + +static const char *const TAG = "cdc_acm_uart"; + +static constexpr size_t UART_TASK_STACK_SIZE = 4096; +static constexpr size_t RINGBUF_RETRY_CHUNK_SIZE = 64; +static constexpr uint32_t LOG_THROTTLE_MS = 1000; +static constexpr uint32_t UART_RELOAD_SETTLE_MS = 20; +// Above the default priority but below the USB/Wi-Fi system tasks. +static constexpr UBaseType_t TASK_PRIORITY = 4; + +static bool should_log_now(uint32_t *last_ms, uint32_t interval_ms) { + uint32_t now = millis(); + if ((now - *last_ms) >= interval_ms) { + *last_ms = now; + return true; + } + return false; +} + +static bool ringbuf_send_with_retry(RingbufHandle_t ringbuf, const uint8_t *data, size_t len, uint32_t *log_ms) { + if (len == 0) { + return true; + } + + if (xRingbufferSend(ringbuf, data, len, pdMS_TO_TICKS(1)) == pdTRUE) { + return true; + } + + size_t offset = 0; + while (offset < len) { + size_t chunk = std::min(RINGBUF_RETRY_CHUNK_SIZE, len - offset); + if (xRingbufferSend(ringbuf, data + offset, chunk, pdMS_TO_TICKS(1)) != pdTRUE) { + if (should_log_now(log_ms, LOG_THROTTLE_MS)) { + ESP_LOGW(TAG, "USB TX buffer full; some data is lost"); + } + return false; + } + offset += chunk; + } + return true; +} + +void CDCACMUARTBridge::setup() { + // Line state starts deasserted (no host yet); active-low DTR#/RTS# wiring is + // handled by configuring the pins inverted, so deasserted idles HIGH. + if (this->dtr_pin_ != nullptr) { + this->dtr_pin_->setup(); + this->dtr_pin_->digital_write(false); + } + + if (this->rts_pin_ != nullptr) { + this->rts_pin_->setup(); + this->rts_pin_->digital_write(false); + } + + // A failed UART never assigned its port number, so the worker tasks would run + // against an indeterminate port. + if (this->uart_parent_->is_failed()) { + ESP_LOGE(TAG, "UART parent failed; aborting"); + this->mark_failed(); + return; + } + + this->configured_baud_rate_ = this->uart_parent_->get_baud_rate(); + this->configured_parity_ = this->uart_parent_->get_parity(); + this->configured_stop_bits_ = this->uart_parent_->get_stop_bits(); + this->configured_data_bits_ = this->uart_parent_->get_data_bits(); + + // usb_cdc_acm sets up first (priority IO > HARDWARE). Any interface failing marks + // the hub failed, and a failed hub no longer runs loop(), so line coding and line + // state events would never reach this bridge even if its own interface is healthy. + if (this->usb_cdc_parent_->get_parent()->is_failed()) { + ESP_LOGE(TAG, "USB CDC ACM failed; aborting"); + this->mark_failed(); + return; + } + + // Per-instance task names (keyed on the CDC interface number) keep task dumps + // unambiguous with multiple bridges. + char tx_task_name[] = "cdc_uart_tx_0"; + char rx_task_name[] = "cdc_uart_rx_0"; + const char itf_char = format_hex_char(this->usb_cdc_parent_->get_itf()); + tx_task_name[sizeof(tx_task_name) - 2] = itf_char; + rx_task_name[sizeof(rx_task_name) - 2] = itf_char; + + xTaskCreate(uart_tx_task_fn, tx_task_name, UART_TASK_STACK_SIZE, this, TASK_PRIORITY, &this->uart_tx_task_handle_); + if (this->uart_tx_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Failed to create UART TX task"); + this->mark_failed(); + return; + } + + xTaskCreate(uart_rx_task_fn, rx_task_name, UART_TASK_STACK_SIZE, this, TASK_PRIORITY, &this->uart_rx_task_handle_); + if (this->uart_rx_task_handle_ == nullptr) { + ESP_LOGE(TAG, "Failed to create UART RX task"); + vTaskDelete(this->uart_tx_task_handle_); + this->uart_tx_task_handle_ = nullptr; + this->mark_failed(); + return; + } + + // Only register callbacks once both tasks exist, so a failed setup never drives + // DTR/RTS from a dead bridge. + this->usb_cdc_parent_->set_line_state_callback([this](bool dtr, bool rts) { this->set_line_state(dtr, rts); }); + this->usb_cdc_parent_->set_line_coding_callback([this](uint32_t, uint8_t, uint8_t, uint8_t) { + this->host_coding_seen_ = true; + // Another component owns the UART's framing while paused; resume() re-syncs. + if (this->paused_ == 0) { + this->set_line_coding(); + } + }); + + // Release the workers only now: until here a failed setup may still delete the TX + // task, which is safe only while it is parked and owns nothing in the driver. + xTaskNotifyGive(this->uart_tx_task_handle_); + xTaskNotifyGive(this->uart_rx_task_handle_); + + // loop() only services line-coding reloads; stay off the main loop until one is + // scheduled. + this->disable_loop(); +} + +void CDCACMUARTBridge::dump_config() { + ESP_LOGCONFIG(TAG, + "CDC-ACM UART Bridge:\n" + " UART Bus: %u\n" + " USB CDC Interface: %u", + this->uart_parent_->get_hw_serial_number(), this->usb_cdc_parent_->get_itf()); + LOG_PIN(" DTR Pin: ", this->dtr_pin_); + LOG_PIN(" RTS Pin: ", this->rts_pin_); +} + +void CDCACMUARTBridge::on_shutdown() { + // The UART (BUS) shuts down after this component (HARDWARE) and deletes its driver, + // freeing the ring buffer and mutexes the worker tasks block on. Suspending the + // tasks unlinks them from those objects first. + if (this->uart_rx_task_handle_ != nullptr) { + vTaskSuspend(this->uart_rx_task_handle_); + } + if (this->uart_tx_task_handle_ != nullptr) { + vTaskSuspend(this->uart_tx_task_handle_); + } +} + +void CDCACMUARTBridge::loop() { + switch (this->state_) { + case MainState::MAIN_STATE_RELOAD_PENDING: + if ((App.get_loop_component_start_time() - this->reload_requested_at_) < UART_RELOAD_SETTLE_MS) { + return; + } + // Deliberately not gated on tx_idle_(): a host that re-codes the line mid-stream + // wants the new framing now, and its own in-flight bytes are its concern. + // apply_settings_live() rewrites the framing registers without reinstalling the + // driver, so the worker tasks blocked inside it are undisturbed. + this->uart_parent_->apply_settings_live(); + this->state_ = MainState::MAIN_STATE_RUNNING; + break; + case MainState::MAIN_STATE_PAUSING: + case MainState::MAIN_STATE_RESUMING: + // Let a host write that was in flight drain, FIFO included, before a reload + // flushes the FIFOs and truncates it. + if (!this->tx_idle_()) { + return; + } + if (this->state_ == MainState::MAIN_STATE_PAUSING) { + this->restore_configured_framing_(); + this->state_ = MainState::MAIN_STATE_PAUSED; + } else { + this->finish_resume_(); + } + break; + default: + break; + } + this->disable_loop(); +} + +void CDCACMUARTBridge::set_line_coding() { + if (!this->sync_host_framing_()) { + return; + } + // Coalesce rapid line-coding updates from the host. + this->reload_requested_at_ = App.get_loop_component_start_time(); + this->state_ = MainState::MAIN_STATE_RELOAD_PENDING; + // Main-loop context (via USBCDCACMInstance::process_events_). + this->enable_loop(); +} + +bool CDCACMUARTBridge::sync_host_framing_() { + // usb_cdc_acm has already translated the wire coding onto the CDC instance (main + // loop); mirror it here so the framing translation has a single source of truth. + bool changed = false; + + // Reject 0 (the CDC B0/hang-up encoding; older IDF revisions divide by the rate) + // and rates above the SoC ceiling. Anything in between is the driver's call, + // matching what a YAML-configured UART accepts. + const uint32_t baud = this->usb_cdc_parent_->get_baud_rate(); + if (baud == 0 || baud > SOC_UART_BITRATE_MAX) { + ESP_LOGW(TAG, "Ignoring unsupported baud rate %" PRIu32 " from host; keeping %" PRIu32, baud, + this->uart_parent_->get_baud_rate()); + } else if (this->uart_parent_->get_baud_rate() != baud) { + this->uart_parent_->set_baud_rate(baud); + changed = true; + } + + const uint8_t stop_bits = this->usb_cdc_parent_->get_stop_bits(); + if (this->uart_parent_->get_stop_bits() != stop_bits) { + this->uart_parent_->set_stop_bits(stop_bits); + changed = true; + } + + const auto parity = this->usb_cdc_parent_->get_parity(); + if (this->uart_parent_->get_parity() != parity) { + this->uart_parent_->set_parity(parity); + changed = true; + } + + // USB CDC permits data-bit counts the UART cannot represent (up to 16). + const uint8_t data_bits = this->usb_cdc_parent_->get_data_bits(); + if (data_bits < 5 || data_bits > 8) { + ESP_LOGW(TAG, "Ignoring unsupported data bits %u from host; keeping %u", data_bits, + this->uart_parent_->get_data_bits()); + } else if (this->uart_parent_->get_data_bits() != data_bits) { + this->uart_parent_->set_data_bits(data_bits); + changed = true; + } + + if (changed) { + ESP_LOGV(TAG, "Line coding: baud=%" PRIu32 ", data_bits=%u, stop_bits=%u, parity=%u", + this->uart_parent_->get_baud_rate(), this->uart_parent_->get_data_bits(), + this->uart_parent_->get_stop_bits(), static_cast(this->uart_parent_->get_parity())); + } + return changed; +} + +void CDCACMUARTBridge::pause() { + if (this->state_ == MainState::MAIN_STATE_PAUSING || this->state_ == MainState::MAIN_STATE_PAUSED) { + return; + } + this->paused_ = 1; + // A null RX task means setup() has not completed (or failed): nothing to stop, and + // the framing snapshot does not exist yet. Should setup() run later, the RX task + // starts parked. + if (this->uart_rx_task_handle_ == nullptr) { + this->state_ = MainState::MAIN_STATE_PAUSED; + return; + } + // Drops a coalesced host reload or a pending resume; loop() restores the framing + // once any host write in flight has drained. + this->state_ = MainState::MAIN_STATE_PAUSING; + this->enable_loop(); +} + +void CDCACMUARTBridge::resume() { + if (this->state_ != MainState::MAIN_STATE_PAUSING && this->state_ != MainState::MAIN_STATE_PAUSED) { + return; + } + if (this->uart_rx_task_handle_ == nullptr) { + this->paused_ = 0; + this->state_ = MainState::MAIN_STATE_RUNNING; + return; + } + // A restore still waiting on the TX side is moot: the host's framing is kept. + if (!this->tx_idle_()) { + this->state_ = MainState::MAIN_STATE_RESUMING; + this->enable_loop(); + return; + } + this->finish_resume_(); + this->disable_loop(); +} + +void CDCACMUARTBridge::finish_resume_() { + // Take the bus back at a known framing before either task runs again: the host's + // if it ever sent one, else the YAML framing (the other owner may have changed it). + if (this->host_coding_seen_) { + this->sync_host_framing_(); + this->uart_parent_->apply_settings_live(); + } else { + this->restore_configured_framing_(); + } + this->paused_ = 0; + this->state_ = MainState::MAIN_STATE_RUNNING; + this->drive_line_state_(); + xTaskNotifyGive(this->uart_rx_task_handle_); +} + +bool CDCACMUARTBridge::tx_idle_() { + const auto uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + return this->tx_busy_ == 0 && uart_wait_tx_done(uart_num, 0) == ESP_OK; +} + +void CDCACMUARTBridge::restore_configured_framing_() { + // Always applied: the cached settings can lead the hardware by a pending reload, + // so they are no proof of what is live. + this->uart_parent_->set_baud_rate(this->configured_baud_rate_); + this->uart_parent_->set_parity(this->configured_parity_); + this->uart_parent_->set_stop_bits(this->configured_stop_bits_); + this->uart_parent_->set_data_bits(this->configured_data_bits_); + this->uart_parent_->apply_settings_live(); +} + +void CDCACMUARTBridge::set_line_state(bool dtr, bool rts) { + ESP_LOGV(TAG, "Line state: DTR=%d, RTS=%d", dtr, rts); + this->host_dtr_ = dtr; + this->host_rts_ = rts; + // Frozen while paused: a host opening the port must not reset a peer that another + // component is talking to. + if (this->paused_ == 0) { + this->drive_line_state_(); + } +} + +void CDCACMUARTBridge::drive_line_state_() { + if (this->dtr_pin_ != nullptr) { + this->dtr_pin_->digital_write(this->host_dtr_); + } + if (this->rts_pin_ != nullptr) { + this->rts_pin_->digital_write(this->host_rts_); + } +} + +void CDCACMUARTBridge::uart_rx_task_fn(void *arg) { + auto *bridge = static_cast(arg); + bridge->uart_rx_task_(); +} + +void CDCACMUARTBridge::uart_tx_task_fn(void *arg) { + auto *bridge = static_cast(arg); + bridge->uart_tx_task_(); +} + +void CDCACMUARTBridge::uart_rx_task_() { + TaskHandle_t usb_tx_handle = this->usb_cdc_parent_->get_tx_task_handle(); + RingbufHandle_t usb_tx_ringbuf = this->usb_cdc_parent_->get_tx_ringbuf(); + uart_port_t uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + // Back-dated so a problem within the first LOG_THROTTLE_MS of uptime still logs. + uint32_t tx_full_log_ms = millis() - LOG_THROTTLE_MS; + uint32_t err_log_ms = millis() - LOG_THROTTLE_MS; + + uint8_t *data = this->uart_rx_buffer_.data(); + const size_t buf_size = this->uart_rx_buffer_.size(); + + // Released by setup() once both tasks exist. + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + + while (true) { + if (this->paused_ != 0) { + // Parked until resume() notifies; nothing is read, so the other owner sees + // every byte. + this->rx_parked_ = 1; + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + this->rx_parked_ = 0; + continue; + } + + // Block until at least one byte is available from UART. + int total_rx_size = uart_read_bytes(uart_num, data, 1, pdMS_TO_TICKS(UART_RX_WAIT_MS)); + if (total_rx_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART read failed: %d", total_rx_size); + } + vTaskDelay(pdMS_TO_TICKS(10)); + continue; + } + if (total_rx_size == 0) { + continue; + } + // pause() landed during the read: don't forward a byte to a host that is gone. + if (this->paused_ != 0) { + continue; + } + + // Drain the currently buffered burst without waiting. + while (true) { + int rx_data_size = uart_read_bytes(uart_num, data + total_rx_size, buf_size - total_rx_size, 0); + if (rx_data_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART read failed: %d", rx_data_size); + } + break; + } + if (rx_data_size == 0) { + break; + } + ESP_LOGV(TAG, "UART RX: %d bytes", rx_data_size); + total_rx_size += rx_data_size; + if (total_rx_size >= (int) buf_size) { + break; + } + } + + ringbuf_send_with_retry(usb_tx_ringbuf, data, total_rx_size, &tx_full_log_ms); + + ESP_LOGV(TAG, "UART RX: waking up USB TX task"); + xTaskNotifyGive(usb_tx_handle); + } +} + +void CDCACMUARTBridge::uart_tx_task_() { + RingbufHandle_t usb_rx_ringbuf = this->usb_cdc_parent_->get_rx_ringbuf(); + uart_port_t uart_num = static_cast(this->uart_parent_->get_hw_serial_number()); + uint8_t *data_to_uart = this->uart_tx_buffer_.data(); + const size_t buf_size = this->uart_tx_buffer_.size(); + size_t rx_size; + // Back-dated so a problem within the first LOG_THROTTLE_MS of uptime still logs. + uint32_t err_log_ms = millis() - LOG_THROTTLE_MS; + uint32_t drop_log_ms = millis() - LOG_THROTTLE_MS; + + // Released by setup() once both tasks exist. + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + + while (true) { + ESP_LOGV(TAG, "Waiting for data to send to UART"); + esp_err_t ret = usb_cdc_acm::ringbuf_read_bytes(usb_rx_ringbuf, data_to_uart, buf_size, &rx_size, portMAX_DELAY); + + if (ret != ESP_OK) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "USB RX RingBuf read failed"); + } + // Yield: this task runs above the main loop, so a persistent failure must not + // become a tight loop. + vTaskDelay(pdMS_TO_TICKS(10)); + continue; + } + + // Another component owns the UART; host bytes must not interleave with its traffic. + // tx_busy_ goes up before the check so is_paused() cannot miss a write in flight. + this->tx_busy_ = 1; + if (this->paused_ != 0) { + this->tx_busy_ = 0; + if (should_log_now(&drop_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGW(TAG, "Paused; dropping %zu bytes from host", rx_size); + } + continue; + } + + ESP_LOGV(TAG, "Sending %zu bytes to UART", rx_size); + // Signed: uart_write_bytes() returns -1 on error. + int xfer_size = uart_write_bytes(uart_num, data_to_uart, rx_size); + this->tx_busy_ = 0; + + if (xfer_size < 0) { + if (should_log_now(&err_log_ms, LOG_THROTTLE_MS)) { + ESP_LOGE(TAG, "UART write failed: %d", xfer_size); + } + } else if (static_cast(xfer_size) != rx_size) { + ESP_LOGW(TAG, "UART write incomplete (%d/%zu bytes)", xfer_size, rx_size); + } + } +} + +} // namespace esphome::cdc_acm_uart +#endif diff --git a/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h new file mode 100644 index 0000000000..64522c86bd --- /dev/null +++ b/esphome/components/cdc_acm_uart/bridge/cdc_acm_uart_bridge.h @@ -0,0 +1,117 @@ +#pragma once +#if defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) +#include "esphome/components/uart/uart_component_esp_idf.h" +#include "esphome/components/usb_cdc_acm/usb_cdc_acm.h" +#include "esphome/core/component.h" + +#include +#include +#include "sdkconfig.h" + +namespace esphome::cdc_acm_uart { + +class CDCACMUARTBridge final : public Component { + public: + // Upper bound on the RX task's blocking read, so pause() takes effect without + // aborting the read. Arriving bytes still unblock it immediately. + static constexpr uint32_t UART_RX_WAIT_MS = 250; + + CDCACMUARTBridge(uart::IDFUARTComponent *uart_parent, usb_cdc_acm::USBCDCACMInstance *usb_cdc_parent) + : uart_parent_(uart_parent), usb_cdc_parent_(usb_cdc_parent) {} + + void setup() override; + void loop() override; + void dump_config() override; + void on_shutdown() override; + float get_setup_priority() const override { return setup_priority::HARDWARE; } + + void set_dtr_pin(GPIOPin *dtr_pin) { this->dtr_pin_ = dtr_pin; } + void set_rts_pin(GPIOPin *rts_pin) { this->rts_pin_ = rts_pin; } + + void set_line_coding(); + void set_line_state(bool dtr, bool rts); + + /** + * Stop forwarding in both directions and hand the UART back to its configured + * framing, so another component may use the bus. Main-loop only. The RX task parks + * within UART_RX_WAIT_MS (a byte it was already reading is discarded). A host write + * already in flight is allowed to drain first, which at low baud rates can take + * seconds; the framing is restored only after that, so poll is_paused() rather than + * waiting a fixed interval. Host bytes not yet written to the UART are discarded. + * The DTR/RTS outputs hold their state while paused and follow the host again on + * resume(). + */ + void pause(); + /** + * Re-apply the host's line coding and line state, then resume forwarding. Main-loop + * only. Deferred until any host write still draining has finished, so the reload + * never truncates it. + */ + void resume(); + /// True once both worker tasks are off the bus and the configured framing is restored. + /// With no RX task (setup() failed or has not run) there is nothing to wait for. + bool is_paused() const { + return this->state_ == MainState::MAIN_STATE_PAUSED && + (this->uart_rx_task_handle_ == nullptr || this->rx_parked_ != 0); + } + + protected: + static void uart_rx_task_fn(void *arg); + static void uart_tx_task_fn(void *arg); + void uart_rx_task_(); + void uart_tx_task_(); + void restore_configured_framing_(); + // True when the TX task has no write in flight and the UART TX FIFO has drained. + bool tx_idle_(); + void finish_resume_(); + void drive_line_state_(); + // Copy the host's line coding onto the UART settings; true if anything changed. + bool sync_host_framing_(); + + TaskHandle_t uart_rx_task_handle_{nullptr}; + TaskHandle_t uart_tx_task_handle_{nullptr}; + + GPIOPin *dtr_pin_{nullptr}; + GPIOPin *rts_pin_{nullptr}; + + uint32_t reload_requested_at_{0}; + + // Worker staging, each sized to the CDC ring buffer it feeds or drains. + std::array uart_rx_buffer_{}; + std::array uart_tx_buffer_{}; + + uart::IDFUARTComponent *uart_parent_; + usb_cdc_acm::USBCDCACMInstance *usb_cdc_parent_; + + // YAML framing, captured at setup; the host's line coding overwrites the UART's + // settings, so pause() needs the original to restore. + uint32_t configured_baud_rate_{0}; + uart::UARTParityOptions configured_parity_{uart::UART_CONFIG_PARITY_NONE}; + uint8_t configured_stop_bits_{0}; + uint8_t configured_data_bits_{0}; + + // Written on the main loop, read by both worker tasks. uint8_t rather than bool: + // GCC on Xtensa emits an out-of-line call for atomic. + std::atomic paused_{0}; + // Raised by the RX task while parked and by the TX task around each UART write, so + // the pause hand-off knows when the bus is actually free. + std::atomic rx_parked_{0}; + std::atomic tx_busy_{0}; + // Main-loop state; paused_ mirrors it for the worker tasks. + enum class MainState : uint8_t { + MAIN_STATE_RUNNING, + MAIN_STATE_RELOAD_PENDING, // host line coding debounced, forwarding continues + MAIN_STATE_PAUSING, // waiting for TX idle to restore the configured framing + MAIN_STATE_PAUSED, + MAIN_STATE_RESUMING, // resume() requested while a host write still drains + }; + MainState state_{MainState::MAIN_STATE_RUNNING}; + // Host line state, recorded even while paused so resume() can re-drive the pins. + bool host_dtr_{false}; + bool host_rts_{false}; + // True once the host has sent any line coding; resume() then re-syncs to it. + bool host_coding_seen_{false}; +}; + +} // namespace esphome::cdc_acm_uart +#endif diff --git a/esphome/components/usb_cdc_acm/usb_cdc_acm.h b/esphome/components/usb_cdc_acm/usb_cdc_acm.h index d8eb91586a..83cb5de89f 100644 --- a/esphome/components/usb_cdc_acm/usb_cdc_acm.h +++ b/esphome/components/usb_cdc_acm/usb_cdc_acm.h @@ -7,15 +7,47 @@ #include "esphome/core/lock_free_queue.h" #include "esphome/components/uart/uart_component.h" +#include #include +#include #include #include "freertos/ringbuf.h" +#include "esp_err.h" #include "tinyusb_cdc_acm.h" namespace esphome::usb_cdc_acm { static const uint8_t EVENT_QUEUE_SIZE = 12; +// Drain up to out_buf_sz bytes from a byte ring buffer, handling FreeRTOS's wrapped +// case with a second read. Shared with the cdc_acm_uart bridge platform, whose worker +// tasks drain the same ring buffers. +inline esp_err_t ringbuf_read_bytes(RingbufHandle_t ring_buf, uint8_t *out_buf, size_t out_buf_sz, size_t *rx_data_size, + TickType_t x_ticks_to_wait) { + size_t read_sz; + uint8_t *buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, x_ticks_to_wait, out_buf_sz)); + + if (buf == nullptr) { + return ESP_FAIL; + } + + memcpy(out_buf, buf, read_sz); + vRingbufferReturnItem(ring_buf, (void *) buf); + *rx_data_size = read_sz; + + // Buffer's data can be wrapped, in which case we should perform another read + if (*rx_data_size < out_buf_sz) { + buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, 0, out_buf_sz - *rx_data_size)); + if (buf != nullptr) { + memcpy(out_buf + *rx_data_size, buf, read_sz); + vRingbufferReturnItem(ring_buf, (void *) buf); + *rx_data_size += read_sz; + } + } + + return ESP_OK; +} + // Callback types for line coding and line state changes using LineCodingCallback = std::function; using LineStateCallback = std::function; @@ -103,6 +135,8 @@ class USBCDCACMInstance final : public uart::UARTComponent, public Parented usb_tx_staging_{}; // Non-zero while the TX task holds bytes it has pulled from the ring buffer but not // yet handed to TinyUSB; lets flush() account for data that is in neither the ring // buffer nor TinyUSB's FIFO. diff --git a/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp b/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp index e46369660d..7aa7b46b7b 100644 --- a/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp +++ b/esphome/components/usb_cdc_acm/usb_cdc_acm_esp32.cpp @@ -104,30 +104,6 @@ static void tinyusb_cdc_line_coding_changed_callback(int itf, cdcacm_event_t *ev instance->queue_line_coding_event(bit_rate, stop_bits, parity, data_bits); } -static esp_err_t ringbuf_read_bytes(RingbufHandle_t ring_buf, uint8_t *out_buf, size_t out_buf_sz, size_t *rx_data_size, - TickType_t x_ticks_to_wait) { - size_t read_sz; - uint8_t *buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, x_ticks_to_wait, out_buf_sz)); - - if (buf == nullptr) { - return ESP_FAIL; - } - - memcpy(out_buf, buf, read_sz); - vRingbufferReturnItem(ring_buf, (void *) buf); - *rx_data_size = read_sz; - - // Buffer's data can be wrapped, in which case we should perform another read - buf = static_cast(xRingbufferReceiveUpTo(ring_buf, &read_sz, 0, out_buf_sz - *rx_data_size)); - if (buf != nullptr) { - memcpy(out_buf + *rx_data_size, buf, read_sz); - vRingbufferReturnItem(ring_buf, (void *) buf); - *rx_data_size += read_sz; - } - - return ESP_OK; -} - //============================================================================== // USBCDCACMInstance Implementation //============================================================================== @@ -192,7 +168,7 @@ void USBCDCACMInstance::usb_tx_task_fn(void *arg) { } void USBCDCACMInstance::usb_tx_task() { - uint8_t data[CONFIG_TINYUSB_CDC_TX_BUFSIZE] = {0}; + uint8_t *data = this->usb_tx_staging_.data(); size_t tx_data_size = 0; // Back-dated so a stall within the first LOG_THROTTLE_MS of uptime still logs // immediately (unsigned arithmetic keeps this wrap-safe). diff --git a/script/analyze_component_buses.py b/script/analyze_component_buses.py index b8ee3066bd..b805d5155a 100755 --- a/script/analyze_component_buses.py +++ b/script/analyze_component_buses.py @@ -81,6 +81,7 @@ ISOLATED_SIGNATURE_PREFIX = "isolated_" # NOTE: This should be kept in sync with both test_build_components and split_components_for_ci.py ISOLATED_COMPONENTS = { "animation": "Has display lambda in common.yaml that requires existing display platform - breaks when merged without display", + "cdc_acm_uart": "Depends on tinyusb which conflicts with usb_host", "esphome": "Defines devices/areas in esphome: section that are referenced in other sections - breaks when merged", "ethernet": "Defines ethernet: which conflicts with wifi: used by most components", "ethernet_info": "Related to ethernet component which conflicts with wifi", diff --git a/tests/component_tests/cdc_acm_uart/__init__.py b/tests/component_tests/cdc_acm_uart/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/cdc_acm_uart/test_init.py b/tests/component_tests/cdc_acm_uart/test_init.py new file mode 100644 index 0000000000..7bbf163fc3 --- /dev/null +++ b/tests/component_tests/cdc_acm_uart/test_init.py @@ -0,0 +1,154 @@ +"""Tests for the bridge cdc_acm_uart platform's final validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.cdc_acm_uart import bridge +from esphome.components.cdc_acm_uart.bridge import CONF_USB_CDC_ACM_ID +from esphome.config import Config +from esphome.const import CONF_DEBUG, CONF_ID, CONF_UART_ID, PlatformFramework +from esphome.core import ID +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + +_final_validate = bridge._final_validate + + +def _set_esp32_s3(set_core_config: SetCoreConfigCallable, **kwargs) -> None: + from esphome.components.esp32 import KEY_VARIANT, VARIANT_ESP32S3 + + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_VARIANT: VARIANT_ESP32S3}, + **kwargs, + ) + + +def _full_config(uarts: list[ConfigType] | None = None, **domains) -> Config: + """A full config declaring uart_0 and uart_1 (plus any extra entries), as the ID + pass leaves it, so the debug check can resolve a uart_id to its declaration.""" + uarts = uarts or [{CONF_ID: ID("uart_0")}, {CONF_ID: ID("uart_1")}] + full = Config() + full["uart"] = uarts + for index, uart_conf in enumerate(uarts): + full.declare_ids.append((uart_conf[CONF_ID], ["uart", index, CONF_ID])) + full.update(domains) + return full + + +def _bridge_config(uart_id: str, cdc_id: str) -> dict: + return {CONF_UART_ID: ID(uart_id), CONF_USB_CDC_ACM_ID: ID(cdc_id)} + + +def test_accepts_distinct_uart_and_cdc_interfaces( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + _final_validate(_bridge_config("uart_1", "cdc_acm_2")) + + +def test_rejects_two_bridges_sharing_a_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + with pytest.raises(cv.Invalid, match="already bridged"): + _final_validate(_bridge_config("uart_0", "cdc_acm_2")) + + +def test_rejects_two_bridges_sharing_a_cdc_interface( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3(set_core_config, full_config=_full_config()) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + with pytest.raises(cv.Invalid, match="already bridged"): + _final_validate(_bridge_config("uart_1", "cdc_acm_1")) + + +def test_rejects_uart_shared_with_another_component( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("uart_0")}], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_cdc_interface_shared_with_another_component( + set_core_config: SetCoreConfigCallable, +) -> None: + # The CDC instance is itself a uart::UARTComponent, so other components can bind + # it as a plain UART via uart_id -- that must be rejected just like UART sharing. + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("cdc_acm_1")}], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_uart_referenced_from_nested_config( + set_core_config: SetCoreConfigCallable, +) -> None: + # References can sit arbitrarily deep, e.g. inside an automation's action list. + _set_esp32_s3( + set_core_config, + full_config=_full_config( + binary_sensor=[ + { + "platform": "gpio", + "on_press": [{"then": [{CONF_UART_ID: ID("uart_0")}]}], + } + ], + ), + ) + with pytest.raises(cv.Invalid, match="exclusive"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_ignores_other_components_on_other_uarts( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + sensor=[{"platform": "pzemac", CONF_UART_ID: ID("uart_1")}], + # The bridge domain itself is skipped: this bridge's own entry (and any + # bridge-vs-bridge sharing, which the seen-set already rejects) must not + # trip the exclusivity scan. + bridge=[_bridge_config("uart_0", "cdc_acm_1")], + ), + ) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_rejects_debug_on_bridged_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + # The bridge talks to the IDF driver directly, so the uart debugger would see + # nothing and its dummy_receiver would steal RX bytes. + _set_esp32_s3( + set_core_config, + full_config=_full_config(uarts=[{CONF_ID: ID("uart_0"), CONF_DEBUG: {}}]), + ) + with pytest.raises(cv.Invalid, match="debug"): + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) + + +def test_allows_debug_on_other_uart( + set_core_config: SetCoreConfigCallable, +) -> None: + _set_esp32_s3( + set_core_config, + full_config=_full_config( + uarts=[{CONF_ID: ID("uart_0")}, {CONF_ID: ID("uart_1"), CONF_DEBUG: {}}] + ), + ) + _final_validate(_bridge_config("uart_0", "cdc_acm_1")) diff --git a/tests/component_tests/conftest.py b/tests/component_tests/conftest.py index 4f0b786cc2..b5eceeedf6 100644 --- a/tests/component_tests/conftest.py +++ b/tests/component_tests/conftest.py @@ -60,7 +60,7 @@ def reset_core() -> Generator[None]: @pytest.fixture(autouse=True) def reset_full_config() -> Generator[None]: """Give each test a clean final-validate config and restore it after.""" - token = final_validate.full_config.set({}) + token = final_validate.full_config.set(Config()) yield final_validate.full_config.reset(token) @@ -75,7 +75,7 @@ def set_core_config() -> Generator[SetCoreConfigCallable]: *, core_data: ConfigType | None = None, platform_data: ConfigType | None = None, - full_config: dict[str, ConfigType] | None = None, + full_config: dict[str, ConfigType] | Config | None = None, ) -> None: platform, framework = platform_framework.value @@ -94,7 +94,12 @@ def set_core_config() -> Generator[SetCoreConfigCallable]: CORE.data[platform.value] = platform_data config.path_context.set([]) - final_validate.full_config.set(full_config or Config()) + # Production always installs a Config (a FinalValidateConfig), never a plain dict. + if not isinstance(full_config, Config): + full = Config() + full.update(full_config or {}) + full_config = full + final_validate.full_config.set(full_config) yield setter diff --git a/tests/component_tests/types.py b/tests/component_tests/types.py index ee9d317339..3587517bde 100644 --- a/tests/component_tests/types.py +++ b/tests/component_tests/types.py @@ -4,6 +4,7 @@ from __future__ import annotations from typing import Protocol +from esphome.config import Config from esphome.const import PlatformFramework from esphome.types import ConfigType @@ -18,5 +19,5 @@ class SetCoreConfigCallable(Protocol): *, core_data: ConfigType | None = None, platform_data: ConfigType | None = None, - full_config: dict[str, ConfigType] | None = None, + full_config: dict[str, ConfigType] | Config | None = None, ) -> None: ... diff --git a/tests/components/cdc_acm_uart/common.yaml b/tests/components/cdc_acm_uart/common.yaml new file mode 100644 index 0000000000..6c43dfc18b --- /dev/null +++ b/tests/components/cdc_acm_uart/common.yaml @@ -0,0 +1,18 @@ +tinyusb: + id: tinyusb_test + usb_lang_id: 0x0123 + usb_manufacturer_str: ESPHomeTestManufacturer + usb_product_id: 0x1234 + usb_product_str: ESPHomeTestProduct + usb_serial_str: ESPHomeTestSerialNumber + usb_vendor_id: 0x2345 + +uart: + - id: uart_0 + tx_pin: 14 + rx_pin: 13 + baud_rate: 115200 + +usb_cdc_acm: + interfaces: + - id: cdc_acm_1 diff --git a/tests/components/cdc_acm_uart/common_dual.yaml b/tests/components/cdc_acm_uart/common_dual.yaml new file mode 100644 index 0000000000..0ce817fbc2 --- /dev/null +++ b/tests/components/cdc_acm_uart/common_dual.yaml @@ -0,0 +1,12 @@ +# Second UART/CDC pair for a two-bridge setup. Kept out of common.yaml because the +# ESP32-S2 has only two UART controllers and the logger occupies one, so a second +# uart there would fail at runtime. +uart: + - id: uart_1 + tx_pin: 15 + rx_pin: 16 + baud_rate: 115200 + +usb_cdc_acm: + interfaces: + - id: cdc_acm_2 diff --git a/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml new file mode 100644 index 0000000000..aa9ec8079f --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-p4-idf.yaml @@ -0,0 +1,15 @@ +packages: + cdc_acm_uart: !include common.yaml + cdc_acm_uart_dual: !include common_dual.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 + - platform: cdc_acm_uart + uart_id: uart_1 + usb_cdc_acm_id: cdc_acm_2 + dtr_pin: 20 + rts_pin: 21 diff --git a/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml new file mode 100644 index 0000000000..0beeb80bfa --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-s2-idf.yaml @@ -0,0 +1,14 @@ +# ESP32-S2 has no USB_SERIAL_JTAG, so the logger defaults to USB_CDC, which shares +# the USB OTG peripheral with tinyusb. Use a hardware UART for logging instead. +logger: + hardware_uart: UART0 + +packages: + cdc_acm_uart: !include common.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 diff --git a/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml b/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml new file mode 100644 index 0000000000..cbb1fc2a3a --- /dev/null +++ b/tests/components/cdc_acm_uart/test.esp32-s3-idf.yaml @@ -0,0 +1,17 @@ +packages: + cdc_acm_uart: !include common.yaml + cdc_acm_uart_dual: !include common_dual.yaml + +bridge: + - platform: cdc_acm_uart + uart_id: uart_0 + usb_cdc_acm_id: cdc_acm_1 + dtr_pin: 40 + rts_pin: 41 + - platform: cdc_acm_uart + uart_id: uart_1 + usb_cdc_acm_id: cdc_acm_2 + # GPIO19/20 are USB D-/D+ on the S3 (which the CDC side itself uses); use + # unrelated free pins here. + dtr_pin: 17 + rts_pin: 18